From 253fb880cbbafa0868db3e70708b17d72fc48dca Mon Sep 17 00:00:00 2001 From: wenhuanhuang Date: Fri, 10 Nov 2023 15:20:58 +0800 Subject: [PATCH] Initial xft v1.0.0 Signed-off-by: wenhuanhuang --- 3rdparty/INIReader.h | 501 + 3rdparty/prepare_oneccl.sh | 15 + CMakeLists.txt | 108 + CODE_OF_CONDUCT.md | 131 + CODING_STANDARDS.md | 56 + CONTRIBUTING.md | 116 + LICENSE | 201 + README.en.md | 36 - README.md | 289 +- SECURITY.md | 5 + benchmark/README.md | 34 + benchmark/baichuan2-13b/baichuan2-13b.sh | 31 + benchmark/baichuan2-13b/prompt_pool.json | 13 + benchmark/baichuan2-13b/run-baichuan2-13b.sh | 58 + benchmark/baichuan2-7b/baichuan2-7b.sh | 31 + benchmark/baichuan2-7b/prompt_pool.json | 13 + benchmark/baichuan2-7b/run-baichuan-7b.sh | 57 + benchmark/benchmark.py | 128 + benchmark/chatglm-6b/chatglm-6b.sh | 17 + benchmark/chatglm-6b/prompt_pool.json | 10 + benchmark/chatglm-6b/run-chatglm.sh | 43 + benchmark/chatglm2-6b/chatglm2-6b.sh | 19 + benchmark/chatglm2-6b/prompt_pool.json | 13 + benchmark/chatglm2-6b/run-chatglm2.sh | 43 + benchmark/llama-13b/llama-13b.sh | 17 + benchmark/llama-13b/prompt_pool.json | 10 + benchmark/llama-13b/run-llama-13b.sh | 44 + benchmark/llama-7b/llama-7b.sh | 17 + benchmark/llama-7b/prompt_pool.json | 10 + benchmark/llama-7b/run-llama-7b.sh | 43 + ci_build | 51 + cmake/cmdline.cmake | 26 + cmake/ig.cmake | 26 + cmake/jsoncpp.cmake | 27 + cmake/mklml.cmake | 26 + cmake/onednn.cmake | 26 + cmake/sentencepiece.cmake | 27 + cmake/xdnn.cmake | 38 + dockerfiles/Dockerfile | 57 + dockerfiles/Dockerfile.dev | 44 + dockerfiles/Dockerfile.release | 45 + examples/README.md | 15 + examples/cpp/CMakeLists.txt | 23 + examples/cpp/README.md | 32 + examples/cpp/example.cpp | 309 + examples/cpp/timer.h | 35 + examples/cpp/vocab_opt.cpp | 50267 +++++++++++++++++ examples/pytorch/README.md | 36 + examples/pytorch/demo.py | 91 + examples/web_demo/ChatGLM.py | 75 + examples/web_demo/ChatGLM2.py | 75 + examples/web_demo/Llama2.py | 83 + examples/web_demo/README.md | 37 + examples/web_demo/demo_utils.py | 133 + examples/web_demo/requirements.txt | 7 + include/abstract_decoder.h | 34 + include/abstract_searcher.h | 38 + include/dtype.h | 11 + include/layers_norm.h | 64 + include/models.h | 53 + include/xfastertransformer.h | 4 + pyproject.toml | 3 + requirements.txt | 5 + run_dev_docker.sh | 74 + setup.py | 53 + src/CMakeLists.txt | 39 + src/common/bfloat16.h | 163 + src/common/bit_cast.h | 18 + src/common/float16.h | 175 + src/common/kvcache_tensor.h | 146 + src/common/my_types.h | 356 + src/common/transformer_ctx.h | 179 + src/kernels/CMakeLists.txt | 6 + src/kernels/gemm_kernel_ext.cpp | 392 + src/kernels/gemm_kernel_ext.h | 12 + src/kernels/gemm_kernel_ext_fp16.cpp | 407 + src/kernels/layernorm_kernels.cpp | 142 + src/kernels/layernorm_kernels.h | 90 + src/kernels/rmsnorm_kernels.cpp | 138 + src/kernels/rmsnorm_kernels.h | 31 + src/layers/CMakeLists.txt | 6 + src/layers/alibi_embedding.cpp | 56 + src/layers/alibi_embedding.h | 41 + src/layers/attention.h | 619 + src/layers/attn_baichuan.h | 67 + src/layers/attn_chatglm.h | 28 + src/layers/attn_chatglm2.h | 167 + src/layers/decoder_layer.h | 124 + src/layers/dist_linear.h | 98 + src/layers/layer_norm.cpp | 52 + src/layers/layer_norm.h | 39 + src/layers/mlp_chatglm.h | 17 + src/layers/mlp_chatglm2.h | 86 + src/layers/mlp_llama.h | 211 + src/layers/mlp_standard.h | 217 + src/layers/opt_embedding.h | 89 + src/layers/rms_norm.cpp | 47 + src/layers/rms_norm.h | 87 + src/layers/rope_2d.cpp | 172 + src/layers/rope_2d.h | 21 + src/layers/rotary_embedding.cpp | 127 + src/layers/rotary_embedding.h | 31 + src/layers/rotary_embedding_chatglm2.cpp | 103 + src/layers/rotary_embedding_chatglm2.h | 32 + src/layers/token_embedding.h | 56 + src/models/CMakeLists.txt | 6 + src/models/baichuan.cpp | 146 + src/models/baichuan.h | 41 + src/models/chatglm.cpp | 220 + src/models/chatglm.h | 53 + src/models/chatglm2.cpp | 166 + src/models/chatglm2.h | 41 + src/models/common_decoder.h | 537 + src/models/hybrid_model.h | 75 + src/models/kvcache_manager.cpp | 187 + src/models/kvcache_manager.h | 47 + src/models/llama.cpp | 107 + src/models/llama.h | 26 + src/models/models.cpp | 160 + src/models/opt_decoder.cpp | 110 + src/models/opt_decoder.h | 34 + src/pytorch/CMakeLists.txt | 38 + src/pytorch/auto_model.h | 103 + src/pytorch/pytorch_warpper.cpp | 13 + src/searchers/CMakeLists.txt | 6 + src/searchers/beam_search.cpp | 531 + src/searchers/beam_search.h | 119 + src/searchers/greedy_search.cpp | 185 + src/searchers/greedy_search.h | 36 + src/searchers/sample_search.cpp | 211 + src/searchers/sample_search.h | 57 + src/searchers/searcher.h | 5 + src/utils/CMakeLists.txt | 7 + src/utils/bert_util.h | 64 + src/utils/compile_util.h | 36 + src/utils/debugger.h | 280 + src/utils/decoder_util.h | 718 + src/utils/matmul_helper.h | 1225 + src/utils/messenger.h | 165 + src/utils/numa_allocator.cpp | 34 + src/utils/numa_allocator.h | 17 + src/utils/shm_reduction.cpp | 75 + src/utils/shm_reduction.h | 118 + src/utils/split_util.h | 46 + src/utils/timeline.h | 179 + src/utils/transformer_util.h | 217 + src/utils/transpose_util.h | 347 + src/utils/weight_util.h | 138 + src/xfastertransformer/__init__.py | 6 + src/xfastertransformer/automodel.py | 83 + tests/CMakeLists.txt | 6 + tests/ut/CMakeLists.txt | 86 + tests/ut/alibi_embedding_test.cpp | 47 + tests/ut/beam_search_test.cpp | 122 + tests/ut/beam_search_test.h | 226 + tests/ut/gemm_kernel_ext_test.cpp | 120 + tests/ut/kv_reorder_test.cpp | 300 + tests/ut/messenger_test.cpp | 80 + tests/ut/rotary_embedding_test.cpp | 36 + tests/ut/shm_test.cpp | 99 + tests/ut/small_gemm_test.cpp | 131 + tests/ut/transpose_test.cpp | 50 + tools/README.md | 10 + tools/baichuan_convert.py | 218 + tools/chatglm2_convert.py | 259 + tools/chatglm_convert.py | 240 + tools/gptq/README.md | 36 + tools/gptq/gptq.py | 134 + tools/gptq/quantizer.py | 115 + tools/gptq/run_model_quant.py | 167 + tools/llama_convert.py | 228 + tools/opt_convert.py | 292 + 172 files changed, 68901 insertions(+), 62 deletions(-) create mode 100644 3rdparty/INIReader.h create mode 100755 3rdparty/prepare_oneccl.sh create mode 100644 CMakeLists.txt create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CODING_STANDARDS.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE delete mode 100644 README.en.md create mode 100644 SECURITY.md create mode 100644 benchmark/README.md create mode 100755 benchmark/baichuan2-13b/baichuan2-13b.sh create mode 100644 benchmark/baichuan2-13b/prompt_pool.json create mode 100755 benchmark/baichuan2-13b/run-baichuan2-13b.sh create mode 100755 benchmark/baichuan2-7b/baichuan2-7b.sh create mode 100644 benchmark/baichuan2-7b/prompt_pool.json create mode 100755 benchmark/baichuan2-7b/run-baichuan-7b.sh create mode 100644 benchmark/benchmark.py create mode 100755 benchmark/chatglm-6b/chatglm-6b.sh create mode 100644 benchmark/chatglm-6b/prompt_pool.json create mode 100755 benchmark/chatglm-6b/run-chatglm.sh create mode 100755 benchmark/chatglm2-6b/chatglm2-6b.sh create mode 100644 benchmark/chatglm2-6b/prompt_pool.json create mode 100755 benchmark/chatglm2-6b/run-chatglm2.sh create mode 100755 benchmark/llama-13b/llama-13b.sh create mode 100644 benchmark/llama-13b/prompt_pool.json create mode 100755 benchmark/llama-13b/run-llama-13b.sh create mode 100755 benchmark/llama-7b/llama-7b.sh create mode 100644 benchmark/llama-7b/prompt_pool.json create mode 100755 benchmark/llama-7b/run-llama-7b.sh create mode 100755 ci_build create mode 100644 cmake/cmdline.cmake create mode 100644 cmake/ig.cmake create mode 100644 cmake/jsoncpp.cmake create mode 100644 cmake/mklml.cmake create mode 100644 cmake/onednn.cmake create mode 100644 cmake/sentencepiece.cmake create mode 100644 cmake/xdnn.cmake create mode 100644 dockerfiles/Dockerfile create mode 100644 dockerfiles/Dockerfile.dev create mode 100644 dockerfiles/Dockerfile.release create mode 100644 examples/README.md create mode 100644 examples/cpp/CMakeLists.txt create mode 100644 examples/cpp/README.md create mode 100644 examples/cpp/example.cpp create mode 100644 examples/cpp/timer.h create mode 100644 examples/cpp/vocab_opt.cpp create mode 100644 examples/pytorch/README.md create mode 100644 examples/pytorch/demo.py create mode 100644 examples/web_demo/ChatGLM.py create mode 100644 examples/web_demo/ChatGLM2.py create mode 100644 examples/web_demo/Llama2.py create mode 100644 examples/web_demo/README.md create mode 100644 examples/web_demo/demo_utils.py create mode 100644 examples/web_demo/requirements.txt create mode 100644 include/abstract_decoder.h create mode 100644 include/abstract_searcher.h create mode 100644 include/dtype.h create mode 100644 include/layers_norm.h create mode 100644 include/models.h create mode 100644 include/xfastertransformer.h create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 run_dev_docker.sh create mode 100644 setup.py create mode 100644 src/CMakeLists.txt create mode 100644 src/common/bfloat16.h create mode 100644 src/common/bit_cast.h create mode 100644 src/common/float16.h create mode 100644 src/common/kvcache_tensor.h create mode 100644 src/common/my_types.h create mode 100644 src/common/transformer_ctx.h create mode 100644 src/kernels/CMakeLists.txt create mode 100644 src/kernels/gemm_kernel_ext.cpp create mode 100644 src/kernels/gemm_kernel_ext.h create mode 100644 src/kernels/gemm_kernel_ext_fp16.cpp create mode 100644 src/kernels/layernorm_kernels.cpp create mode 100644 src/kernels/layernorm_kernels.h create mode 100644 src/kernels/rmsnorm_kernels.cpp create mode 100644 src/kernels/rmsnorm_kernels.h create mode 100644 src/layers/CMakeLists.txt create mode 100644 src/layers/alibi_embedding.cpp create mode 100644 src/layers/alibi_embedding.h create mode 100644 src/layers/attention.h create mode 100644 src/layers/attn_baichuan.h create mode 100644 src/layers/attn_chatglm.h create mode 100644 src/layers/attn_chatglm2.h create mode 100644 src/layers/decoder_layer.h create mode 100644 src/layers/dist_linear.h create mode 100644 src/layers/layer_norm.cpp create mode 100644 src/layers/layer_norm.h create mode 100644 src/layers/mlp_chatglm.h create mode 100644 src/layers/mlp_chatglm2.h create mode 100644 src/layers/mlp_llama.h create mode 100644 src/layers/mlp_standard.h create mode 100644 src/layers/opt_embedding.h create mode 100644 src/layers/rms_norm.cpp create mode 100644 src/layers/rms_norm.h create mode 100644 src/layers/rope_2d.cpp create mode 100644 src/layers/rope_2d.h create mode 100644 src/layers/rotary_embedding.cpp create mode 100644 src/layers/rotary_embedding.h create mode 100644 src/layers/rotary_embedding_chatglm2.cpp create mode 100644 src/layers/rotary_embedding_chatglm2.h create mode 100644 src/layers/token_embedding.h create mode 100644 src/models/CMakeLists.txt create mode 100644 src/models/baichuan.cpp create mode 100644 src/models/baichuan.h create mode 100644 src/models/chatglm.cpp create mode 100644 src/models/chatglm.h create mode 100644 src/models/chatglm2.cpp create mode 100644 src/models/chatglm2.h create mode 100644 src/models/common_decoder.h create mode 100644 src/models/hybrid_model.h create mode 100644 src/models/kvcache_manager.cpp create mode 100644 src/models/kvcache_manager.h create mode 100644 src/models/llama.cpp create mode 100644 src/models/llama.h create mode 100644 src/models/models.cpp create mode 100644 src/models/opt_decoder.cpp create mode 100644 src/models/opt_decoder.h create mode 100644 src/pytorch/CMakeLists.txt create mode 100644 src/pytorch/auto_model.h create mode 100644 src/pytorch/pytorch_warpper.cpp create mode 100644 src/searchers/CMakeLists.txt create mode 100644 src/searchers/beam_search.cpp create mode 100644 src/searchers/beam_search.h create mode 100644 src/searchers/greedy_search.cpp create mode 100644 src/searchers/greedy_search.h create mode 100644 src/searchers/sample_search.cpp create mode 100644 src/searchers/sample_search.h create mode 100644 src/searchers/searcher.h create mode 100644 src/utils/CMakeLists.txt create mode 100644 src/utils/bert_util.h create mode 100644 src/utils/compile_util.h create mode 100644 src/utils/debugger.h create mode 100644 src/utils/decoder_util.h create mode 100644 src/utils/matmul_helper.h create mode 100644 src/utils/messenger.h create mode 100644 src/utils/numa_allocator.cpp create mode 100644 src/utils/numa_allocator.h create mode 100644 src/utils/shm_reduction.cpp create mode 100644 src/utils/shm_reduction.h create mode 100644 src/utils/split_util.h create mode 100644 src/utils/timeline.h create mode 100644 src/utils/transformer_util.h create mode 100644 src/utils/transpose_util.h create mode 100644 src/utils/weight_util.h create mode 100644 src/xfastertransformer/__init__.py create mode 100644 src/xfastertransformer/automodel.py create mode 100644 tests/CMakeLists.txt create mode 100644 tests/ut/CMakeLists.txt create mode 100644 tests/ut/alibi_embedding_test.cpp create mode 100644 tests/ut/beam_search_test.cpp create mode 100644 tests/ut/beam_search_test.h create mode 100644 tests/ut/gemm_kernel_ext_test.cpp create mode 100644 tests/ut/kv_reorder_test.cpp create mode 100644 tests/ut/messenger_test.cpp create mode 100644 tests/ut/rotary_embedding_test.cpp create mode 100644 tests/ut/shm_test.cpp create mode 100644 tests/ut/small_gemm_test.cpp create mode 100644 tests/ut/transpose_test.cpp create mode 100644 tools/README.md create mode 100644 tools/baichuan_convert.py create mode 100644 tools/chatglm2_convert.py create mode 100644 tools/chatglm_convert.py create mode 100644 tools/gptq/README.md create mode 100644 tools/gptq/gptq.py create mode 100644 tools/gptq/quantizer.py create mode 100644 tools/gptq/run_model_quant.py create mode 100644 tools/llama_convert.py create mode 100644 tools/opt_convert.py diff --git a/3rdparty/INIReader.h b/3rdparty/INIReader.h new file mode 100644 index 0000000..7d40f06 --- /dev/null +++ b/3rdparty/INIReader.h @@ -0,0 +1,501 @@ +// Read an INI file into easy-to-access name/value pairs. + +// inih and INIReader are released under the New BSD license. +// Go to the project home page for more info: +// +// https://github.com/benhoyt/inih (Initial repo) +// https://github.com/jtilly/inih (The reference of this header file) +/* inih -- simple .INI file parser +inih is released under the New BSD license (see LICENSE.txt). Go to the project +home page for more info: +https://github.com/benhoyt/inih +https://github.com/jtilly/inih +*/ + +#ifndef __INI_H__ +#define __INI_H__ + +/* Make this header file easier to include in C++ code */ +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Typedef for prototype of handler function. */ +typedef int (*ini_handler)(void* user, const char* section, + const char* name, const char* value); + +/* Typedef for prototype of fgets-style reader function. */ +typedef char* (*ini_reader)(char* str, int num, void* stream); + +/* Parse given INI-style file. May have [section]s, name=value pairs + (whitespace stripped), and comments starting with ';' (semicolon). Section + is "" if name=value pair parsed before any section heading. name:value + pairs are also supported as a concession to Python's configparser. + For each name=value pair parsed, call handler function with given user + pointer as well as section, name, and value (data only valid for duration + of handler call). Handler should return nonzero on success, zero on error. + Returns 0 on success, line number of first error on parse error (doesn't + stop on first error), -1 on file open error, or -2 on memory allocation + error (only when INI_USE_STACK is zero). +*/ +int ini_parse(const char* filename, ini_handler handler, void* user); + +/* Same as ini_parse(), but takes a FILE* instead of filename. This doesn't + close the file when it's finished -- the caller must do that. */ +int ini_parse_file(FILE* file, ini_handler handler, void* user); + +/* Same as ini_parse(), but takes an ini_reader function pointer instead of + filename. Used for implementing custom or string-based I/O. */ +int ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, + void* user); + +/* Nonzero to allow multi-line value parsing, in the style of Python's + configparser. If allowed, ini_parse() will call the handler with the same + name for each subsequent line parsed. */ +#ifndef INI_ALLOW_MULTILINE +#define INI_ALLOW_MULTILINE 1 +#endif + +/* Nonzero to allow a UTF-8 BOM sequence (0xEF 0xBB 0xBF) at the start of + the file. See http://code.google.com/p/inih/issues/detail?id=21 */ +#ifndef INI_ALLOW_BOM +#define INI_ALLOW_BOM 1 +#endif + +/* Nonzero to allow inline comments (with valid inline comment characters + specified by INI_INLINE_COMMENT_PREFIXES). Set to 0 to turn off and match + Python 3.2+ configparser behaviour. */ +#ifndef INI_ALLOW_INLINE_COMMENTS +#define INI_ALLOW_INLINE_COMMENTS 1 +#endif +#ifndef INI_INLINE_COMMENT_PREFIXES +#define INI_INLINE_COMMENT_PREFIXES ";" +#endif + +/* Nonzero to use stack, zero to use heap (malloc/free). */ +#ifndef INI_USE_STACK +#define INI_USE_STACK 1 +#endif + +/* Stop parsing on first error (default is to keep parsing). */ +#ifndef INI_STOP_ON_FIRST_ERROR +#define INI_STOP_ON_FIRST_ERROR 0 +#endif + +/* Maximum line length for any line in INI file. */ +#ifndef INI_MAX_LINE +#define INI_MAX_LINE 200 +#endif + +#ifdef __cplusplus +} +#endif + +/* inih -- simple .INI file parser +inih is released under the New BSD license (see LICENSE.txt). Go to the project +home page for more info: +https://github.com/benhoyt/inih +*/ + +#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif + +#include +#include +#include + +#if !INI_USE_STACK +#include +#endif + +#define MAX_SECTION 50 +#define MAX_NAME 50 + +/* Strip whitespace chars off end of given string, in place. Return s. */ +inline static char* rstrip(char* s) +{ + char* p = s + strlen(s); + while (p > s && isspace((unsigned char)(*--p))) + *p = '\0'; + return s; +} + +/* Return pointer to first non-whitespace char in given string. */ +inline static char* lskip(const char* s) +{ + while (*s && isspace((unsigned char)(*s))) + s++; + return (char*)s; +} + +/* Return pointer to first char (of chars) or inline comment in given string, + or pointer to null at end of string if neither found. Inline comment must + be prefixed by a whitespace character to register as a comment. */ +inline static char* find_chars_or_comment(const char* s, const char* chars) +{ +#if INI_ALLOW_INLINE_COMMENTS + int was_space = 0; + while (*s && (!chars || !strchr(chars, *s)) && + !(was_space && strchr(INI_INLINE_COMMENT_PREFIXES, *s))) { + was_space = isspace((unsigned char)(*s)); + s++; + } +#else + while (*s && (!chars || !strchr(chars, *s))) { + s++; + } +#endif + return (char*)s; +} + +/* Version of strncpy that ensures dest (size bytes) is null-terminated. */ +inline static char* strncpy0(char* dest, const char* src, size_t size) +{ + strncpy(dest, src, size); + dest[size - 1] = '\0'; + return dest; +} + +/* See documentation in header file. */ +inline int ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, + void* user) +{ + /* Uses a fair bit of stack (use heap instead if you need to) */ +#if INI_USE_STACK + char line[INI_MAX_LINE]; +#else + char* line; +#endif + char section[MAX_SECTION] = ""; + char prev_name[MAX_NAME] = ""; + + char* start; + char* end; + char* name; + char* value; + int lineno = 0; + int error = 0; + +#if !INI_USE_STACK + line = (char*)malloc(INI_MAX_LINE); + if (!line) { + return -2; + } +#endif + + /* Scan through stream line by line */ + while (reader(line, INI_MAX_LINE, stream) != NULL) { + lineno++; + + start = line; +#if INI_ALLOW_BOM + if (lineno == 1 && (unsigned char)start[0] == 0xEF && + (unsigned char)start[1] == 0xBB && + (unsigned char)start[2] == 0xBF) { + start += 3; + } +#endif + start = lskip(rstrip(start)); + + if (*start == ';' || *start == '#') { + /* Per Python configparser, allow both ; and # comments at the + start of a line */ + } +#if INI_ALLOW_MULTILINE + else if (*prev_name && *start && start > line) { + +#if INI_ALLOW_INLINE_COMMENTS + end = find_chars_or_comment(start, NULL); + if (*end) + *end = '\0'; + rstrip(start); +#endif + + /* Non-blank line with leading whitespace, treat as continuation + of previous name's value (as per Python configparser). */ + if (!handler(user, section, prev_name, start) && !error) + error = lineno; + } +#endif + else if (*start == '[') { + /* A "[section]" line */ + end = find_chars_or_comment(start + 1, "]"); + if (*end == ']') { + *end = '\0'; + strncpy0(section, start + 1, sizeof(section)); + *prev_name = '\0'; + } + else if (!error) { + /* No ']' found on section line */ + error = lineno; + } + } + else if (*start) { + /* Not a comment, must be a name[=:]value pair */ + end = find_chars_or_comment(start, "=:"); + if (*end == '=' || *end == ':') { + *end = '\0'; + name = rstrip(start); + value = lskip(end + 1); +#if INI_ALLOW_INLINE_COMMENTS + end = find_chars_or_comment(value, NULL); + if (*end) + *end = '\0'; +#endif + rstrip(value); + + /* Valid name[=:]value pair found, call handler */ + strncpy0(prev_name, name, sizeof(prev_name)); + if (!handler(user, section, name, value) && !error) + error = lineno; + } + else if (!error) { + /* No '=' or ':' found on name[=:]value line */ + error = lineno; + } + } + +#if INI_STOP_ON_FIRST_ERROR + if (error) + break; +#endif + } + +#if !INI_USE_STACK + free(line); +#endif + + return error; +} + +/* See documentation in header file. */ +inline int ini_parse_file(FILE* file, ini_handler handler, void* user) +{ + return ini_parse_stream((ini_reader)fgets, file, handler, user); +} + +/* See documentation in header file. */ +inline int ini_parse(const char* filename, ini_handler handler, void* user) +{ + FILE* file; + int error; + + file = fopen(filename, "r"); + if (!file) + return -1; + error = ini_parse_file(file, handler, user); + fclose(file); + return error; +} + +#endif /* __INI_H__ */ + + +#ifndef __INIREADER_H__ +#define __INIREADER_H__ + +#include +#include +#include + +// Read an INI file into easy-to-access name/value pairs. (Note that I've gone +// for simplicity here rather than speed, but it should be pretty decent.) +class INIReader +{ +public: + // Empty Constructor + INIReader() {}; + + // Construct INIReader and parse given filename. See ini.h for more info + // about the parsing. + INIReader(std::string filename); + + // Construct INIReader and parse given file. See ini.h for more info + // about the parsing. + INIReader(FILE *file); + ~INIReader(); + // Return the result of ini_parse(), i.e., 0 on success, line number of + // first error on parse error, or -1 on file open error. + int ParseError() const; + + // Return the list of sections found in ini file + const std::set& Sections() const; + + // Get a string value from INI file, returning default_value if not found. + std::string Get(std::string section, std::string name, + std::string default_value) const; + std::string Get(std::string section, std::string name) const; + + // Get an integer (long) value from INI file, returning default_value if + // not found or not a valid integer (decimal "1234", "-1234", or hex "0x4d2"). + long GetInteger(std::string section, std::string name, long default_value) const; + long GetInteger(std::string section, std::string name) const; + + // Get a real (floating point double) value from INI file, returning + // default_value if not found or not a valid floating point value + // according to strtod(). + double GetReal(std::string section, std::string name, double default_value) const; + + // Get a single precision floating point number value from INI file, returning + // default_value if not found or not a valid floating point value + // according to strtof(). + float GetFloat(std::string section, std::string name, float default_value) const; + float GetFloat(std::string section, std::string name) const; + + // Get a boolean value from INI file, returning default_value if not found or if + // not a valid true/false value. Valid true values are "true", "yes", "on", "1", + // and valid false values are "false", "no", "off", "0" (not case sensitive). + bool GetBoolean(std::string section, std::string name, bool default_value) const; + +protected: + int _error; + std::map _values; + std::set _sections; + static std::string MakeKey(std::string section, std::string name); + static int ValueHandler(void* user, const char* section, const char* name, + const char* value); +}; + +#endif // __INIREADER_H__ + + +#ifndef __INIREADER__ +#define __INIREADER__ + +#include +#include +#include + +inline INIReader::INIReader(std::string filename) +{ + _error = ini_parse(filename.c_str(), ValueHandler, this); +} + +inline INIReader::INIReader(FILE *file) +{ + _error = ini_parse_file(file, ValueHandler, this); +} + +inline int INIReader::ParseError() const +{ + return _error; +} + +inline INIReader::~INIReader() { } + +inline const std::set& INIReader::Sections() const +{ + return _sections; +} + +inline std::string INIReader::Get(std::string section, std::string name, std::string default_value) const +{ + std::string key = MakeKey(section, name); + return _values.count(key) ? _values.at(key) : default_value; +} + +inline std::string INIReader::Get(std::string section, std::string name) const +{ + std::string key = MakeKey(section, name); + if(_values.count(key)) return _values.at(key); + else + { + printf("[ERROR] Does not find the section %s with name %s. \n", section.c_str(), name.c_str()); + exit(-1); + } +} + +inline long INIReader::GetInteger(std::string section, std::string name, long default_value) const +{ + std::string valstr = Get(section, name, ""); + const char* value = valstr.c_str(); + char* end; + // This parses "1234" (decimal) and also "0x4D2" (hex) + long n = strtol(value, &end, 0); + return end > value ? n : default_value; +} + +inline long INIReader::GetInteger(std::string section, std::string name) const +{ + std::string valstr = Get(section, name, ""); + const char* value = valstr.c_str(); + char* end; + // This parses "1234" (decimal) and also "0x4D2" (hex) + long n = strtol(value, &end, 0); + if(end <= value) + { + printf("[ERROR] Does not find the section %s with name %s. \n", section.c_str(), name.c_str()); + exit(-1); + } + return n; +} + +inline double INIReader::GetReal(std::string section, std::string name, double default_value) const +{ + std::string valstr = Get(section, name, ""); + const char* value = valstr.c_str(); + char* end; + double n = strtod(value, &end); + return end > value ? n : default_value; +} + +inline float INIReader::GetFloat(std::string section, std::string name, float default_value) const +{ + std::string valstr = Get(section, name, ""); + const char* value = valstr.c_str(); + char* end; + float n = strtof(value, &end); + return end > value ? n : default_value; +} + +inline float INIReader::GetFloat(std::string section, std::string name) const +{ + std::string valstr = Get(section, name, ""); + const char* value = valstr.c_str(); + char* end; + float n = strtof(value, &end); + if(end <= value) + { + printf("[ERROR] Does not find the section %s with name %s. \n", section.c_str(), name.c_str()); + exit(-1); + } + return n; +} + +inline bool INIReader::GetBoolean(std::string section, std::string name, bool default_value) const +{ + std::string valstr = Get(section, name, ""); + // Convert to lower case to make string comparisons case-insensitive + std::transform(valstr.begin(), valstr.end(), valstr.begin(), ::tolower); + if (valstr == "true" || valstr == "yes" || valstr == "on" || valstr == "1") + return true; + else if (valstr == "false" || valstr == "no" || valstr == "off" || valstr == "0") + return false; + else + return default_value; +} + +inline std::string INIReader::MakeKey(std::string section, std::string name) +{ + std::string key = section + "=" + name; + // Convert to lower case to make section/name lookups case-insensitive + std::transform(key.begin(), key.end(), key.begin(), ::tolower); + return key; +} + +inline int INIReader::ValueHandler(void* user, const char* section, const char* name, + const char* value) +{ + INIReader* reader = (INIReader*)user; + std::string key = MakeKey(section, name); + if (reader->_values[key].size() > 0) + reader->_values[key] += "\n"; + reader->_values[key] += value; + reader->_sections.insert(section); + return 1; +} + +#endif // __INIREADER__ \ No newline at end of file diff --git a/3rdparty/prepare_oneccl.sh b/3rdparty/prepare_oneccl.sh new file mode 100755 index 0000000..bf63919 --- /dev/null +++ b/3rdparty/prepare_oneccl.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Prepare the dependency lib oneCCL +dir="oneCCL" +if [ ! -d "$dir" ]; then + git clone https://github.com/oneapi-src/oneCCL.git $dir + if [ -d "$dir" ]; then + cd $dir + git checkout 2021.9 + sed -i 's/cpu_gpu_dpcpp/./g' cmake/templates/oneCCLConfig.cmake.in + mkdir build && cd build + cmake .. + make -j install + fi +fi diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..3e4739d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,108 @@ +# Copyright (C) 2023-2024 Intel Corporation +cmake_minimum_required(VERSION 3.15.1) +project(xfastertransformer LANGUAGES C CXX) + +# Get gcc version +execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpfullversion + OUTPUT_VARIABLE GCC_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) +message(">>> GCC version: ${GCC_VERSION}") + +find_package(MPI REQUIRED) +find_package(oneCCL REQUIRED) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -fopenmp -mavx512f -mavx512bw -mavx512vl -fPIC -D_GLIBCXX_USE_CXX11_ABI=0") + +if(GCC_VERSION VERSION_GREATER_EQUAL "12.3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512bf16") +endif() + +if(CMAKE_BUILD_TYPE MATCHES "Debug") + message("Notice: Using Debug mode.") + set(CMAKE_C_FLAGS "-O0 -g") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g") +else() + set(CMAKE_C_FLAGS "-O2") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2") +endif() + +### Require out-of-source builds +file(TO_CMAKE_PATH "${PROJECT_BINARY_DIR}/CMakeLists.txt" LOC_PATH) +if(EXISTS "${LOC_PATH}") + message(FATAL_ERROR + "You cannot build in a source directory (or any directory with a CMakeLists.txt file). Please make a build subdirectory. Feel free to remove CMakeCache.txt and CMakeFiles." + ) +endif() + +include("cmake/mklml.cmake") +include("cmake/onednn.cmake") +include("cmake/ig.cmake") +include(GNUInstallDirs) + +include_directories(${CMAKE_SOURCE_DIR}/3rdparty/) +include_directories(${CMAKE_SOURCE_DIR}/3rdparty/mklml/include) +include_directories(${CMAKE_SOURCE_DIR}/3rdparty/onednn/include) +include_directories(${CMAKE_SOURCE_DIR}/3rdparty/onednn/build/include) +include_directories(${CMAKE_SOURCE_DIR}/3rdparty/ig) +include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${CMAKE_SOURCE_DIR}/src/kernels) +include_directories(${CMAKE_SOURCE_DIR}/src/layers) +include_directories(${CMAKE_SOURCE_DIR}/src/searchers) +include_directories(${CMAKE_SOURCE_DIR}/src/utils) +include_directories(${CMAKE_SOURCE_DIR}/src/models) +include_directories(${CMAKE_SOURCE_DIR}/src/common) + +link_directories(${CMAKE_SOURCE_DIR}/src/kernels) +link_directories(${CMAKE_SOURCE_DIR}/3rdparty/mklml/lib) +link_directories(${CMAKE_SOURCE_DIR}/3rdparty/onednn/build/src) +link_directories(${CMAKE_SOURCE_DIR}/3rdparty/ig) + +set(3RDPART_LIB_LIST "MPI::MPI_CXX" "ccl" "dnnl" "numa") +set(DEPEND_LIST "onednn" "mklml" "ig_lib") + +option(BUILD_WITH_SHARED_LIBS "Build with shared libraries" OFF) +if(BUILD_WITH_SHARED_LIBS) + message("Building with shared libraries.") + list(APPEND 3RDPART_LIB_LIST "ig") +else() + message("Building with static libraries.") + list(APPEND 3RDPART_LIB_LIST "ig_static") +endif() + +# Enable AVX512_FP16 optimization +# add_definitions(-DAVX512_FP32_WEIGHT_ONLY_FP16=true) +add_definitions(-DAVX512_FP16_WEIGHT_ONLY_FP16=true) +add_definitions(-DAVX512_BF16_WEIGHT_ONLY_BF16=true) +# add_definitions(-DAVX512_FP32_WEIGHT_ONLY_INT8=true) +add_definitions(-DAVX512_FP16_WEIGHT_ONLY_INT8=true) +# add_definitions(-DDEBUG=true) +# add_definitions(-DSTEP_BY_STEP_ATTN=true) +# add_definitions(-DTIMELINE=true) +add_definitions(-DUSE_SHM=true) +option(XFT_BUILD_TESTS "Build xfastertransformer unit tests" OFF) + +get_directory_property(MYDEFS COMPILE_DEFINITIONS) + +if(MYDEFS MATCHES "^TIMELINE=" OR MYDEFS MATCHES ";TIMELINE=") + include("cmake/jsoncpp.cmake") + include_directories(${CMAKE_SOURCE_DIR}/3rdparty/jsoncpp/include) + link_directories(${CMAKE_SOURCE_DIR}/3rdparty/jsoncpp/${CMAKE_INSTALL_LIBDIR}) + list(APPEND 3RDPART_LIB_LIST "jsoncpp") + list(APPEND DEPEND_LIST "jsoncpp_lib") +endif() + +# .so file +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) +# exe file +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) + +add_subdirectory(src) +add_subdirectory(examples/cpp) + +add_subdirectory(tests) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..58dba18 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,131 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +CommunityCodeOfConduct AT intel DOT com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq diff --git a/CODING_STANDARDS.md b/CODING_STANDARDS.md new file mode 100644 index 0000000..a0f061a --- /dev/null +++ b/CODING_STANDARDS.md @@ -0,0 +1,56 @@ +# Coding Standards + +## Introduction + +The Coding Standards outline and describe a set of rules the xFasterTransformer code base +and code contributions must follow. + +The only goal of the Coding Standards is to maintain productivity for the +development of the library. + +The Coding Standards are subject to change. + +--- +**NOTE** It may happen that not all code can and does follow all the rules +outlined in the Coding Standards. In case enforcing a specific rule makes your +contribution less effective in terms of code design, please do not apply the +corresponding rule. + +--- + +### Coding style + +The coding style consistency in xFasterTransformer is maintained using `clang-format`. +When submitting your contribution, please make sure that it adheres to the existing +coding style with the following command: +```sh +clang-format -style=file -i foo.cpp +``` +This will format the code using the `_clang_format` file found in the xFasterTransformer +top level directory. + +Coding style is secondary to the general code design. + +### General + +- Use properly named constants whenever possible (unless this code is + auto-generated). + * For example, + ~~~cpp + if (x < 4) y = 64; + ~~~ + + In this example, 4 and 64 should be named, in which case the code becomes: + ~~~cpp + if (x < sizeof(float)) y = cache_line_size; + ~~~ + +- Don't use `using namespace XXX` in header files. + +- Avoid code duplication (look for similarities), unless it is necessary. + +- Declare variables in the innermost possible scope until there are some + circumstances that make you declare them somewhere else. + +- Consider using utils to improve readability (`IMPLICATION`, `one_of`, + `everyone_is`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ac43a1d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,116 @@ +# Contributing guidelines + +If you have improvements to the xFasterTransformer code, please send us your pull +requests! To get started, see the GitHub +[howto](https://help.github.com/en/articles/about-pull-requests). + +You can: + +- Submit your changes directly with a + [pull request](https://github.com/intel/xFasterTransformer/pulls) +- Log a bug or feedback with an [issue](https://github.com/intel/xFasterTransformer/issues) + +**See also:** [Contributor Covenant](CODE_OF_CONDUCT.md) code of conduct. + +## Pull request checklist + +Before sending your pull requests, make sure that you have followed this list: + +* Check the [library functionality guidelines](CONTRIBUTING.md#library-functionality-guidelines). + + +* Ensure that the changes are consistent with the + [code contribution guidelines](CONTRIBUTING.md#code-contribution-guidelines) + and [coding standards](CONTRIBUTING.md#coding-standards). + +* Check that [unit tests](CONTRIBUTING.md#unit-tests) pass. + +## Library functionality guidelines + +xFasterTransformer focuses on functionality that satisfies all of the following +criteria: + +1. *Performance*: the functionality has material impact on a workload level. + In other words, this means that for a optimization it should be + demonstrated that it brings visible performance improvement to some + workload. + +2. *Generality*: when introducing new foundational features, their API should + be sufficiently versatile and user-friendly to facilitate integration into + other frameworks. + +3. *Complexity*: it is not trivial to implement the functionality directly in + a LLM application. + +## Code contribution guidelines + +When submitting your contribution, please make sure that it is: + +* *Tested*: xFasterTransformer uses gtests for lightweight functional testing. Please make + your contribution is fully tested by unit tested. + +* *Documented*: Please add essential inline comments to aid others in comprehending the + code. When necessary, include appropriate documentation explanations. + +All code in xFasterTransformer gets promoted to product branches (`main` and `rls-`) +only through GitHub pull requests. Requirements for promotion: + +- The request is reviewed and approved by maintainers for all affected + components. +- All discussions in the pull request are resolved. +- Continuous integration pipeline passed without errors. +- Promotion to release (`rls-`) branches can be done only by maintainers + (enforced by GitHub) +- The pull request author is responsible for collecting all the necessary + approvals, rebasing of the changes, and resolving the discussions. + +To simplify the work of reviewers, make sure that the commits in the pull +request adhere to the following requirements: + +- Commit message should be fit into 50 (at most 72) characters and have the + imperative mood. +- Commit message should follow the format: + `[Scope][Scope: ..] ` + Scope examples: + * Top level: `Build`, `API`, `Doc`, `Tests`, `Common`, `Models`, `Kernels` + * Second level: `BF16`, `Layers`, `Utils`, `Searchers` + * Example commit message: +~~~git +[Kernels][BF16]: Add AMX format BA16a64b2a. +~~~ + +- Commit body should also fit 72 characters. Think of it as a standard e-mail + body or a markdown document in terms of styling - write sentences from the + very left border keeping capital letters and punctuation in place. +- xFasterTransformer branches maintain linear history. Rebase the changes on top of target + branch before creating a pull request. Rebase again after resolving all the + discussions, as well as in case of merge conflicts. +- Use `git add -p` and `git rebase -i` liberally to split unrelated changes + into multiple self-contained commits. This is a courtesy to reviewers: smaller + commits are easier to comprehend. It also helps with bisecting in the future. + Of course judgement needs to be applied whether to split changes or not. For + example, split code cleanup and the actual fix into two separate patches. + +## Coding Standards + +Contributions to xFasterTransformer must follow the [Coding Standards](CODING_STANDARDS.md) +in order to simplify development and review processes. The general principle is +to follow the style of existing/surrounding code. + +The Coding Standards are subject to change and contributions to the Coding +Standards are welcome. + +If you wish to propose changes to the Coding Standards (including `clang-format` + options), please submit the proposal via an pull request. The proposal should + contain the following information: +* *Motivation*: Why should the proposed standard be introduced and applied? +* *Enforcement*: Can the proposed standard be applied via an automated process + or other practical means? +* *Example*: What does the code base look like with the proposed standard + applied? + +## Unit tests + +xFasterTransformer uses gtests for lightweight functional testing. + +Be sure to extend the existing tests when fixing an issue. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.en.md b/README.en.md deleted file mode 100644 index 197866a..0000000 --- a/README.en.md +++ /dev/null @@ -1,36 +0,0 @@ -# xFasterTransformer - -#### Description -{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} - -#### Software Architecture -Software architecture description - -#### Installation - -1. xxxx -2. xxxx -3. xxxx - -#### Instructions - -1. xxxx -2. xxxx -3. xxxx - -#### Contribution - -1. Fork the repository -2. Create Feat_xxx branch -3. Commit your code -4. Create Pull Request - - -#### Gitee Feature - -1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md -2. Gitee blog [blog.gitee.com](https://blog.gitee.com) -3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) -4. The most valuable open source project [GVP](https://gitee.com/gvp) -5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) -6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/README.md b/README.md index 3bcab96..c3ec70d 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,276 @@ # xFasterTransformer -#### 介绍 -{**以下是 Gitee 平台说明，您可以替换此简介** -Gitee 是 OSCHINA 推出的基于 Git 的代码托管平台（同时支持 SVN）。专为开发者提供稳定、高效、安全的云端软件开发协作平台 -无论是个人、团队、或是企业，都能够用 Gitee 实现代码托管、项目管理、协作开发。企业项目请看 [https://gitee.com/enterprises](https://gitee.com/enterprises)} +xFasterTransformer is an exceptionally optimized solution for large language models (LLM) on the X86 platform, which is similar to FasterTransformer on the GPU platform. xFasterTransformer is able to operate in distributed mode across multiple sockets and nodes to support inference on larger models. Additionally, it provides both C++ and Python APIs, spanning from high-level to low-level interfaces, making it easy to adopt and integrate. -#### 软件架构 -软件架构说明 +## Table of Contents +- [xFasterTransformer](#xfastertransformer) + - [Table of Contents](#table-of-contents) + - [Models overview](#models-overview) + - [Support matrix](#support-matrix) + - [Documents](#documents) + - [Installation](#installation) + - [From PyPI](#from-pypi) + - [Using Docker](#using-docker) + - [Built from source](#built-from-source) + - [Prepare Environment](#prepare-environment) + - [Manually](#manually) + - [Docker(Recommended)](#dockerrecommended) + - [How to build](#how-to-build) + - [Model Preparation](#prepare-model) + - [API usage](#api-usage) + - [Python API(PyTorch)](#python-apipytorch) + - [C++ API](#c-api) + - [How to run](#how-to-run) + - [Single rank](#single-rank) + - [Multi ranks](#multi-ranks) + - [Command line](#command-line) + - [Code](#code) + - [Python](#python) + - [C++](#c) + - [Web Demo](#web-demo) + - [Benchmark](#benchmark) +## Models overview +Large Language Models (LLMs) develops very fast and are more widely used in many AI scenarios. xFasterTransformer is an optimized solution for LLM inference using the mainstream and popular LLM models on Xeon. xFasterTransformer fully leverages the hardware capabilities of Xeon platforms to achieve the high performance and high scalability of LLM inference both on single socket and multiple sockets/multiple nodes. -#### 安装教程 +xFasterTransformer provides a series of APIs, both of C++ and Python, for end users to integrate xFasterTransformer into their own solutions or services directly. Many kinds of example codes are also provided to demonstrate the usage. Benchmark codes and scripts are provided for users to show the performance. Web demos for popular LLM models are also provided. -1. xxxx -2. xxxx -3. xxxx -#### 使用说明 +### Support matrix -1. xxxx -2. xxxx -3. xxxx +| Models | Framework | | Distribution | DataType | | | | | +| :------: | :-------: | :------: | :----------: | :------: | :------: | :------: | :-------: | :-------: | +| | PyTorch | C++ | | FP16 | BF16 | INT8 | BF16+FP16 | BF16+INT8 | +| ChatGLM | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| ChatGLM2 | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| Llama | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| Llama2 | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| Opt | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -#### 参与贡献 +## Documents +xFasterTransformer Documents provides the following resources: +- An introduction to xFasterTransformer. +- Comprehensive API references for both high-level and low-level interfaces in C++ and PyTorch. +- Practical API usage examples for xFasterTransformer in both C++ and PyTorch. -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request +## Installation +### From PyPI +```bash +pip install xfastertransformer +``` +### Using Docker +```bash +docker pull intel/xfastertransformer:latest +``` -#### 特技 +### Built from source +#### Prepare Environment +##### Manually +- [oneCCL](https://github.com/oneapi-src/oneCCL) + - Use provided scripts to build it from source code. + ```bash + cd 3rdparty + sh prepare_oneccl.sh + source ./oneCCL/build/_install/env/setvars.sh + ``` + - Install oneCCL through installing [Intel® oneAPI Base Toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html). +- [PyTorch](https://pytorch.org/get-started/locally/) v2.0+ (When using the PyTorch API, it's required, but it's not needed when using the C++ API.) + ```bash + pip install torch --index-url https://download.pytorch.org/whl/cpu + ``` -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +##### Docker(Recommended) +- Pull docker image from dockerhub + ```bash + docker pull intel/xfastertransformer:dev-ubuntu22.04 + ``` +- Build docker image from Dockerfile + ```bash + docker build \ + -f dockerfiles/Dockerfile \ + --build-arg "HTTP_PROXY=${http_proxy}" \ + --build-arg "HTTPS_PROXY=${https_proxy}" \ + -t intel/xfastertransformer:dev-ubuntu22.04 . + ``` +Then run the docker with the command or bash script in repo (Assume model files are in `/data/` directory): +```bash +# A new image will be created to ensure both the user and file directories are consistent with the host if the user is not root. +bash run_dev_docker.sh + +# or run docker manually by following command. +docker run -it \ + --name xfastertransformer-dev \ + --privileged \ + --shm-size=16g \ + -v "${PWD}":/root/xfastertransformer \ + -v /data/:/data/ \ + -w /root/xfastertransformer \ + -e "http_proxy=$http_proxy" \ + -e "https_proxy=$https_proxy" \ + intel/xfastertransformer:dev-ubuntu22.04 +``` +**Notice!!!**: Please enlarge `--shm-size` if **bus error** occurred while running in the multi-ranks mode . The default docker limits the shared memory size to 64MB and our implementation uses many shared memories to achieve a better performance. + +##### How to build +- Using 'CMake' + ```bash + # Build xFasterTransformer + git clone https://github.com/intel/xFasterTransformer.git xFasterTransformer + cd xFasterTransformer + # Please make sure torch is installed when run python example + mkdir build && cd build + cmake .. + make -j + ``` +- Using 'python setup.py' + ```bash + # Build xFasterTransformer library and C++ example. + python setup.py build + + # Install xFasterTransformer into pip environment. + python setup.py install + ``` + +## [Models Preparation](tools/README.md) +xFasterTransformer supports a different model format from Huggingface, but it's compatible with FasterTransformer's format. +1. Download the huggingface format model firstly. +2. After that, convert the model into xFasterTransformer format using the script in 'tools' folder. You will see many bin files in the output directory. +```bash + python ./tools/chatglm_convert.py -i ${HF_DATASET_DIR} -o ${OUTPUT_DIR} + +``` + +## API usage +For more details, please see API document and [examples](examples/README.md). +### Python API(PyTorch) +Firstly, please install the dependencies. +- Python dependencies + ```bash + pip install -r requirements.txt + ``` +- oneCCL + Install oneCCL and setup the environment. Please refer to [Prepare Environment](#prepare-environment). + + +xFasterTransformer's Python API is similar to transformers and also supports transformers's streamer to achieve the streaming output. In the example, we use transformers to encode input prompts to token ids. +```Python +import xfastertransformer +from transformers import AutoTokenizer, TextStreamer +# Assume huggingface model dir is `/data/chatglm-6b-hf` and converted model dir is `/data/chatglm-6b-cpu`. +MODEL_PATH="/data/chatglm-6b-cpu" +TOKEN_PATH="/data/chatglm-6b-hf" + +INPUT_PROMPT = "Once upon a time, there existed a little girl who liked to have adventures." +tokenizer = AutoTokenizer.from_pretrained(TOKEN_PATH, use_fast=False, padding_side="left", trust_remote_code=True) +streamer = TextStreamer(tokenizer, skip_special_tokens=True, skip_prompt=False) + +input_ids = tokenizer(INPUT_PROMPT, return_tensors="pt", padding=False).input_ids +model = xfastertransformer.AutoModel.from_pretrained(MODEL_PATH, dtype="bf16") +generated_ids = model.generate(input_ids, max_length=200, streamer=streamer) +``` + +### C++ API +[SentencePiece](https://github.com/google/sentencepiece) can be used to tokenizer and detokenizer text. +```C++ +#include +#include +#include "xfastertransformer.h" +// ChatGLM token ids for prompt "Once upon a time, there existed a little girl who liked to have adventures." +std::vector input( + {3393, 955, 104, 163, 6, 173, 9166, 104, 486, 2511, 172, 7599, 103, 127, 17163, 7, 130001, 130004}); + +// Assume converted model dir is `/data/chatglm-6b-cpu`. +xft::AutoModel model("/data/chatglm-6b-cpu", xft::DataType::bf16); + +model.config(/*max length*/ 100, /*num beams*/ 1); +model.input(/*input token ids*/ input, /*batch size*/ 1); + +while (!model.isDone()) { + std::vector nextIds = model.generate(); +} + +std::vector result = model.finalize(); +for (auto id : result) { + std::cout << id << " "; +} +std::cout << std::endl; +``` + +## How to run +Recommend preloading `libiomp5.so` to get a better performance. `libiomp5.so` file will be in `3rdparty/mklml/lib` directory after building xFasterTransformer successfully. +### Single rank +Recommend using `SINGLE_INSTANCE=1` env to avoid MPI initialization. + +### Multi ranks +#### Command line +Use MPI to run in the multi-ranks mode. Here is a example on local. Install oneCCL firstly, please refer to [Prepare Environment](#prepare-environment). +```bash +OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 ${RUN_WORKLOAD} : \ + -n 1 numactl -N 1 -m 1 ${RUN_WORKLOAD} +``` + +#### Code +For more details, please refer to examples. +##### Python +`model.rank` can get the process's rank, `model.rank == 0` is the Master. +For Slaves, after loading the model, the only thing needs to do is `model.generate()`. The input and generation configuration will be auto synced. +```Python +model = xfastertransformer.AutoModel.from_pretrained(MODEL_PATH, dtype="bf16") + +# Slave +while True: + model.generate() +``` +##### C++ +`model.getRank()` can get the process's rank, `model.getRank() == 0` is the Master. +For Slaves, any value can be input to `model.config()` and `model.input` since Master's value will be synced. +```C++ +xft::AutoModel model("/data/chatglm-6b-cpu", xft::DataType::bf16); + +// Slave +while (1) { + model.config(); + std::vector input_ids; + model.input(/*input token ids*/ input_ids, /*batch size*/ 1); + + while (!model.isDone()) { + model.generate(); + } +} +``` + +## [Web Demo](examples/web_demo/README.md) +A web demo based on [Gradio](https://www.gradio.app/) is provided in repo. Now support ChatGLM, ChatGLM2 and Llama2 models. +- [Perpare the model](#prepare-model). +- Install the dependencies + ```bash + pip install -r examples/web_demo/requirements.txt + ``` +- Run the script corresponding to the model. After the web server started, open the output URL in the browser to use the demo. Please specify the paths of model and tokenizer directory, and data type. `transformer`'s tokenizer is used to encode and decode text so `${TOKEN_PATH}` means the huggingface model directory. This demo also support multi-rank. +```bash +# Recommend preloading `libiomp5.so` to get a better performance. +# `libiomp5.so` file will be in `3rdparty/mklml/lib` directory after build xFasterTransformer. +SINGLE_INSTANCE=1 LD_PRELOAD=libiomp5.so python examples/web_demo/ChatGLM.py \ + --dtype=bf16 \ + --token_path=${TOKEN_PATH} \ + --model_path=${MODEL_PATH} +``` + +## [Benchmark](benchmark/README.md) + +Benchmark scripts are provided to get the model inference performance quickly. +- [Prepare the model](#prepare-model). +- Enter the folder corresponding to the model, for example + ```bash + cd benchmark/chatglm6b/ + ``` +- Run scripts `run_${MODEL}.sh`. Please modify the model and tokenizer path in `${MODEL}.sh` before running. + - Shell script will automatically check the number of numa nodes. By default, at least there are 2 nodes and 48 physics cores per node (If the system is in sub-numa status, there are 12 cores for each sub-numa). + - By default, you will get the performance of "input token=32, output token=32, Beam_width=1, FP16". + - If more datatype and scenarios performance needed, please modify the parameters in `${MODEL}.sh` + - If system configuration needs modification, please change run-chatglm-6b.sh. + - If you want the custom input, please modify the `prompt_pool.json` file. + +**Notes!!!**: The system and CPU configuration may be different. For the best performance, please try to modify OMP_NUM_THREADS, datatype and the memory nodes number (check the memory nodes using `numactl -H`) according to your test environment. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..cb59eb8 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy +Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. + +## Reporting a Vulnerability +Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..f18e7f5 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,34 @@ +# Benchmark + +Benchmark scripts is provided to quickly get the model inference performance. + +## Step 1: Prepare xFasterTransformer +Please refer to [Installation](../../README.md#installation). This example supports using source code which means you don't need install xFasterTransformer into pip and just build xFasterTransformer library, and it will search library in src directory. + +## Step 2: Prepare models +Please refer to [Prepare model](../README.md#prepare-model) + +## Step 3: Install the dependencies. +- Please refer to [Prepare Environment](#prepare-environment) to install oneCCL. +- Python dependencies. + ```bash + # requirements.txt in root directory. + pip install -r requirements.txt + ``` + +## Step 4: Run scripts +Enter the folder corresponding to the model and run `run_${MODEL}.sh`. Please modify the model and tokenizer path in `${MODEL}.sh` before running. +```bash +# ChatGLM for example. +cd benchmark/chatglm-6b +bash run_chatglm-6b.sh +``` + +- Shell script will automatically check number of numa nodes, default at least 2 nodes and there is 48 physics cores in each node (12core for subnuma). +- By default, you will get the performance of "input token=32, output token=32, Beam_width=1, FP16". +- If more datatype and scenarios performance needed, please modify the parameters in `${MODEL}.sh` +- If system configuration needs modification, please change run-chatglm-6b.sh. +- If you want the custom input, please modify the `prompt_pool.json` file. + +**Notes!!!**: The system and CPU configuration may be different. For the best performance, please try to modify OMP_NUM_THREADS, datatype and the memory nodes number (check the memory nodes using `numactl -H`) according to your test environment. + diff --git a/benchmark/baichuan2-13b/baichuan2-13b.sh b/benchmark/baichuan2-13b/baichuan2-13b.sh new file mode 100755 index 0000000..258c7be --- /dev/null +++ b/benchmark/baichuan2-13b/baichuan2-13b.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "FP16 Performance " +python "${SCRIPT_DIR}"/../benchmark.py \ + --token_path /data/Baichuan2-13B-Chat \ + --model_path /data/Baichuan2-13B-Chat/cpu \ + --prompt_path "${SCRIPT_DIR}"/prompt_pool.json \ + --model_name "Baichuan2-13B" \ + --dtype fp16 \ + --token_in 32 \ + --token_out 32 --beam_width 1 --iteration 100 + +# In this benchmark case, token_in only can be "demo","32","64","128","256","512","1024","2016" +# "32" means the token length is 32, if needs more test, add it into input_token.py + diff --git a/benchmark/baichuan2-13b/prompt_pool.json b/benchmark/baichuan2-13b/prompt_pool.json new file mode 100644 index 0000000..a796f56 --- /dev/null +++ b/benchmark/baichuan2-13b/prompt_pool.json @@ -0,0 +1,13 @@ +{ + "demo": "从前有个小姑娘，她非常喜欢冒险", + "9": "脚崴了该怎么处理?", + "16": "脚不小心崴了应该怎么处理?", + "32": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满", + "64": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原", + "128": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨", + "256": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，", + "512": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，发现里面的一切都超出了她的想象。墙壁上挂满了古老的画作，地板上铺满了华丽的地毯，而在城堡的深处，她发现了一个闪烁着宝石的宝箱。当她打开宝箱时，她发现里面并没有金银财宝，而是一面镜子。当她凝视着镜中的自己时，她突然明白，整个探险的旅程其实是在寻找自己内心的勇气和智慧。艾莉丝明白了，冒险并不仅仅是外部世界的探索，更是内心的成长和启发。她带着这份领悟，回到了她宁静的小村庄，分享着她的经历和故事，激励着其他小孩子们去追寻自己的冒险和梦想。从此以后，小村庄的孩子们都称艾莉丝为“勇敢的冒险者”，而她的故事也在小村庄中代代相传，成为了鼓舞人心的传奇。在勇敢的冒险者艾莉丝的激励下，小村庄的氛围变得热闹非凡。孩子们开始组成小队，模仿着艾莉丝的足迹，寻找着属于他们自己的冒险。每个人都在自己的心中埋下了一颗探索未知的种子。艾莉丝成为了村子里的一位贵人。她不仅与孩子们分享自己的故事，还在村子中建立了一个冒险学校，教授孩子们如何在不同的情况下保持勇气和坚韧，如何寻找创新的解决方案。孩子们在冒险学校中学到了团队合作、领导能力和适应变化的重要性。而艾莉丝自己也在教导的过程中不断成长。她意识到，冒险", + "1196": "这是一个全民AI的时代。如果你不能张口ChatGPT、闭口大模型，都不好啥意思跟人打招呼。如果你不在AI上搞点东西，都不好意思说自己是科技企业。当然了，AI的历史其实相当悠久，远不只是对个话>、做个图那么简单。无论是云侧还是端侧，无论是生成式还是决策式，无论硬件还是算法，无论是训练推理还是应用场景，都是相当深奥的学问。想真正做好AI，基础硬件、开发软件、生态场景都缺一不可，必须高效、合理地处理各种各样的数据、模型、应用，真正落到使用。能有如此综合实力的企业屈指可数，Intel无疑就是一个典型标杆，从云到端都有丰富的AI解决方案，CPU通用处理器、GPU加速器、AI加速器任君按需>选择。7月11日，Intel在中国举办了Intel AI产品战略暨Gaudi2新品发布会，正式面向中国市场推出第二代深度学习加速器——Habana Gaudi2。Intel Gaudi2加速器不但拥有极高的深度学习性能、效率，最大优势就>是极高的性价比，对于中国用户来说堪称大规模部署AI的上佳之选。Intel执行副总裁兼数据中心与人工智能事业部总经理Sandra Rivera在发布会上表示：“Intel致力于通过为客户提供广泛的硬件选择，并支持开放的软件环境，加速AI技术的发展。凭借包括至强可扩展处理器、Gaudi2深度学习加速器在内的产品组合，Intel正在降低AI的准入门槛，并强化客户在云端通过网络和智能边缘部署这一关键业务技术的能力，从而帮>助构建中国AI的未来。”Habana Labs成立于2016年，致力于研发世界一流的AI加速器，满足人工智能、深度学习计算快速发展的需求，创业初期就得到了Intel的投资，2019年12月被Intel正式收购。Habana的第二代加速器Gaudi2采用台积电7nm工艺制造，集成24个可编程的Tenor张量核心(TPC)、48MB SRAM缓存、21个10万兆内部互连以太网接口(ROCEv2 RDMA)、96GB HBM2E高带宽内存(总带宽2.4TB/s)、多媒体引擎等，支持PCIe 4.0 x16，最高功耗800W。基于Gaudi2加速器芯片，Intel还设计了夹层卡HL-225B，采用标准的OAM封装接口，方便客户部署与使用。凭借高性能和高效扩展性，Gaudi2加速器可以满足大规模语言模型、生成式AI模>型的强算力需求。Gaudi系列加速器优异的深度学习训练吞吐量、推理速度性能，已经得到了业界领先机构、客户的普遍认可。比如，正是在第一代Gaudi加速器的加持下，亚马逊EC2 DL1实例相比于在AWS云上运行NVIDIA GPU的同类实例，性价比高出多达40％。机器学习与人工智能开放产业联盟MLCommons在六月底公布的AI性能基准测试MLPerf Training 3.0的最新结果，更是进一步凸显了Gaudi2加速器的高性能、高性价比，联合Intel第四代至强可扩展处理器，已经成为唯一能够可靠取代NVIDIA GPU的方案。截止2023年6月，Gaudi2是除了NVIDIA H100 GPU以外，向GPT-3大模型训练基准提交性能结果的解决方案。测试结果显示，面对要求极为苛刻的、1750亿参数的GPT-3模型，384个Gaudi2加速器上的训练时间仅为311.9分钟，而且从256个加速器到384个加速器，性能扩展幅度达95％，非常接近理想的线性提升。Stable Diffusion训练上，Gaudi2加>速器从1张卡到64张卡，扩展性更是达到了惊人的99％。此外，在计算机视觉模型ResNet-50（8个加速器）和Unet3D（8个加速器），以及自然语言处理模型BERT（8个和64个加速器）上，Gaudi2都取得了优异的训练>结果。与去年11月提交的数据相比，BERT和ResNet模型的性能分别提高了10％、4％。值得一提的是，本次MLPerf 3.0的Gaudi2结果以BF16数据类型提交，在四种不同模型上的性能均优于NVIDIA A100，价格更便宜。第三季度还会发布对FP8数据类型的软件支持与新功能，预计届时Gaudi2的性能将有明显飞跃，预计性价比将超越NVIDIA H100。Gaudi2加速器还得到了AI与机器学习开源软件供应商Hugging Face的采纳。其测试结果显示，从预训练BERT （NLP模型）到稳定扩散（流行的多模态模型）推理，再到1760亿参数的大型开源聊天模 BLOOMZ的推理，Gaudi2的表现都领先于NVIDIA A100 GPU。工欲善其事，必先利其器。为了充分发挥Gaudi2加速器的性能潜力，满足日益增长的生成式AI、大语言模型需求，Intel一直在同步打造高效、成熟的软件支持。比如说SynapseAI软件套件，针对Gaudi平台深度学习业务进行了优化，可以大大简化模型的开发与>迁移，能够将当前基于GPU的模型业务和系统，快速迁移到基于全新Gaudi2的服务器。请总结以上内容主题", + "2050": "人类文明的发展，从农业社会、工业社会、信息社会，再到今天的智能社会，技术的每一次进步都带来生产力的极大跃升。当下，数字技术驱动生产力从“量变到质变”，人类正在加速迈向智能世界。智能世界的核心是感知、连接和计算。人类对于未来的追求永无止境，这需要数字技术提升成百上千倍能力。面向未来，只有大胆提出假设和愿景，敢于打破既有理论与技术瓶颈的条条框框，才能大踏步前行。今天，我们对智能世界的所有想象都是保守的。数字化是全世界、全行业的共同机遇。在数字经济新时代，联接从人到物和机器，从家庭走向工厂，从地面走进地下矿道，未来更丰富的行业场景要求联接无处不在、无时不快。算力将成为新生产力，数字化浪潮中，个人、家庭、企业需要算力像水和电一样，触手可及、随取随用。数字化、智能化使能绿色低碳， 创造新的社会价值 使能行业数字化转型，共创商业价值与社会价值。当 前，数字化、智能化成为经济社会发展的关键驱动力， 引领新一轮产业变革。对于华为而言，我们致力于用技 术创新服务于全社会、全人类，在为客户创造商业价值 的同时，努力创造新的社会价值。 过去一年，华为和运营商、合作伙伴一起，在港口、制 造、煤矿、钢铁、化工等 20 多个行业实践数字化转型 升级，累计签署了超过 3,000 个 5G 行业应用商用合同。 在港口，通过 5G+ 高清视频，实现龙门吊的远程控制， 操作人员不再需要爬上爬下，风吹日晒，以及每天 10 到 12 个小时的长时间俯视作业，大幅改善了工作环境 与体验，吸引更多年轻人加入龙门吊司机的行列；在 炼钢车间，5G 远程操控让工人脱离嘈杂、高温的工作 环境，在提升生产效率的同时，让工人享有安全、舒适 的工作环境；在面临易燃、易爆、高温等挑战的化工行 业，通过 5G 机器人巡检、5G 移动作业监控等手段对 厂区的全要素进行视频采集和全天候分析，使得安全预 判效率得到巨大提升。 数字化、智能化使能行业转型升级与经济发展，不仅体 现在生产效率提升与商业价值创造，也凸显出较大社会 价值，让生产更安全，让劳动者更舒适。 绿色 ICT 助力环境保护，促进绿色低碳发展。绿色发 展已成为全球共识。数字化、智能化正前所未有地渗透 到人们生产生活的方方面面，促进更加低碳的生活、工 作与生产方式。未来，数字化和绿色低碳将进一步融 合，带来巨大发展机遇的同时，也将推动社会进步。 华为相信，数字技术是使能绿色发展的基本要素，我 们致力于通过数字技术创新，帮助客户和全社会实现 低碳发展。首先是围绕 ICT 产业自身的节能减排和绿色 发展。一直以来，华为通过节能技术创新，不断提升 ICT 产品的能效。我们与全球运营商一起，打造极简、 绿色、智能的 ICT 基础设施。截至 2021 年底，华为已 支持 100 多个国家的运营商，部署了绿色站点解决方 案，助力全球运营商节省约 842 亿度电，减少约 4,000 万吨二氧化碳排放。同时，华为致力于把 ICT 作为一种 使能技术助力其他行业转型升级，实现绿色发展。在瑞 士，数字技术与传统农业相结合，实施杂草精准清除， 节省约 90% 的除草剂使用，减少了农药对于环境的影 响。在中国，通过智慧供暖实现按需供热，哈尔滨道外 区把平均能耗降低了 10%。 此外，华为数字能源创新融合了数字技术、电力电子技 术、储能技术与热技术，实现比特管理瓦特，助力客户 节能减排。截至 2021 年底，已助力客户累计实现绿色 发电 4,829 亿度，节约用电约 142 亿度，减少二氧化 碳排放近 2.3 亿吨，相当于种植 3.2 亿棵树。 跨越数字鸿沟，推动数字人才培养。数字经济时代，数 字人才是助力数字化转型、推动经济增长的关键。华为 助力各国构建数字人才生态，持续推进 TECH4ALL 数 字包容倡议，实现技术普惠，让人人享受数字生活的便 利，促进数字经济的发展。 从 2008 年开始，华为发起“未来种子”项目，陆续推 出和赞助了各类人才培养、竞赛等项目，包括未来种子 奖学金、ICT 学院、开发者学院、云开发者学堂等，致 力于支持业务所在国培养 ICT 人才，累计投入超过 1.5 亿美元，覆盖 150 多个国家，受益人数超过 154 万人。 我们将持续为数字人才培养做贡献。2021 年，华为发 布了“未来种子 2.0”计划，承诺在未来五年投入 1.5 亿美元用于数字化人才培养，新增受益人数预计将超过 300 万人。我们希望，更多的个人、组织加入华为的数 字包容倡议，共同推进数字人才的发展计划。 2021 年年度报告 09 开放合作，持续创新，坚持全球运 营，与合作伙伴共同成长 科技创新是科技公司对人类社会的重要贡献之一。华为 公司努力探索科学技术的前沿，与世界开放合作，坚持 基础研究、坚持开放创新。当前是华为发展的关键时 期，我们仍将进一步加大在研究与创新领域投资，为各 行各业创造增量价值，让每个人、家庭与组织都从技 术进步中受益。2021 年，华为研发费用支出为人民币 1,427 亿元，约占全年收入的 22.4%。过去十年，我们 在研发领域累计投资超过人民币 8,450 亿元，近几年， 每年在基础研究上的投资超过人民币 200 亿元。 我们建立了 86 个基础技术实验室，扎根基础研究，建 立相关的核心技术体系；我们提出了 AI 时代可信计算 的新架构模型，实现多语言、跨平台运行，实现数据处 理量提升 3 个数量级；我们与合作伙伴一起共同促进欧 拉和鸿蒙开源生态的繁荣，共同打造覆盖未来计算场景 的开源操作系统，促进算力和数据流通，早日实现算力 普惠，让算力像电力一样非常方便地按需而用。 同时，我们持续开放合作，积极融入全球学术组织，加 强与全球高校、科研机构开展合作。通过与科学家碰 撞、分享难题与知识，共同应对世界级的挑战和难题， 促进科技进步。华为将用最大的诚意，最热烈的呼唤拥 抱和欢迎全世界科学家和优秀人才。过去几年，我们在 北京、上海、深圳等地建立了“黄大年茶思屋”和“九 章院”，希望通过提供宽松、宁静的创新环境，让科学 家们心无旁骛、潜心研究，真正面向未来、开创未来， 为人类社会发展做贡献。 华为坚持全球运营，并实施多元化的供应策略，构建长 期、持续、稳定的供应能力，保障供应连续性和面向未 来的可持续发展。我们与全球上万家供应商和合作伙伴 广泛合作，建立了长期合作关系，加强开放合作，在开 放合作中解决发展中的问题和挑战。华为有信心、有能 力继续与全球合作伙伴共同奋斗，秉持“合作共赢、共 同发展”理念，打造安全、可靠、有竞争力的健康产业 链，持续为全球客户提供优质的产品与服务。 完善公司治理，坚持合规运营， 更好地为全球客户服务 良好的公司治理是公司稳健发展的重要基石。华为坚持 以客户为中心、以奋斗者为本，持续优化公司治理架 构、组织、流程和考核机制，使公司长期保持有效增 长。2021 年，持股员工代表会举行了两次会议，审议 了一系列治理文件：通过监事会制度，进一步完善公司 监事会的定位和职权，规范监事会运作；逐步明确集团 董事会与子公司董事会的相互关系。同时，面向一线组 织，采用数字化的方式，构建疫情下的新型业务管理体 系和平台，持续加强数字化建设与运营，快速响应，实 现“多打粮食，增加土地肥力”。 华为坚持以法律遵从的确定性，应对国际政治的不确定 性。坚持诚信经营、恪守商业道德、致力于遵从业务所 在国适用的法律法规是华为管理层一直秉持的核心理 念。公司的基本政策要求全球各子公司、各部门遵从所 在国家和地区适用的法律法规。我们重视并持续营造诚 信文化，要求每一位员工遵守商业行为准则。 同时，作为全球化公司，华为持续积极与外界开放沟 通，进一步开放透明，让所有客户、合作伙伴和利益 相关人更加深入了解华为。我们也欢迎各国政府、媒 体、专家学者等各界人士来访公司，了解和认识真实的 华为。 没有退路就是胜利之路。华为的选择注定是一条布满荆 棘，但同时也是值得为之奋斗的漫漫长路。未来，尽管 面临更多不确定性，华为将坚持开放合作，保持战略定 力，保证公司的生存与发展。无论前路多么坎坷，我们 将始终坚守愿景与使命：把数字世界带入每个人、每个 家庭、每个组织，构建万物互联的智能世界。请总结以上内容主题", + "3294": "智能手表（Smart Watch）是安装有嵌入式系统、用于增强基于报时等功能的腕部手表，其功能相似于一台个人数码助理。智能手表除指示时间之外，还应具有提醒、导航、校准、监测、交互等其中一种或者多种功能；显示方式包括指针、数字、图像等。《2023-2027年智能手表市场调查研究报告》研究显示：随着科技的进步和发展，智能穿戴设备市场从2012年至今得到了快速增长，在这10年间，有越来越多的玩家加入，智能穿戴设备产品不管是在技术上、还是使用体验上，都有了很大提升和改善。智能手表是智能穿戴的主要代表之一，在健康监测、记步、拨打电话、定位、与智能家居联动等功能的加持下，广受欢迎。智能手表“智能”的地方体现在，它拥有一套独立的嵌入式操作系统，有一个数据处理中心，需要调用各类传感器收集到的信息，还要有屏幕、存储器、电池、电源管理系统、无线射频系统等，在内部芯片用料和结构设计上与智能手机较为相似。2017年我国智能手表市场规模为131.23亿元，2021年为317.58亿元，同比2020年增长16.60%。脱硝催化剂泛指选择性催化还原（SCR）脱硝催化剂，也叫SCR脱硝催化剂。SCR脱硝催化剂是指应用于选择性催化还原（SCR）脱硝系统上的催化剂，是在选择性催化还原（SCR）脱硝技术基础上发展起来的。《脱硝催化剂行业数据深度调研分析与发展战略规划报告》研究显示:早在2011年，环保部颁布了《火电厂大气污染物排放标准》，开启中国火电环保风暴，各火电厂纷纷加快升级改造步伐。将火电厂脱硝催化剂市场拆分为二部分：新建机组的初装需求以及已投运机组的补装（更换）需求。2021年新建机组配套催化剂需求为3.89万m3，已投运的脱硝机组催化剂更换需求为24.05万m3，合计27.94万m3。酸枣（学名：Ziziphus jujuba Mill. var. spinosa（Bunge）Hu ex H. F. Chow）鼠李科枣属植物，是枣的变种。又名棘、棘子、野枣、山枣、葛针等，原产中国华北，中南各省亦有分布。多野生，常为灌木，也有的为小乔木。树势较强。枝、叶、花的形态与普通枣相似，但枝条节间较短，托刺发达，除生长枝各节均具托刺外，结果枝托叶也成尖细的托刺。叶小而密生，果小、多圆或椭圆形、果皮厚、光滑、紫红或紫褐色，肉薄，味大多很酸，核圆或椭圆形，核面较光滑，内含种子1至2枚，种仁饱满可作中药。其适应性较普通枣强，花期很长，可为蜜源植物。果皮红色或紫红色，果肉较薄、疏松，味酸甜。《酸枣行业现状分析及发展战略研究报告》研究显示：随着生活节奏的不断加快，出现睡眠障碍的人群不断增多，身心健康状况也逐渐下降。酸枣常被用来治疗心烦失眠、多梦、盗汗、易惊等病。通过深入挖掘酸枣助眠机理，开发相应产品，可改善人们的睡眠质量及健康状况，为健康中国助力。2017年，中国酸枣行业市场规模为17.67亿元，2021年为33.24亿元，同比增长4.25%。丙烯酸树脂（acrylic resin）是丙烯酸、甲基丙烯酸及其衍生物聚合物的总称。丙烯酸树脂涂料就是以（甲基）丙烯酸酯、苯乙烯为主体，同其他丙烯酸酯共聚所得丙烯酸树脂制得的热塑性或热固性树脂涂料或丙烯酸辐射涂料。含羟基丙烯酸树脂是指加入含羟基的丙烯酸酯和甲基丙烯酸酯等单体，使之与其他（甲基）丙烯酸类单体一起共聚而制成的带羟基官能团的丙烯酸树脂。一般情况下，羟基丙烯酸树脂和热固性树脂含有-OH基团，部分热塑性树脂也含有少量-OH。羟基是丙烯酸树脂侧链上作为后期交联的活性官能团，作用是增大丙烯酸树脂的极性，改良漆膜的附着力和润湿分散性。因此，丙烯酸树脂中羟基含量越大，使得漆膜交联密度变大，制备出的涂料漆膜物理性、化学性、机械性能和漆膜外观越好。同时，羟基含量越大时，可加快-OH基团与-NCO基团的活性反应，提高涂料的固化交联速度和干燥时间。《含羟基丙烯酸树脂行业数据深度调研分析与发展战略规划报告》研究显示：随着世界各国对环保问题的逐渐重视，涂料行业正在逐渐由传统的有机溶剂型涂料向环保型涂料转型，为响应新型涂料发展的需要，制备“高性能、低污染、长寿命的环保型涂料产品具有极高的社会意义。而限制环保型涂料发展的最重要原因是环保型树脂的升级，水性树脂作为环保型树脂中最重要的品类之一，是“油转水”转型中最重要的一环；而含羟基丙烯酸树脂具有“高硬度、高光泽、高保护”的使用性能，在工业领域中也得到广泛应用。2017年我国含羟基丙烯酸树脂行业产能为62.45万吨，2021年产能为85.82万吨，同比2020年增长7.25%。城市轨道交通是城市公共交通的骨干，具有节能、省地、运量大、全天候、无污染（或少污染）又安全等特点，属绿色环保交通体系，特别适应于大中城市。进入新世纪，尤其是2008年以后，我国通过扩大内需，促进经济平稳增长的一揽子计划，带动了国内基础建设的发展，同时我国大型城市逐渐面临交通拥堵的问题，加快了城市轨道交通建设。《城市轨道交通行业数据深度调研分析与发展战略规划报告》研究显示：根据城市轨道交通协会发布的数据显示，我国城市轨道交通运营里程稳步攀升，并且我国已经成为了全球城市轨道运营里程数第一，远超德国、俄罗斯、美国等发达国家。2021年中国开通轨道交通的城市数量从2015年的26个增长到51个，在6年期间开通轨道交通的城市数量几乎翻备。2021年，中国城市轨道交通运营里程8708公里，较2015年的3618公里，是2015年3618公里的2.4倍。别墅电梯即安装在私人住宅中，仅供单一家庭成员使用的电梯。它也可以安装在非单一家庭使用的建筑物内，作为单一家庭进入其住所的工具，但是建筑物内的公众或其他居住者无法进入和使用。《别墅电梯行业现状分析及发展战略研究报告》研究显示：作为基础设施配套工程的重要组成部分，电梯与国家经济建设尤其是房地产的发展以及人民生活质量的提高密切相关。近些年，随着全球人口增长、城市化进程加快以及人们对便捷生活要求的提高，别墅电梯得到越来越广泛的使用。目前，国际别墅电梯市场呈现发达国家和地区需求稳步增长、新兴市场需求快速增长的特征。2017年全球别墅电梯行业产能规模为4.79万台，2021年为7.89万台，同比2020年增长12.55%。物质、能量和信息是构成自然界的基本要素。“能源”这一术语，过去人们谈论得很少，正是两次石油危机使它成了人们议论的热点。关于能源的定义，约有20种。例如：说：“能源是可从其获得热、光和动力之类能量的资源”；《大英百科全书》说：“能源是一个包括着所有燃料、流水、阳光和风的术语，人类用适当的转换手段便可让它为自己提供所需的能量”；《日本大百科全书》说：“在各种生产活动中，我们利用热能、机械能、光能、电能等来作功，可利用来作为这些能量源泉的自然界中的各种载体，称为能源”；我国的《能源百科全书》说：“能源是可以直接或经转换提供人类所需的光、热、动力等任一形式能量的载能体资源。”可见，能源是一种呈多种形式的，且可以相互转换的能量的源泉。确切而简单地说，能源是自然界中能为人类提供某种形式能量的物质资源。能源亦称能量资源或能源资源。是指可产生各种能量（如热量、电能、光能和机械能等）或可作功的物质的统称。是指能够直接取得或者通过加工、转换而取得有用能的各种资源，包括煤炭、原油、天然气、煤层气、水能、核能、风能、太阳能、地热能、生物质能等一次能源和电力、热力、成品油等二次能源，以及其他新能源和可再生能源。能源（Energy Source）亦称能量资源或能源资源，是国民经济的重要物质基础，未来国家命运取决于能源的掌控。能源的开发和有效利用程度以及人均消费量是生产技术和生活水平的重要标志。（中国大百科全书·机械工程卷）在《中华人民共和国节约能源法》中所称能源，是指煤炭、石油、天然气、生物质能和电力、热力以及其他直接或者通过加工、转换而取得有用能的各种资源。《能源领域信息化与IT应用行业数据深度调研分析与发展战略规划报告》研究显示：“互联网+智慧能源”的理念推动着能源行业的改革，传统企业需要思考如何将重资产的使用效率提升，进一步拥抱数字化，而运维服务也是企业对外输出的核心竞争力之一。在数字化时代，企业数据上云成为趋势，云计算和云存储成为重要的数字化能力。然而对于关系国计民生的能源行业来说，大力推进绿色低碳科技创新将成为碳中和发展的主要路径，推动绿色低碳技术重大突破，加快能源行业的数字化转型，变得尤为重要。能源云是紧密结合能源行业业务，用云计算、大数据、移动、物联网、人工智能等新一代的技术架构和体系为能源企业提供的云服务平台，它涵盖行业云服务、领域云服务、云工具和云平台，覆盖能源企业运营中的采购、生产、营销、数据、财务、人力、社交与协同办公等多个业务场景和领域，它可以与传统软件融合，让企业的业务自然延伸到云端应用。采用以公有云、专属云、混合云等云服务模式为主，行业解决方案为辅的多种服务形式，为能源企业互联网化提供坚实的支撑和保障。公有云通常指第三方提供商为用户提供的能够使用的云，公有云一般可通过Internet使用，可能是免费或成本低廉的，公有云的核心属性是共享资源服务。花青素（anthocyan），又称花色素，是自然界一类广泛存在于植物中的水溶性天然色素，是花色苷（anthocyains）水解而得的有颜色的苷元。水果、蔬菜、花卉中的主要呈色物质大部分与之有关。在植物细胞液泡不同的PH值条件下，花青素使花瓣呈现五彩缤纷的颜色。已知花青素有20多种，食物中重要的有6种，即天竺葵色素、矢车菊色素、飞燕草色素、芍药色素、牵牛花色素和锦葵色素。自然状态的花青素都以糖苷形式存在，称为花色苷，很少有游离的花青素存在。《花青素市场现状分析及行业前景预测报告》研究显示：我国花青素开发较晚，且主要以粗制品为主，2011年之前，我国对花青素的提取基本上是空白。2011年臻多美研发成功的花青素提取和保存两项专利技术，填补了国内花青素技术的空白，打破了之前国内花青素精制品90%靠进口的局面，并大量销往国外。随着人们健康意识的加强，以及我国与国际规范的接轨，对花青素的需求逐渐凸显。2017年我国花青素市场规模为3.44亿元，2021年为6.93亿元，同比2020年增长17.20%。发动机（Engine）是一种能够把其它形式的能转化为机械能的机器，包括如内燃机（往复活塞式发动机）、外燃机（斯特林发动机、蒸汽机等）、喷气发动机、电动机等。如内燃机通常是把化学能转化为机械能。发动机既适用于动力发生装置，也可指包括动力装置的整个机器（如：汽油发动机、航空发动机）。从字面上理解，道路发动机就是用于驱动在公共道路上行驶的车辆的发动机，而非道路发动机就是所有这个用途以外的发动机，比如船用发动机、工程机械发动机、发电用发动机、航空发动机等等。《非道路发动机行业数据深度调研分析与发展战略规划报告》研究显示：近年来，全球非道路发动机市场呈现稳定态势，到2019年市场产能增长至4726.31万台；2020年，新冠疫情爆发，非道路发动机产能下降至4526.39万台；2021年产能增速恢复至10.78%，总体产能达5014.27万台。发动机是一种能够把其它形式的能转化为机械能的机器，发动机既适用于动力发生装置，也可指包括动力装置的整个机器，农用发动机就是农业机械用发动机。农业机械是指在作物种植业和畜牧业生产过程中，以及农、畜产品初加工和处理过程中所使用的各种机械。《农用发动机行业现状分析及发展战略研究报告》研究显示：随着农业供给侧结构性改革的推进，农用发动机行业进入了深度调整期，技术升级，结构优化也在同步进行中。农业是支撑国民经济建设与发展的基础产业，而农业机械化是建设现代农业的重要物质基础，也是实现农业现代化的重要标志与内容。近年来，在国家政策补贴支持及机械化率稳步增长等因素的推动下，农用发动机行业规模不断增长。市场规模方面，据统计，近年来我国农用发动机市场规模不断增长，截至2021年我国农用发动机市场规模为429.61亿元，同比增长27.60%。为适应文明演进的新趋势和新要求，人类必须从根本上解决文明前行的动力困扰，实现能源的安全、稳定、清洁和永续利用。智慧能源就是充分开发人类的智力和能力，通过不断技术创新和制度变革，在能源开发利用、生产消费的全过程和各环节融汇人类独有的智慧，建立和完善符合生态文明和可持续发展要求的能源技术和能源制度体系，从而呈现出的一种全新能源形式。简而言之，智慧能源就是指拥有自组织、自检查、自平衡、自优化等人类大脑功能，满足系统、安全、清洁和经济要求的能源形式。《智慧能源行业现状分析及发展战略研究报告》研究显示：电力行业是智慧能源行业最大的市场，随着我国经济的快速发展，智慧能源行业作为电力行业产业链中的重要一环也随之蓬勃发展，尤其近几年我国经济发展中面临能源、电力紧张的瓶颈性问题，国家不断加大对智慧能源行业的投资，使得该行业步入了飞跃发展期，2021年中国智能电网市场规模已经达到854.6亿元。请从上述信息中总结出最佳的投资方向并给出理由：" +} diff --git a/benchmark/baichuan2-13b/run-baichuan2-13b.sh b/benchmark/baichuan2-13b/run-baichuan2-13b.sh new file mode 100755 index 0000000..744e2f9 --- /dev/null +++ b/benchmark/baichuan2-13b/run-baichuan2-13b.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# On different systems, different models and differnt modes, OMP_NUM_THREADS need to be ajusted. +output=$(numactl -H) +first_line=$(echo "$output" | head -n 1) +nodes=$(echo "$first_line" | awk '{print $2}') + +echo "memory node number: $nodes" +if [ "$nodes" -eq 16 ]; then +#HBM SNC-4 mode, Confirm that there are 8 HBM memory nodes and 8 DRAM memory nodes through "numactl -H" +#0-7 is DRAM memory node, 8-15 is HBM node + echo "HBM SNC4 mode" +# Run Baichuan2-13B on 1 socket HBM SNC4 mode + OMP_NUM_THREADS=12 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -m 8 -N 0 sh baichuan2-13b.sh : \ + -n 1 numactl -m 9 -N 1 sh baichuan2-13b.sh : \ + -n 1 numactl -m 10 -N 2 sh baichuan2-13b.sh : \ + -n 1 numactl -m 11 -N 3 sh baichuan2-13b.sh : \ + -n 1 numactl -m 12 -N 4 sh baichuan2-13b.sh : \ + -n 1 numactl -m 13 -N 5 sh baichuan2-13b.sh : \ + -n 1 numactl -m 14 -N 6 sh baichuan2-13b.sh : \ + -n 1 numactl -m 15 -N 7 sh baichuan2-13b.sh + +elif [ "$nodes" -eq 4 ]; then +#HBM Quad-mode, Confirm that there are 2 HBM memory nodes and 2 DRAM memory nodes through "nuamctl -H" + echo "HBM Quad mode" +# Run Baichuan2-13B on 1 socket HBM Quad mode,Flat mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 2 sh baichuan2-13b.sh : \ + -n 1 numactl -N 1 -m 3 sh baichuan2-13b.sh + +elif [ "$nodes" -eq 2 ]; then +#SPR Quad-mode, Confirm that there are 2 DRAM memory nodes through "nuamctl -H" + echo "SPR Quad mode" +# Run Baichuan2-13B on 1 socket SPR Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 sh baichuan2-13b.sh : \ + -n 1 numactl -N 1 -m 1 sh baichuan2-13b.sh + +else + echo "Please double check the memory nodes" +fi + + diff --git a/benchmark/baichuan2-7b/baichuan2-7b.sh b/benchmark/baichuan2-7b/baichuan2-7b.sh new file mode 100755 index 0000000..ff65d3a --- /dev/null +++ b/benchmark/baichuan2-7b/baichuan2-7b.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "FP16 Performance " +python "${SCRIPT_DIR}"/../benchmark.py \ + --token_path /data/Baichuan2-7B-Chat \ + --model_path /data/Baichuan2-7B-Chat/cpu \ + --prompt_path "${SCRIPT_DIR}"/prompt_pool.json \ + --model_name "Baichuan2-7B" \ + --dtype fp16 \ + --token_in 32 \ + --token_out 32 --beam_width 1 --batch_size 2 --iteration 100 --padding=False + +# In this benchmark case, token_in only can be "demo","32","64","128","256","512","1024","2016" +# "32" means the token length is 32, if needs more test, add it into input_token.py + diff --git a/benchmark/baichuan2-7b/prompt_pool.json b/benchmark/baichuan2-7b/prompt_pool.json new file mode 100644 index 0000000..a796f56 --- /dev/null +++ b/benchmark/baichuan2-7b/prompt_pool.json @@ -0,0 +1,13 @@ +{ + "demo": "从前有个小姑娘，她非常喜欢冒险", + "9": "脚崴了该怎么处理?", + "16": "脚不小心崴了应该怎么处理?", + "32": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满", + "64": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原", + "128": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨", + "256": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，", + "512": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，发现里面的一切都超出了她的想象。墙壁上挂满了古老的画作，地板上铺满了华丽的地毯，而在城堡的深处，她发现了一个闪烁着宝石的宝箱。当她打开宝箱时，她发现里面并没有金银财宝，而是一面镜子。当她凝视着镜中的自己时，她突然明白，整个探险的旅程其实是在寻找自己内心的勇气和智慧。艾莉丝明白了，冒险并不仅仅是外部世界的探索，更是内心的成长和启发。她带着这份领悟，回到了她宁静的小村庄，分享着她的经历和故事，激励着其他小孩子们去追寻自己的冒险和梦想。从此以后，小村庄的孩子们都称艾莉丝为“勇敢的冒险者”，而她的故事也在小村庄中代代相传，成为了鼓舞人心的传奇。在勇敢的冒险者艾莉丝的激励下，小村庄的氛围变得热闹非凡。孩子们开始组成小队，模仿着艾莉丝的足迹，寻找着属于他们自己的冒险。每个人都在自己的心中埋下了一颗探索未知的种子。艾莉丝成为了村子里的一位贵人。她不仅与孩子们分享自己的故事，还在村子中建立了一个冒险学校，教授孩子们如何在不同的情况下保持勇气和坚韧，如何寻找创新的解决方案。孩子们在冒险学校中学到了团队合作、领导能力和适应变化的重要性。而艾莉丝自己也在教导的过程中不断成长。她意识到，冒险", + "1196": "这是一个全民AI的时代。如果你不能张口ChatGPT、闭口大模型，都不好啥意思跟人打招呼。如果你不在AI上搞点东西，都不好意思说自己是科技企业。当然了，AI的历史其实相当悠久，远不只是对个话>、做个图那么简单。无论是云侧还是端侧，无论是生成式还是决策式，无论硬件还是算法，无论是训练推理还是应用场景，都是相当深奥的学问。想真正做好AI，基础硬件、开发软件、生态场景都缺一不可，必须高效、合理地处理各种各样的数据、模型、应用，真正落到使用。能有如此综合实力的企业屈指可数，Intel无疑就是一个典型标杆，从云到端都有丰富的AI解决方案，CPU通用处理器、GPU加速器、AI加速器任君按需>选择。7月11日，Intel在中国举办了Intel AI产品战略暨Gaudi2新品发布会，正式面向中国市场推出第二代深度学习加速器——Habana Gaudi2。Intel Gaudi2加速器不但拥有极高的深度学习性能、效率，最大优势就>是极高的性价比，对于中国用户来说堪称大规模部署AI的上佳之选。Intel执行副总裁兼数据中心与人工智能事业部总经理Sandra Rivera在发布会上表示：“Intel致力于通过为客户提供广泛的硬件选择，并支持开放的软件环境，加速AI技术的发展。凭借包括至强可扩展处理器、Gaudi2深度学习加速器在内的产品组合，Intel正在降低AI的准入门槛，并强化客户在云端通过网络和智能边缘部署这一关键业务技术的能力，从而帮>助构建中国AI的未来。”Habana Labs成立于2016年，致力于研发世界一流的AI加速器，满足人工智能、深度学习计算快速发展的需求，创业初期就得到了Intel的投资，2019年12月被Intel正式收购。Habana的第二代加速器Gaudi2采用台积电7nm工艺制造，集成24个可编程的Tenor张量核心(TPC)、48MB SRAM缓存、21个10万兆内部互连以太网接口(ROCEv2 RDMA)、96GB HBM2E高带宽内存(总带宽2.4TB/s)、多媒体引擎等，支持PCIe 4.0 x16，最高功耗800W。基于Gaudi2加速器芯片，Intel还设计了夹层卡HL-225B，采用标准的OAM封装接口，方便客户部署与使用。凭借高性能和高效扩展性，Gaudi2加速器可以满足大规模语言模型、生成式AI模>型的强算力需求。Gaudi系列加速器优异的深度学习训练吞吐量、推理速度性能，已经得到了业界领先机构、客户的普遍认可。比如，正是在第一代Gaudi加速器的加持下，亚马逊EC2 DL1实例相比于在AWS云上运行NVIDIA GPU的同类实例，性价比高出多达40％。机器学习与人工智能开放产业联盟MLCommons在六月底公布的AI性能基准测试MLPerf Training 3.0的最新结果，更是进一步凸显了Gaudi2加速器的高性能、高性价比，联合Intel第四代至强可扩展处理器，已经成为唯一能够可靠取代NVIDIA GPU的方案。截止2023年6月，Gaudi2是除了NVIDIA H100 GPU以外，向GPT-3大模型训练基准提交性能结果的解决方案。测试结果显示，面对要求极为苛刻的、1750亿参数的GPT-3模型，384个Gaudi2加速器上的训练时间仅为311.9分钟，而且从256个加速器到384个加速器，性能扩展幅度达95％，非常接近理想的线性提升。Stable Diffusion训练上，Gaudi2加>速器从1张卡到64张卡，扩展性更是达到了惊人的99％。此外，在计算机视觉模型ResNet-50（8个加速器）和Unet3D（8个加速器），以及自然语言处理模型BERT（8个和64个加速器）上，Gaudi2都取得了优异的训练>结果。与去年11月提交的数据相比，BERT和ResNet模型的性能分别提高了10％、4％。值得一提的是，本次MLPerf 3.0的Gaudi2结果以BF16数据类型提交，在四种不同模型上的性能均优于NVIDIA A100，价格更便宜。第三季度还会发布对FP8数据类型的软件支持与新功能，预计届时Gaudi2的性能将有明显飞跃，预计性价比将超越NVIDIA H100。Gaudi2加速器还得到了AI与机器学习开源软件供应商Hugging Face的采纳。其测试结果显示，从预训练BERT （NLP模型）到稳定扩散（流行的多模态模型）推理，再到1760亿参数的大型开源聊天模 BLOOMZ的推理，Gaudi2的表现都领先于NVIDIA A100 GPU。工欲善其事，必先利其器。为了充分发挥Gaudi2加速器的性能潜力，满足日益增长的生成式AI、大语言模型需求，Intel一直在同步打造高效、成熟的软件支持。比如说SynapseAI软件套件，针对Gaudi平台深度学习业务进行了优化，可以大大简化模型的开发与>迁移，能够将当前基于GPU的模型业务和系统，快速迁移到基于全新Gaudi2的服务器。请总结以上内容主题", + "2050": "人类文明的发展，从农业社会、工业社会、信息社会，再到今天的智能社会，技术的每一次进步都带来生产力的极大跃升。当下，数字技术驱动生产力从“量变到质变”，人类正在加速迈向智能世界。智能世界的核心是感知、连接和计算。人类对于未来的追求永无止境，这需要数字技术提升成百上千倍能力。面向未来，只有大胆提出假设和愿景，敢于打破既有理论与技术瓶颈的条条框框，才能大踏步前行。今天，我们对智能世界的所有想象都是保守的。数字化是全世界、全行业的共同机遇。在数字经济新时代，联接从人到物和机器，从家庭走向工厂，从地面走进地下矿道，未来更丰富的行业场景要求联接无处不在、无时不快。算力将成为新生产力，数字化浪潮中，个人、家庭、企业需要算力像水和电一样，触手可及、随取随用。数字化、智能化使能绿色低碳， 创造新的社会价值 使能行业数字化转型，共创商业价值与社会价值。当 前，数字化、智能化成为经济社会发展的关键驱动力， 引领新一轮产业变革。对于华为而言，我们致力于用技 术创新服务于全社会、全人类，在为客户创造商业价值 的同时，努力创造新的社会价值。 过去一年，华为和运营商、合作伙伴一起，在港口、制 造、煤矿、钢铁、化工等 20 多个行业实践数字化转型 升级，累计签署了超过 3,000 个 5G 行业应用商用合同。 在港口，通过 5G+ 高清视频，实现龙门吊的远程控制， 操作人员不再需要爬上爬下，风吹日晒，以及每天 10 到 12 个小时的长时间俯视作业，大幅改善了工作环境 与体验，吸引更多年轻人加入龙门吊司机的行列；在 炼钢车间，5G 远程操控让工人脱离嘈杂、高温的工作 环境，在提升生产效率的同时，让工人享有安全、舒适 的工作环境；在面临易燃、易爆、高温等挑战的化工行 业，通过 5G 机器人巡检、5G 移动作业监控等手段对 厂区的全要素进行视频采集和全天候分析，使得安全预 判效率得到巨大提升。 数字化、智能化使能行业转型升级与经济发展，不仅体 现在生产效率提升与商业价值创造，也凸显出较大社会 价值，让生产更安全，让劳动者更舒适。 绿色 ICT 助力环境保护，促进绿色低碳发展。绿色发 展已成为全球共识。数字化、智能化正前所未有地渗透 到人们生产生活的方方面面，促进更加低碳的生活、工 作与生产方式。未来，数字化和绿色低碳将进一步融 合，带来巨大发展机遇的同时，也将推动社会进步。 华为相信，数字技术是使能绿色发展的基本要素，我 们致力于通过数字技术创新，帮助客户和全社会实现 低碳发展。首先是围绕 ICT 产业自身的节能减排和绿色 发展。一直以来，华为通过节能技术创新，不断提升 ICT 产品的能效。我们与全球运营商一起，打造极简、 绿色、智能的 ICT 基础设施。截至 2021 年底，华为已 支持 100 多个国家的运营商，部署了绿色站点解决方 案，助力全球运营商节省约 842 亿度电，减少约 4,000 万吨二氧化碳排放。同时，华为致力于把 ICT 作为一种 使能技术助力其他行业转型升级，实现绿色发展。在瑞 士，数字技术与传统农业相结合，实施杂草精准清除， 节省约 90% 的除草剂使用，减少了农药对于环境的影 响。在中国，通过智慧供暖实现按需供热，哈尔滨道外 区把平均能耗降低了 10%。 此外，华为数字能源创新融合了数字技术、电力电子技 术、储能技术与热技术，实现比特管理瓦特，助力客户 节能减排。截至 2021 年底，已助力客户累计实现绿色 发电 4,829 亿度，节约用电约 142 亿度，减少二氧化 碳排放近 2.3 亿吨，相当于种植 3.2 亿棵树。 跨越数字鸿沟，推动数字人才培养。数字经济时代，数 字人才是助力数字化转型、推动经济增长的关键。华为 助力各国构建数字人才生态，持续推进 TECH4ALL 数 字包容倡议，实现技术普惠，让人人享受数字生活的便 利，促进数字经济的发展。 从 2008 年开始，华为发起“未来种子”项目，陆续推 出和赞助了各类人才培养、竞赛等项目，包括未来种子 奖学金、ICT 学院、开发者学院、云开发者学堂等，致 力于支持业务所在国培养 ICT 人才，累计投入超过 1.5 亿美元，覆盖 150 多个国家，受益人数超过 154 万人。 我们将持续为数字人才培养做贡献。2021 年，华为发 布了“未来种子 2.0”计划，承诺在未来五年投入 1.5 亿美元用于数字化人才培养，新增受益人数预计将超过 300 万人。我们希望，更多的个人、组织加入华为的数 字包容倡议，共同推进数字人才的发展计划。 2021 年年度报告 09 开放合作，持续创新，坚持全球运 营，与合作伙伴共同成长 科技创新是科技公司对人类社会的重要贡献之一。华为 公司努力探索科学技术的前沿，与世界开放合作，坚持 基础研究、坚持开放创新。当前是华为发展的关键时 期，我们仍将进一步加大在研究与创新领域投资，为各 行各业创造增量价值，让每个人、家庭与组织都从技 术进步中受益。2021 年，华为研发费用支出为人民币 1,427 亿元，约占全年收入的 22.4%。过去十年，我们 在研发领域累计投资超过人民币 8,450 亿元，近几年， 每年在基础研究上的投资超过人民币 200 亿元。 我们建立了 86 个基础技术实验室，扎根基础研究，建 立相关的核心技术体系；我们提出了 AI 时代可信计算 的新架构模型，实现多语言、跨平台运行，实现数据处 理量提升 3 个数量级；我们与合作伙伴一起共同促进欧 拉和鸿蒙开源生态的繁荣，共同打造覆盖未来计算场景 的开源操作系统，促进算力和数据流通，早日实现算力 普惠，让算力像电力一样非常方便地按需而用。 同时，我们持续开放合作，积极融入全球学术组织，加 强与全球高校、科研机构开展合作。通过与科学家碰 撞、分享难题与知识，共同应对世界级的挑战和难题， 促进科技进步。华为将用最大的诚意，最热烈的呼唤拥 抱和欢迎全世界科学家和优秀人才。过去几年，我们在 北京、上海、深圳等地建立了“黄大年茶思屋”和“九 章院”，希望通过提供宽松、宁静的创新环境，让科学 家们心无旁骛、潜心研究，真正面向未来、开创未来， 为人类社会发展做贡献。 华为坚持全球运营，并实施多元化的供应策略，构建长 期、持续、稳定的供应能力，保障供应连续性和面向未 来的可持续发展。我们与全球上万家供应商和合作伙伴 广泛合作，建立了长期合作关系，加强开放合作，在开 放合作中解决发展中的问题和挑战。华为有信心、有能 力继续与全球合作伙伴共同奋斗，秉持“合作共赢、共 同发展”理念，打造安全、可靠、有竞争力的健康产业 链，持续为全球客户提供优质的产品与服务。 完善公司治理，坚持合规运营， 更好地为全球客户服务 良好的公司治理是公司稳健发展的重要基石。华为坚持 以客户为中心、以奋斗者为本，持续优化公司治理架 构、组织、流程和考核机制，使公司长期保持有效增 长。2021 年，持股员工代表会举行了两次会议，审议 了一系列治理文件：通过监事会制度，进一步完善公司 监事会的定位和职权，规范监事会运作；逐步明确集团 董事会与子公司董事会的相互关系。同时，面向一线组 织，采用数字化的方式，构建疫情下的新型业务管理体 系和平台，持续加强数字化建设与运营，快速响应，实 现“多打粮食，增加土地肥力”。 华为坚持以法律遵从的确定性，应对国际政治的不确定 性。坚持诚信经营、恪守商业道德、致力于遵从业务所 在国适用的法律法规是华为管理层一直秉持的核心理 念。公司的基本政策要求全球各子公司、各部门遵从所 在国家和地区适用的法律法规。我们重视并持续营造诚 信文化，要求每一位员工遵守商业行为准则。 同时，作为全球化公司，华为持续积极与外界开放沟 通，进一步开放透明，让所有客户、合作伙伴和利益 相关人更加深入了解华为。我们也欢迎各国政府、媒 体、专家学者等各界人士来访公司，了解和认识真实的 华为。 没有退路就是胜利之路。华为的选择注定是一条布满荆 棘，但同时也是值得为之奋斗的漫漫长路。未来，尽管 面临更多不确定性，华为将坚持开放合作，保持战略定 力，保证公司的生存与发展。无论前路多么坎坷，我们 将始终坚守愿景与使命：把数字世界带入每个人、每个 家庭、每个组织，构建万物互联的智能世界。请总结以上内容主题", + "3294": "智能手表（Smart Watch）是安装有嵌入式系统、用于增强基于报时等功能的腕部手表，其功能相似于一台个人数码助理。智能手表除指示时间之外，还应具有提醒、导航、校准、监测、交互等其中一种或者多种功能；显示方式包括指针、数字、图像等。《2023-2027年智能手表市场调查研究报告》研究显示：随着科技的进步和发展，智能穿戴设备市场从2012年至今得到了快速增长，在这10年间，有越来越多的玩家加入，智能穿戴设备产品不管是在技术上、还是使用体验上，都有了很大提升和改善。智能手表是智能穿戴的主要代表之一，在健康监测、记步、拨打电话、定位、与智能家居联动等功能的加持下，广受欢迎。智能手表“智能”的地方体现在，它拥有一套独立的嵌入式操作系统，有一个数据处理中心，需要调用各类传感器收集到的信息，还要有屏幕、存储器、电池、电源管理系统、无线射频系统等，在内部芯片用料和结构设计上与智能手机较为相似。2017年我国智能手表市场规模为131.23亿元，2021年为317.58亿元，同比2020年增长16.60%。脱硝催化剂泛指选择性催化还原（SCR）脱硝催化剂，也叫SCR脱硝催化剂。SCR脱硝催化剂是指应用于选择性催化还原（SCR）脱硝系统上的催化剂，是在选择性催化还原（SCR）脱硝技术基础上发展起来的。《脱硝催化剂行业数据深度调研分析与发展战略规划报告》研究显示:早在2011年，环保部颁布了《火电厂大气污染物排放标准》，开启中国火电环保风暴，各火电厂纷纷加快升级改造步伐。将火电厂脱硝催化剂市场拆分为二部分：新建机组的初装需求以及已投运机组的补装（更换）需求。2021年新建机组配套催化剂需求为3.89万m3，已投运的脱硝机组催化剂更换需求为24.05万m3，合计27.94万m3。酸枣（学名：Ziziphus jujuba Mill. var. spinosa（Bunge）Hu ex H. F. Chow）鼠李科枣属植物，是枣的变种。又名棘、棘子、野枣、山枣、葛针等，原产中国华北，中南各省亦有分布。多野生，常为灌木，也有的为小乔木。树势较强。枝、叶、花的形态与普通枣相似，但枝条节间较短，托刺发达，除生长枝各节均具托刺外，结果枝托叶也成尖细的托刺。叶小而密生，果小、多圆或椭圆形、果皮厚、光滑、紫红或紫褐色，肉薄，味大多很酸，核圆或椭圆形，核面较光滑，内含种子1至2枚，种仁饱满可作中药。其适应性较普通枣强，花期很长，可为蜜源植物。果皮红色或紫红色，果肉较薄、疏松，味酸甜。《酸枣行业现状分析及发展战略研究报告》研究显示：随着生活节奏的不断加快，出现睡眠障碍的人群不断增多，身心健康状况也逐渐下降。酸枣常被用来治疗心烦失眠、多梦、盗汗、易惊等病。通过深入挖掘酸枣助眠机理，开发相应产品，可改善人们的睡眠质量及健康状况，为健康中国助力。2017年，中国酸枣行业市场规模为17.67亿元，2021年为33.24亿元，同比增长4.25%。丙烯酸树脂（acrylic resin）是丙烯酸、甲基丙烯酸及其衍生物聚合物的总称。丙烯酸树脂涂料就是以（甲基）丙烯酸酯、苯乙烯为主体，同其他丙烯酸酯共聚所得丙烯酸树脂制得的热塑性或热固性树脂涂料或丙烯酸辐射涂料。含羟基丙烯酸树脂是指加入含羟基的丙烯酸酯和甲基丙烯酸酯等单体，使之与其他（甲基）丙烯酸类单体一起共聚而制成的带羟基官能团的丙烯酸树脂。一般情况下，羟基丙烯酸树脂和热固性树脂含有-OH基团，部分热塑性树脂也含有少量-OH。羟基是丙烯酸树脂侧链上作为后期交联的活性官能团，作用是增大丙烯酸树脂的极性，改良漆膜的附着力和润湿分散性。因此，丙烯酸树脂中羟基含量越大，使得漆膜交联密度变大，制备出的涂料漆膜物理性、化学性、机械性能和漆膜外观越好。同时，羟基含量越大时，可加快-OH基团与-NCO基团的活性反应，提高涂料的固化交联速度和干燥时间。《含羟基丙烯酸树脂行业数据深度调研分析与发展战略规划报告》研究显示：随着世界各国对环保问题的逐渐重视，涂料行业正在逐渐由传统的有机溶剂型涂料向环保型涂料转型，为响应新型涂料发展的需要，制备“高性能、低污染、长寿命的环保型涂料产品具有极高的社会意义。而限制环保型涂料发展的最重要原因是环保型树脂的升级，水性树脂作为环保型树脂中最重要的品类之一，是“油转水”转型中最重要的一环；而含羟基丙烯酸树脂具有“高硬度、高光泽、高保护”的使用性能，在工业领域中也得到广泛应用。2017年我国含羟基丙烯酸树脂行业产能为62.45万吨，2021年产能为85.82万吨，同比2020年增长7.25%。城市轨道交通是城市公共交通的骨干，具有节能、省地、运量大、全天候、无污染（或少污染）又安全等特点，属绿色环保交通体系，特别适应于大中城市。进入新世纪，尤其是2008年以后，我国通过扩大内需，促进经济平稳增长的一揽子计划，带动了国内基础建设的发展，同时我国大型城市逐渐面临交通拥堵的问题，加快了城市轨道交通建设。《城市轨道交通行业数据深度调研分析与发展战略规划报告》研究显示：根据城市轨道交通协会发布的数据显示，我国城市轨道交通运营里程稳步攀升，并且我国已经成为了全球城市轨道运营里程数第一，远超德国、俄罗斯、美国等发达国家。2021年中国开通轨道交通的城市数量从2015年的26个增长到51个，在6年期间开通轨道交通的城市数量几乎翻备。2021年，中国城市轨道交通运营里程8708公里，较2015年的3618公里，是2015年3618公里的2.4倍。别墅电梯即安装在私人住宅中，仅供单一家庭成员使用的电梯。它也可以安装在非单一家庭使用的建筑物内，作为单一家庭进入其住所的工具，但是建筑物内的公众或其他居住者无法进入和使用。《别墅电梯行业现状分析及发展战略研究报告》研究显示：作为基础设施配套工程的重要组成部分，电梯与国家经济建设尤其是房地产的发展以及人民生活质量的提高密切相关。近些年，随着全球人口增长、城市化进程加快以及人们对便捷生活要求的提高，别墅电梯得到越来越广泛的使用。目前，国际别墅电梯市场呈现发达国家和地区需求稳步增长、新兴市场需求快速增长的特征。2017年全球别墅电梯行业产能规模为4.79万台，2021年为7.89万台，同比2020年增长12.55%。物质、能量和信息是构成自然界的基本要素。“能源”这一术语，过去人们谈论得很少，正是两次石油危机使它成了人们议论的热点。关于能源的定义，约有20种。例如：说：“能源是可从其获得热、光和动力之类能量的资源”；《大英百科全书》说：“能源是一个包括着所有燃料、流水、阳光和风的术语，人类用适当的转换手段便可让它为自己提供所需的能量”；《日本大百科全书》说：“在各种生产活动中，我们利用热能、机械能、光能、电能等来作功，可利用来作为这些能量源泉的自然界中的各种载体，称为能源”；我国的《能源百科全书》说：“能源是可以直接或经转换提供人类所需的光、热、动力等任一形式能量的载能体资源。”可见，能源是一种呈多种形式的，且可以相互转换的能量的源泉。确切而简单地说，能源是自然界中能为人类提供某种形式能量的物质资源。能源亦称能量资源或能源资源。是指可产生各种能量（如热量、电能、光能和机械能等）或可作功的物质的统称。是指能够直接取得或者通过加工、转换而取得有用能的各种资源，包括煤炭、原油、天然气、煤层气、水能、核能、风能、太阳能、地热能、生物质能等一次能源和电力、热力、成品油等二次能源，以及其他新能源和可再生能源。能源（Energy Source）亦称能量资源或能源资源，是国民经济的重要物质基础，未来国家命运取决于能源的掌控。能源的开发和有效利用程度以及人均消费量是生产技术和生活水平的重要标志。（中国大百科全书·机械工程卷）在《中华人民共和国节约能源法》中所称能源，是指煤炭、石油、天然气、生物质能和电力、热力以及其他直接或者通过加工、转换而取得有用能的各种资源。《能源领域信息化与IT应用行业数据深度调研分析与发展战略规划报告》研究显示：“互联网+智慧能源”的理念推动着能源行业的改革，传统企业需要思考如何将重资产的使用效率提升，进一步拥抱数字化，而运维服务也是企业对外输出的核心竞争力之一。在数字化时代，企业数据上云成为趋势，云计算和云存储成为重要的数字化能力。然而对于关系国计民生的能源行业来说，大力推进绿色低碳科技创新将成为碳中和发展的主要路径，推动绿色低碳技术重大突破，加快能源行业的数字化转型，变得尤为重要。能源云是紧密结合能源行业业务，用云计算、大数据、移动、物联网、人工智能等新一代的技术架构和体系为能源企业提供的云服务平台，它涵盖行业云服务、领域云服务、云工具和云平台，覆盖能源企业运营中的采购、生产、营销、数据、财务、人力、社交与协同办公等多个业务场景和领域，它可以与传统软件融合，让企业的业务自然延伸到云端应用。采用以公有云、专属云、混合云等云服务模式为主，行业解决方案为辅的多种服务形式，为能源企业互联网化提供坚实的支撑和保障。公有云通常指第三方提供商为用户提供的能够使用的云，公有云一般可通过Internet使用，可能是免费或成本低廉的，公有云的核心属性是共享资源服务。花青素（anthocyan），又称花色素，是自然界一类广泛存在于植物中的水溶性天然色素，是花色苷（anthocyains）水解而得的有颜色的苷元。水果、蔬菜、花卉中的主要呈色物质大部分与之有关。在植物细胞液泡不同的PH值条件下，花青素使花瓣呈现五彩缤纷的颜色。已知花青素有20多种，食物中重要的有6种，即天竺葵色素、矢车菊色素、飞燕草色素、芍药色素、牵牛花色素和锦葵色素。自然状态的花青素都以糖苷形式存在，称为花色苷，很少有游离的花青素存在。《花青素市场现状分析及行业前景预测报告》研究显示：我国花青素开发较晚，且主要以粗制品为主，2011年之前，我国对花青素的提取基本上是空白。2011年臻多美研发成功的花青素提取和保存两项专利技术，填补了国内花青素技术的空白，打破了之前国内花青素精制品90%靠进口的局面，并大量销往国外。随着人们健康意识的加强，以及我国与国际规范的接轨，对花青素的需求逐渐凸显。2017年我国花青素市场规模为3.44亿元，2021年为6.93亿元，同比2020年增长17.20%。发动机（Engine）是一种能够把其它形式的能转化为机械能的机器，包括如内燃机（往复活塞式发动机）、外燃机（斯特林发动机、蒸汽机等）、喷气发动机、电动机等。如内燃机通常是把化学能转化为机械能。发动机既适用于动力发生装置，也可指包括动力装置的整个机器（如：汽油发动机、航空发动机）。从字面上理解，道路发动机就是用于驱动在公共道路上行驶的车辆的发动机，而非道路发动机就是所有这个用途以外的发动机，比如船用发动机、工程机械发动机、发电用发动机、航空发动机等等。《非道路发动机行业数据深度调研分析与发展战略规划报告》研究显示：近年来，全球非道路发动机市场呈现稳定态势，到2019年市场产能增长至4726.31万台；2020年，新冠疫情爆发，非道路发动机产能下降至4526.39万台；2021年产能增速恢复至10.78%，总体产能达5014.27万台。发动机是一种能够把其它形式的能转化为机械能的机器，发动机既适用于动力发生装置，也可指包括动力装置的整个机器，农用发动机就是农业机械用发动机。农业机械是指在作物种植业和畜牧业生产过程中，以及农、畜产品初加工和处理过程中所使用的各种机械。《农用发动机行业现状分析及发展战略研究报告》研究显示：随着农业供给侧结构性改革的推进，农用发动机行业进入了深度调整期，技术升级，结构优化也在同步进行中。农业是支撑国民经济建设与发展的基础产业，而农业机械化是建设现代农业的重要物质基础，也是实现农业现代化的重要标志与内容。近年来，在国家政策补贴支持及机械化率稳步增长等因素的推动下，农用发动机行业规模不断增长。市场规模方面，据统计，近年来我国农用发动机市场规模不断增长，截至2021年我国农用发动机市场规模为429.61亿元，同比增长27.60%。为适应文明演进的新趋势和新要求，人类必须从根本上解决文明前行的动力困扰，实现能源的安全、稳定、清洁和永续利用。智慧能源就是充分开发人类的智力和能力，通过不断技术创新和制度变革，在能源开发利用、生产消费的全过程和各环节融汇人类独有的智慧，建立和完善符合生态文明和可持续发展要求的能源技术和能源制度体系，从而呈现出的一种全新能源形式。简而言之，智慧能源就是指拥有自组织、自检查、自平衡、自优化等人类大脑功能，满足系统、安全、清洁和经济要求的能源形式。《智慧能源行业现状分析及发展战略研究报告》研究显示：电力行业是智慧能源行业最大的市场，随着我国经济的快速发展，智慧能源行业作为电力行业产业链中的重要一环也随之蓬勃发展，尤其近几年我国经济发展中面临能源、电力紧张的瓶颈性问题，国家不断加大对智慧能源行业的投资，使得该行业步入了飞跃发展期，2021年中国智能电网市场规模已经达到854.6亿元。请从上述信息中总结出最佳的投资方向并给出理由：" +} diff --git a/benchmark/baichuan2-7b/run-baichuan-7b.sh b/benchmark/baichuan2-7b/run-baichuan-7b.sh new file mode 100755 index 0000000..28249c1 --- /dev/null +++ b/benchmark/baichuan2-7b/run-baichuan-7b.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# On different systems, different models and differnt modes, OMP_NUM_THREADS need to be ajusted. +output=$(numactl -H) +first_line=$(echo "$output" | head -n 1) +nodes=$(echo "$first_line" | awk '{print $2}') + +echo "memory node number: $nodes" +if [ "$nodes" -eq 16 ]; then +#HBM SNC-4 mode, Confirm that there are 8 HBM memory nodes and 8 DRAM memory nodes through "numactl -H" +#0-7 is DRAM memory node, 8-15 is HBM node + echo "HBM SNC4 mode" +# Run Baichuan2-7B on 1 socket HBM SNC4 mode + OMP_NUM_THREADS=12 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -m 8 -N 0 sh baichuan2-7b.sh : \ + -n 1 numactl -m 9 -N 1 sh baichuan2-7b.sh : \ + -n 1 numactl -m 10 -N 2 sh baichuan2-7b.sh : \ + -n 1 numactl -m 11 -N 3 sh baichuan2-7b.sh +# -n 1 numactl -m 12 -N 4 sh baichuan2-7b.sh : \ +# -n 1 numactl -m 13 -N 5 sh baichuan2-7b.sh : \ +# -n 1 numactl -m 14 -N 6 sh baichuan2-7b.sh : \ +# -n 1 numactl -m 15 -N 7 sh baichuan2-7b.sh + +elif [ "$nodes" -eq 4 ]; then +#HBM Quad-mode, Confirm that there are 2 HBM memory nodes and 2 DRAM memory nodes through "nuamctl -H" + echo "HBM Quad mode" +# Run Baichuan2-7B on 1 socket HBM Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 2 sh baichuan2-7b.sh : \ + -n 1 numactl -N 1 -m 3 sh baichuan2-7b.sh + +elif [ "$nodes" -eq 2 ]; then +#SPR Quad-mode, Confirm that there are 2 DRAM memory nodes through "nuamctl -H" + echo "SPR Quad mode" +# Run Baichuan2-7B on 1 socket SPR Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 sh baichuan2-7b.sh + +else + echo "Please double check the memory nodes" +fi + + diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py new file mode 100644 index 0000000..e65f81f --- /dev/null +++ b/benchmark/benchmark.py @@ -0,0 +1,128 @@ +import os + +# Ignore Tensor-RT warning from huggingface +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" + +import torch +import time +from transformers import AutoTokenizer, TextStreamer +import json +import pathlib + +import argparse + + +def boolean_string(string): + low_string = string.lower() + if low_string not in {"false", "true"}: + raise ValueError("Not a valid boolean string") + return low_string == "true" + + +DTYPE_LIST = ["fp16", "bf16", "int8", "bf16_fp16", "bf16_int8"] + +parser = argparse.ArgumentParser() +parser.add_argument("--token_path", type=str, default="/data/chatglm-6b", help="Path to token file") +parser.add_argument("--model_path", type=str, default="/data/chatglm-6b/cpu", help="Path to model file") +parser.add_argument("--model_name", type=str, default="Model", help="Model name") +parser.add_argument("--prompt_path", type=str, default="./prompt_pool.json", help="Path to model file") +parser.add_argument("--token_in", type=str, default=32, help="Input Token Len") +parser.add_argument("--token_out", type=int, default=32, help="Output Token Len, MaxLen=IN+OUT") +parser.add_argument("--beam_width", type=int, default=1, help="Beam Search Width") +parser.add_argument("--input_prompt", type=str, default=None, help="Input Prompt") +parser.add_argument("--batch_size", type=int, default=1, help="Batch size") +parser.add_argument("--iteration", type=int, default=10, help=" Benchmakr Iterations") +parser.add_argument("--warmup", type=int, default=2, help="Warm up Iterations") +parser.add_argument("--dtype", type=str, choices=DTYPE_LIST, default="fp16", help="Data type") +parser.add_argument("--padding", help="Enable padding, Default to True.", type=boolean_string, default=True) + +import importlib.util + +xft_spec = importlib.util.find_spec("xfastertransformer") + +if xft_spec is None: + import sys + + sys.path.append("../../src") + print("[INFO] xfastertransformer is not installed in pip, using source code.") +else: + print("[INFO] xfastertransformer is installed, using pip installed package.") + + +import xfastertransformer + +if __name__ == "__main__": + args = parser.parse_args() + with open(args.prompt_path, "r") as json_file: + prompt_pool = json.load(json_file) + + tokenizer = AutoTokenizer.from_pretrained( + args.token_path, use_fast=False, padding_side="left", trust_remote_code=True + ) + model = xfastertransformer.AutoModel.from_pretrained(args.model_path, dtype=args.dtype) + input_prompts = [] + if model.rank == 0: + # input prompt + print("======start=======") + if args.input_prompt is not None: + input_prompt = args.input_prompt + elif args.token_in in prompt_pool: + input_prompt = prompt_pool[args.token_in] + print(input_prompt) + else: + raise SystemExit("[ERROR] Plese use --input_prompt if you want custom input.") + for _ in range(args.batch_size): + input_prompts.append(input_prompt) + # Master + input_ids = tokenizer(input_prompts, return_tensors="pt", padding=args.padding).input_ids + input_token_nums = int(torch.numel(input_ids) / args.batch_size) + print("Input token Length is", input_token_nums) + print("Batch_size:", args.batch_size) + max_len = input_token_nums + args.token_out + print("Max_len:", max_len) + print("=" * 50) + # Perform 100 runs and store execution times in a list + execution_times = [] + first_token_times = [] + remained_token_times = [] + # warm up + for i in range(args.warmup): + model.generate(input_ids, num_beams=args.beam_width, max_length=max_len, streamer=None) + print("Start benchmark:") + for i in range(args.iteration): + print("iteration", i, ":") + model.config(max_length=max_len, num_beams=args.beam_width) + model.input(input_ids) + # first token + start_time = time.perf_counter() + next_tokens = model.forward() + end_time = time.perf_counter() + first_token_time = end_time - start_time + first_token_times.append(first_token_time) + # remaining tokens + cost_list = [] + while not model.is_done(): + start_time = time.perf_counter() + next_tokens = model.forward() + end_time = time.perf_counter() + cost_list.append(end_time - start_time) + # print(next_tokens) + generated_ids = model.finalize() + remained_token_times.append(sum(cost_list)) + + output_token_nums = int(torch.numel(generated_ids) / args.batch_size) - input_token_nums + # Sort the execution times in ascending order + remained_token_times.sort() + # Get the 90th element (index 89) from the sorted list + latency_90 = remained_token_times[int(args.iteration * 0.9) - 1] * 1000 / (output_token_nums - 1) + # Calculate the first token latency + first_token_latency = sum(first_token_times) / len(first_token_times) * 1000 + Next_token_throughput = 1000 / latency_90 * args.batch_size + print("\n") + print("=" * 50 + args.model_name + " Final Performance" + "=" * 50) + print(f"First token Latency:\t{first_token_latency:.2f} ms") + print(f"Next token Latency:\t{latency_90:.2f} ms") + print(f"Throughput without 1st token:\t{Next_token_throughput:.2f} tokens/s") + else: + for i in range(args.warmup + args.iteration): + model.generate() diff --git a/benchmark/chatglm-6b/chatglm-6b.sh b/benchmark/chatglm-6b/chatglm-6b.sh new file mode 100755 index 0000000..1794f3e --- /dev/null +++ b/benchmark/chatglm-6b/chatglm-6b.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "FP16 Performance " +python "${SCRIPT_DIR}"/../benchmark.py \ + --token_path /data/chatglm-6b \ + --model_path /data/chatglm-6b/cpu \ + --prompt_path "${SCRIPT_DIR}"/prompt_pool.json \ + --model_name "ChatGLM-6B" \ + --dtype fp16 \ + --token_in 32 \ + --token_out 32 --beam_width 1 --iteration 100 + +# In this benchmark case, token_in only can be "demo","32","64","128","256","512","1024","2016" +# "32" means the token length is 32, if needs more test, add it into input_token.py + diff --git a/benchmark/chatglm-6b/prompt_pool.json b/benchmark/chatglm-6b/prompt_pool.json new file mode 100644 index 0000000..19253bd --- /dev/null +++ b/benchmark/chatglm-6b/prompt_pool.json @@ -0,0 +1,10 @@ +{ + "demo": "从前有个小姑娘，她非常喜欢冒险", + "32": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满", + "64": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原", + "128": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨", + "256": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，", + "512": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，发现里面的一切都超出了她的想象。墙壁上挂满了古老的画作，地板上铺满了华丽的地毯，而在城堡的深处，她发现了一个闪烁着宝石的宝箱。当她打开宝箱时，她发现里面并没有金银财宝，而是一面镜子。当她凝视着镜中的自己时，她突然明白，整个探险的旅程其实是在寻找自己内心的勇气和智慧。艾莉丝明白了，冒险并不仅仅是外部世界的探索，更是内心的成长和启发。她带着这份领悟，回到了她宁静的小村庄，分享着她的经历和故事，激励着其他小孩子们去追寻自己的冒险和梦想。从此以后，小村庄的孩子们都称艾莉丝为“勇敢的冒险者”，而她的故事也在小村庄中代代相传，成为了鼓舞人心的传奇。在勇敢的冒险者艾莉丝的激励下，小村庄的氛围变得热闹非凡。孩子们开始组成小队，模仿着艾莉丝的足迹，寻找着属于他们自己的冒险。每个人都在自己的心中埋下了一颗探索未知的种子。艾莉丝成为了村子里的一位贵人。她不仅与孩子们分享自己的故事，还在村子中建立了一个冒险学校，教授孩子们如何在不同的情况下保持勇气和坚韧，如何寻找创新的解决方案。孩子们在冒险学校中学到了团队合作、领导能力和适应变化的重要性。而艾莉丝自己也在教导的过程中不断成长。她意识到，冒险", + "1024": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，发现里面的一切都超出了她的想象。墙壁上挂满了古老的画作，地板上铺满了华丽的地毯，而在城堡的深处，她发现了一个闪烁着宝石的宝箱。当她打开宝箱时，她发现里面并没有金银财宝，而是一面镜子。当她凝视着镜中的自己时，她突然明白，整个探险的旅程其实是在寻找自己内心的勇气和智慧。艾莉丝明白了，冒险并不仅仅是外部世界的探索，更是内心的成长和启发。她带着这份领悟，回到了她宁静的小村庄，分享着她的经历和故事，激励着其他小孩子们去追寻自己的冒险和梦想。从此以后，小村庄的孩子们都称艾莉丝为“勇敢的冒险者”，而她的故事也在小村庄中代代相传，成为了鼓舞人心的传奇。在勇敢的冒险者艾莉丝的激励下，小村庄的氛围变得热闹非凡。孩子们开始组成小队，模仿着艾莉丝的足迹，寻找着属于他们自己的冒险。每个人都在自己的心中埋下了一颗探索未知的种子。艾莉丝成为了村子里的一位贵人。她不仅与孩子们分享自己的故事，还在村子中建立了一个冒险学校，教授孩子们如何在不同的情况下保持勇气和坚韧，如何寻找创新的解决方案。孩子们在冒险学校中学到了团队合作、领导能力和适应变化的重要性。而艾莉丝自己也在教导的过程中不断成长。她意识到，真正的冒险不仅仅是探索外部的世界，还包括挑战自己的极限，克服内心的恐惧，勇敢地去面对生活中的各种困难和未知。随着时间的推移，艾莉丝的名字传遍了整个大陆。人们远道而来，想要亲自听她的故事，领略她的智慧。她的冒险学校也成了一个聚集智慧和勇气的地方，吸引着来自各地的年轻人前来学习。然而，即使在成就了如此多的事情后，艾莉丝仍然保持着谦虚和平和的态度。她知道，冒险的旅程永无止境，人类的潜力是无限的。她希望每个人都能找到自己的内心冒险，不断超越自己，创造属于自己的传奇。在那个小村庄里，勇敢的冒险者艾莉丝的故事成为了永恒的传说，永远激励着人们勇往直前，追寻梦想，挑战未知，探索生命的意义。而每当夕阳余晖洒在小村庄的屋顶上，人们都会凝视着远方的群山，心中充满了无限的勇气和希望。然而，故事的英雄不止是艾莉丝，而是每一个被她激励的人。小村庄逐渐变成了一个充满活力和创意的社区，每个人都在日常生活中追求着自己的激情和目标。年复一年，冒险学校的学生们走出村庄，将艾莉丝的智慧和勇气传播到更远的地方。他们成为了各行各业的领袖、创新者和慈善家。有的人建立起了新的城市，有的人探索未知的星球，有的人用自己的音乐、艺术和文字，将勇气的故事传唱下去。艾莉丝自己也并未停止探险的脚步。随着时间的流逝，她继续走访世界的角落，分享她的经验和智慧。她成了一位演讲家和作家，用自己的文字激励着更多人敢于追求自己的梦想，迎接人生的挑战。然而，在她的探险旅程中，有一次，她再次回到了那座古老的城堡。城堡的大门仍然敞开着，但这一次，里面没有镜子，而是一面写满了文字的墙壁。墙上记录着无数个人的名字和他们的冒险故事，每一个名字都是一个闪耀的星星，代表着他们的勇气和贡献。艾莉丝感叹着，她的初心从未改变。她明白，一个人的冒险不仅影响着自己，更能启发他人，甚至改变整个世界。她在城堡里停留了一段时间，默默地感受着所有人的努力和奋斗，同时也为自己的冒险旅程感到骄傲。最终，艾莉丝返回了她宁静的小村庄，坐在家门前的台阶上，凝视着远方的群山，回想起自己的冒险之旅。她心中充满了满足和感激，因为她知道，她所追寻的不仅仅是外部的世界，更是内心的成长和连接。她的故事，已经成为了永恒的传奇，为无数人点燃了勇气的火焰，鼓舞着他们追求自己的梦想，创造属于自己的独特冒险。这段文字讲述了一个小女孩艾莉丝的冒险故事,她非常喜欢探险,听到一个传说后决定前往群山深处寻找古老的城堡。她克服了各种艰难险阻，", + "2016": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，发现里面的一切都超出了她的想象。墙壁上挂满了古老的画作，地板上铺满了华丽的地毯，而在城堡的深处，她发现了一个闪烁着宝石的宝箱。当她打开宝箱时，她发现里面并没有金银财宝，而是一面镜子。当她凝视着镜中的自己时，她突然明白，整个探险的旅程其实是在寻找自己内心的勇气和智慧。艾莉丝明白了，冒险并不仅仅是外部世界的探索，更是内心的成长和启发。她带着这份领悟，回到了她宁静的小村庄，分享着她的经历和故事，激励着其他小孩子们去追寻自己的冒险和梦想。从此以后，小村庄的孩子们都称艾莉丝为“勇敢的冒险者”，而她的故事也在小村庄中代代相传，成为了鼓舞人心的传奇。在勇敢的冒险者艾莉丝的激励下，小村庄的氛围变得热闹非凡。孩子们开始组成小队，模仿着艾莉丝的足迹，寻找着属于他们自己的冒险。每个人都在自己的心中埋下了一颗探索未知的种子。艾莉丝成为了村子里的一位贵人。她不仅与孩子们分享自己的故事，还在村子中建立了一个冒险学校，教授孩子们如何在不同的情况下保持勇气和坚韧，如何寻找创新的解决方案。孩子们在冒险学校中学到了团队合作、领导能力和适应变化的重要性。而艾莉丝自己也在教导的过程中不断成长。她意识到，真正的冒险不仅仅是探索外部的世界，还包括挑战自己的极限，克服内心的恐惧，勇敢地去面对生活中的各种困难和未知。随着时间的推移，艾莉丝的名字传遍了整个大陆。人们远道而来，想要亲自听她的故事，领略她的智慧。她的冒险学校也成了一个聚集智慧和勇气的地方，吸引着来自各地的年轻人前来学习。然而，即使在成就了如此多的事情后，艾莉丝仍然保持着谦虚和平和的态度。她知道，冒险的旅程永无止境，人类的潜力是无限的。她希望每个人都能找到自己的内心冒险，不断超越自己，创造属于自己的传奇。在那个小村庄里，勇敢的冒险者艾莉丝的故事成为了永恒的传说，永远激励着人们勇往直前，追寻梦想，挑战未知，探索生命的意义。而每当夕阳余晖洒在小村庄的屋顶上，人们都会凝视着远方的群山，心中充满了无限的勇气和希望。然而，故事的英雄不止是艾莉丝，而是每一个被她激励的人。小村庄逐渐变成了一个充满活力和创意的社区，每个人都在日常生活中追求着自己的激情和目标。年复一年，冒险学校的学生们走出村庄，将艾莉丝的智慧和勇气传播到更远的地方。他们成为了各行各业的领袖、创新者和慈善家。有的人建立起了新的城市，有的人探索未知的星球，有的人用自己的音乐、艺术和文字，将勇气的故事传唱下去。艾莉丝自己也并未停止探险的脚步。随着时间的流逝，她继续走访世界的角落，分享她的经验和智慧。她成了一位演讲家和作家，用自己的文字激励着更多人敢于追求自己的梦想，迎接人生的挑战。然而，在她的探险旅程中，有一次，她再次回到了那座古老的城堡。城堡的大门仍然敞开着，但这一次，里面没有镜子，而是一面写满了文字的墙壁。墙上记录着无数个人的名字和他们的冒险故事，每一个名字都是一个闪耀的星星，代表着他们的勇气和贡献。艾莉丝感叹着，她的初心从未改变。她明白，一个人的冒险不仅影响着自己，更能启发他人，甚至改变整个世界。她在城堡里停留了一段时间，默默地感受着所有人的努力和奋斗，同时也为自己的冒险旅程感到骄傲。最终，艾莉丝返回了她宁静的小村庄，坐在家门前的台阶上，凝视着远方的群山，回想起自己的冒险之旅。她心中充满了满足和感激，因为她知道，她所追寻的不仅仅是外部的世界，更是内心的成长和连接。她的故事，已经成为了永恒的传奇，为无数人点燃了勇气的火焰，鼓舞着他们追求自己的梦想，创造属于自己的独特冒险。在茫茫人海中，每个人都怀揣着一颗探索未知的心，渴望冒险的激情点燃内心的火焰。正如这个温馨而感人的故事中，勇敢的小姑娘艾莉丝用她的冒险之旅，为我们讲述了一个关于勇气、成长和内心启示的故事。这篇评论将以3000字的篇幅，深入探讨故事所传达的主题和智慧，以及它如何在现实生活中激励和影响我们。1.探索未知的冒险精神：艾莉丝是一个生活在宁静小村庄的小姑娘，然而她内心的渴望却像那群远方的群山一样高不可攀。她对未知世界的好奇心和渴望冒险的精神，将她推向了一个前所未有的旅程。故事中的艾莉丝代表着人类永恒的冒险精神，这是人类进步和发展的源动力。正如故事中的艾莉丝一样，我们都可以从内心深处找到勇气，超越自己的舒适区，去追寻那些令人心跳加速的未知。2.内心的成长和智慧之旅：然而，艾莉丝的冒险之旅并不仅仅是在外部世界的探索，更是一次内心的成长和智慧之旅。故事中的她通过一次次的挑战和困难，学会了如何面对恐惧、如何寻找解决问题的方法，如何在困境中坚持不懈。这种内心的成长是一个人在成为更好的自己的过程中不可或缺的一部分。正如镜子所反映的，我们需要不断审视自己，正视内心的弱点和恐惧，才能真正成长为内外兼修的人。3.平凡与传奇的交织：小村庄与古堡是故事中平凡与传奇的两种象征。小村庄是安逸、宁静和普通的代表，而古堡则是神秘、珍贵和特殊的象征。艾莉丝作为小村庄中的孩子，从平凡的生活中催生出无限的梦想。她的冒险旅程不仅是对外部世界的挑战，更是对平凡生活的重新审视和赋予新意。这让我们想起，每个人都有属于自己的特殊之处，平凡中蕴含着无限的可能性。3.冒险的内外联结：艾莉丝的探险旅程也带来了她与外部世界的联结，而这种内外联结也在她的心灵深处发生。她不仅通过探索外部世界来理解自己，也通过面对内心的挑战来理解世界。这种内外联结告诉我们，冒险不仅是追求外部的目标，更是连接内心与外界的过程。只有在与自己、与世界建立真实的联系后，我们才能更加深刻地理解自己和世界。4.故事的智慧和启示：故事中蕴含着丰富的智慧和启示，它们深刻地触及到人类内心的弱点和潜能。首先，故事告诉我们，勇气并不是没有恐惧，而是在恐惧面前依然前行。艾莉丝的勇气激励我们不畏艰险，勇往直前。其次，故事强调了成长的重要性。每一次的挑战和困境都是一次内心的磨练和提升，让我们变得更加坚强和智慧。最后，故事传递了一个深刻的信息：每个人都有内心的勇士，只需要找到勇气去释放他们。5.现实中的启发和影响：这个故事不仅是一个虚构的童话，更是现实生活中的启发和影响。在现实世界中，我们常常被琐事和困难所压制，忘记了自己内心的激情和渴望。艾莉丝的故事提醒我们，无论年龄如何，我们都应该保持对未知的好奇心，保持对生活的热情。当我们面对困难时，我们可以从她的经历中找到力量和勇气，坚持前行。这个故事还告诉我们，每个人都有内心的宝藏，我们需要发掘它们，将它们带到现实中，让它们在生活中发光发热。6.故事的永恒性：正如故事中的古堡和镜子，这个故事也具有永恒的性质。它不仅适用于任何年龄段的人，更适用于不同背景的人。无论是孩子还是成年人，无论是在城市还是在乡村，我们都能从中找到共鸣。这个故事会一直传承下去，激励着更多的人去探索内心的深处，去追求梦想。探索内心的真正意义正如故事中的艾莉丝一样，我们每个人都生活在自己的小村庄，被日常琐事和舒适区所限制。然而，正是勇敢的内心，驱使着我们追求更大的梦想和意义。艾莉丝的故事鼓励我们不要被舒适所束缚，而是要勇敢地迈出一步，探索内心深处的渴望和激情。我们可以将自己视为自己的勇敢冒险者，去发现未知、挑战恐惧，以便发现更多关于自己的真正意义。7.勇气与挑战的对立：故事中的艾莉丝经历了无数挑战和困难，但她的勇气和决心却从未动摇。这与现实生活中的我们息息相关。我们也常常面临困境和逆境，而勇气往往就是在面对这些逆境时被唤醒的。正是因为有了挑战，我们才能体验到成长的乐趣和内心的满足感。因此，勇敢面对挑战，正是我们实现梦想和成长的关键。8.共鸣的故事：故事中的艾莉丝和她的冒险旅程，很容易引发读者的共鸣。我们或许不是真的走向大山去寻找宝藏，但我们每个人都曾有过一段属于自己的冒险。这可能是远行他乡，也可能是尝试新的工作，甚至是踏出社交的舒适圈。艾莉丝的故事让我们想起，生活中的每一步都是一次冒险，每个人都在追求内心的勇气和梦想。这种共鸣使得故事更具触动，更容易将其与自身经历联系起来。9.结语：在这个喧嚣的世界里，我们常常迷失了自己，忘记了内心的声音。而《勇敢的冒险者艾莉丝：永恒的传奇》通过温暖的故事，唤起了我们内心的勇气和激情。它不仅是一则童话，更是一本关于生活、成长和内心启示的启迪之书。这个故事不仅鼓舞着我们在现实生活中勇往直前，追求梦想，还引发了我们对自己内心的深刻思考，每个人都是勇敢的冒险者，无论我们走到哪里，都可以从艾莉丝的故事中汲取力量，勇敢地走向未知，追寻属于自己的传奇之路。" +} \ No newline at end of file diff --git a/benchmark/chatglm-6b/run-chatglm.sh b/benchmark/chatglm-6b/run-chatglm.sh new file mode 100755 index 0000000..d8b8c4b --- /dev/null +++ b/benchmark/chatglm-6b/run-chatglm.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# On different systems, different models and differnt modes, OMP_NUM_THREADS need to be ajusted. +output=$(numactl -H) +first_line=$(echo "$output" | head -n 1) +nodes=$(echo "$first_line" | awk '{print $2}') + +echo "memory node number: $nodes" +if [ "$nodes" -eq 16 ]; then +#HBM SNC-4 mode, Confirm that there are 8 HBM memory nodes and 8 DRAM memory nodes through "numactl -H" +#0-7 is DRAM memory node, 8-15 is HBM node + echo "HBM SNC4 mode" +# Run chatglm on 1 socket HBM SNC4 mode + OMP_NUM_THREADS=12 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -m 8 -N 0 sh chatglm-6b.sh : \ + -n 1 numactl -m 9 -N 1 sh chatglm-6b.sh : \ + -n 1 numactl -m 10 -N 2 sh chatglm-6b.sh : \ + -n 1 numactl -m 11 -N 3 sh chatglm-6b.sh +# -n 1 numactl -m 12 -N 4 sh chatglm-6b.sh : \ +# -n 1 numactl -m 13 -N 5 sh chatglm-6b.sh : \ +# -n 1 numactl -m 14 -N 6 sh chatglm-6b.sh : \ +# -n 1 numactl -m 15 -N 7 sh chatglm-6b.sh + +elif [ "$nodes" -eq 4 ]; then +#HBM Quad-mode, Confirm that there are 2 HBM memory nodes and 2 DRAM memory nodes through "nuamctl -H" + echo "HBM Quad mode" +# Run chatglm on 1 socket HBM Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 2 sh chatglm-6b.sh : \ + -n 1 numactl -N 1 -m 3 sh chatglm-6b.sh + +elif [ "$nodes" -eq 2 ]; then +#SPR Quad-mode, Confirm that there are 2 DRAM memory nodes through "nuamctl -H" + echo "SPR Quad mode" +# Run chatglm on 1 socket SPR Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 sh chatglm-6b.sh + +else + echo "Please double check the memory nodes" +fi + + diff --git a/benchmark/chatglm2-6b/chatglm2-6b.sh b/benchmark/chatglm2-6b/chatglm2-6b.sh new file mode 100755 index 0000000..acf8609 --- /dev/null +++ b/benchmark/chatglm2-6b/chatglm2-6b.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +data_type=bf16_fp16 +ilen=3294 +olen=512 +echo "${data_type} Performance, input len ${ilen}, output len ${olen}" +python "${SCRIPT_DIR}"/../benchmark.py \ + --token_path /data/chatglm2-6b \ + --model_path /data/chatglm2-6b/cpu \ + --prompt_path "${SCRIPT_DIR}"/prompt_pool.json \ + --model_name "ChatGLM2-6B" \ + --dtype ${data_type} \ + --token_in ${ilen} \ + --token_out ${olen} --beam_width 1 --iteration 20 + +# In this benchmark case, token_in only can be "9","32","64","128","256","512","1196","2050","3294" +# "32" means the token length is 32, if needs more test, add it into input_token.py + diff --git a/benchmark/chatglm2-6b/prompt_pool.json b/benchmark/chatglm2-6b/prompt_pool.json new file mode 100644 index 0000000..a796f56 --- /dev/null +++ b/benchmark/chatglm2-6b/prompt_pool.json @@ -0,0 +1,13 @@ +{ + "demo": "从前有个小姑娘，她非常喜欢冒险", + "9": "脚崴了该怎么处理?", + "16": "脚不小心崴了应该怎么处理?", + "32": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满", + "64": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原", + "128": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨", + "256": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，", + "512": "从前有个小姑娘，她非常喜欢冒险,这个小姑娘名叫艾莉丝。她生活在一个宁静的小村庄，但她总是对村外的未知世界充满好奇。每当夕阳的余晖洒在小村庄的屋顶上，艾莉丝就会坐在她家门前的台阶上，凝视着远方的群山和茫茫草原。她想象着那些山脉后面隐藏着怎样的冒险和故事。有一天，村子里传来了一则神秘的传说。据说，在群山的深处，有一座神秘的古老城堡，城堡里珍藏着无数宝藏和奇幻的生物。听到这个传说，艾莉丝的心燃起了火焰，她决定踏上探险的征程。一个清晨,在一个清晨，艾莉丝偷偷地离开了家，背着一个小包，装满了她认为会在冒险中有用的东西。她穿过茂密的森林，越过草原，攀登山脉，一步一个脚印地走向未知。在探险的过程中，她遇到了许多挑战和难题。有一次，她遇到了一只受伤的小动物，艾莉丝心地善良，不顾一切地照料它，直到它康复起来。在另一次，她遇到了一个神秘的迷宫，需要解开谜题才能通过。每一次的挑战，都让艾莉丝变得更加坚强和聪明。终于，她来到了传说中的古老城堡。城堡的大门敞开着，透出一股神秘的光芒。艾莉丝走进城堡，发现里面的一切都超出了她的想象。墙壁上挂满了古老的画作，地板上铺满了华丽的地毯，而在城堡的深处，她发现了一个闪烁着宝石的宝箱。当她打开宝箱时，她发现里面并没有金银财宝，而是一面镜子。当她凝视着镜中的自己时，她突然明白，整个探险的旅程其实是在寻找自己内心的勇气和智慧。艾莉丝明白了，冒险并不仅仅是外部世界的探索，更是内心的成长和启发。她带着这份领悟，回到了她宁静的小村庄，分享着她的经历和故事，激励着其他小孩子们去追寻自己的冒险和梦想。从此以后，小村庄的孩子们都称艾莉丝为“勇敢的冒险者”，而她的故事也在小村庄中代代相传，成为了鼓舞人心的传奇。在勇敢的冒险者艾莉丝的激励下，小村庄的氛围变得热闹非凡。孩子们开始组成小队，模仿着艾莉丝的足迹，寻找着属于他们自己的冒险。每个人都在自己的心中埋下了一颗探索未知的种子。艾莉丝成为了村子里的一位贵人。她不仅与孩子们分享自己的故事，还在村子中建立了一个冒险学校，教授孩子们如何在不同的情况下保持勇气和坚韧，如何寻找创新的解决方案。孩子们在冒险学校中学到了团队合作、领导能力和适应变化的重要性。而艾莉丝自己也在教导的过程中不断成长。她意识到，冒险", + "1196": "这是一个全民AI的时代。如果你不能张口ChatGPT、闭口大模型，都不好啥意思跟人打招呼。如果你不在AI上搞点东西，都不好意思说自己是科技企业。当然了，AI的历史其实相当悠久，远不只是对个话>、做个图那么简单。无论是云侧还是端侧，无论是生成式还是决策式，无论硬件还是算法，无论是训练推理还是应用场景，都是相当深奥的学问。想真正做好AI，基础硬件、开发软件、生态场景都缺一不可，必须高效、合理地处理各种各样的数据、模型、应用，真正落到使用。能有如此综合实力的企业屈指可数，Intel无疑就是一个典型标杆，从云到端都有丰富的AI解决方案，CPU通用处理器、GPU加速器、AI加速器任君按需>选择。7月11日，Intel在中国举办了Intel AI产品战略暨Gaudi2新品发布会，正式面向中国市场推出第二代深度学习加速器——Habana Gaudi2。Intel Gaudi2加速器不但拥有极高的深度学习性能、效率，最大优势就>是极高的性价比，对于中国用户来说堪称大规模部署AI的上佳之选。Intel执行副总裁兼数据中心与人工智能事业部总经理Sandra Rivera在发布会上表示：“Intel致力于通过为客户提供广泛的硬件选择，并支持开放的软件环境，加速AI技术的发展。凭借包括至强可扩展处理器、Gaudi2深度学习加速器在内的产品组合，Intel正在降低AI的准入门槛，并强化客户在云端通过网络和智能边缘部署这一关键业务技术的能力，从而帮>助构建中国AI的未来。”Habana Labs成立于2016年，致力于研发世界一流的AI加速器，满足人工智能、深度学习计算快速发展的需求，创业初期就得到了Intel的投资，2019年12月被Intel正式收购。Habana的第二代加速器Gaudi2采用台积电7nm工艺制造，集成24个可编程的Tenor张量核心(TPC)、48MB SRAM缓存、21个10万兆内部互连以太网接口(ROCEv2 RDMA)、96GB HBM2E高带宽内存(总带宽2.4TB/s)、多媒体引擎等，支持PCIe 4.0 x16，最高功耗800W。基于Gaudi2加速器芯片，Intel还设计了夹层卡HL-225B，采用标准的OAM封装接口，方便客户部署与使用。凭借高性能和高效扩展性，Gaudi2加速器可以满足大规模语言模型、生成式AI模>型的强算力需求。Gaudi系列加速器优异的深度学习训练吞吐量、推理速度性能，已经得到了业界领先机构、客户的普遍认可。比如，正是在第一代Gaudi加速器的加持下，亚马逊EC2 DL1实例相比于在AWS云上运行NVIDIA GPU的同类实例，性价比高出多达40％。机器学习与人工智能开放产业联盟MLCommons在六月底公布的AI性能基准测试MLPerf Training 3.0的最新结果，更是进一步凸显了Gaudi2加速器的高性能、高性价比，联合Intel第四代至强可扩展处理器，已经成为唯一能够可靠取代NVIDIA GPU的方案。截止2023年6月，Gaudi2是除了NVIDIA H100 GPU以外，向GPT-3大模型训练基准提交性能结果的解决方案。测试结果显示，面对要求极为苛刻的、1750亿参数的GPT-3模型，384个Gaudi2加速器上的训练时间仅为311.9分钟，而且从256个加速器到384个加速器，性能扩展幅度达95％，非常接近理想的线性提升。Stable Diffusion训练上，Gaudi2加>速器从1张卡到64张卡，扩展性更是达到了惊人的99％。此外，在计算机视觉模型ResNet-50（8个加速器）和Unet3D（8个加速器），以及自然语言处理模型BERT（8个和64个加速器）上，Gaudi2都取得了优异的训练>结果。与去年11月提交的数据相比，BERT和ResNet模型的性能分别提高了10％、4％。值得一提的是，本次MLPerf 3.0的Gaudi2结果以BF16数据类型提交，在四种不同模型上的性能均优于NVIDIA A100，价格更便宜。第三季度还会发布对FP8数据类型的软件支持与新功能，预计届时Gaudi2的性能将有明显飞跃，预计性价比将超越NVIDIA H100。Gaudi2加速器还得到了AI与机器学习开源软件供应商Hugging Face的采纳。其测试结果显示，从预训练BERT （NLP模型）到稳定扩散（流行的多模态模型）推理，再到1760亿参数的大型开源聊天模 BLOOMZ的推理，Gaudi2的表现都领先于NVIDIA A100 GPU。工欲善其事，必先利其器。为了充分发挥Gaudi2加速器的性能潜力，满足日益增长的生成式AI、大语言模型需求，Intel一直在同步打造高效、成熟的软件支持。比如说SynapseAI软件套件，针对Gaudi平台深度学习业务进行了优化，可以大大简化模型的开发与>迁移，能够将当前基于GPU的模型业务和系统，快速迁移到基于全新Gaudi2的服务器。请总结以上内容主题", + "2050": "人类文明的发展，从农业社会、工业社会、信息社会，再到今天的智能社会，技术的每一次进步都带来生产力的极大跃升。当下，数字技术驱动生产力从“量变到质变”，人类正在加速迈向智能世界。智能世界的核心是感知、连接和计算。人类对于未来的追求永无止境，这需要数字技术提升成百上千倍能力。面向未来，只有大胆提出假设和愿景，敢于打破既有理论与技术瓶颈的条条框框，才能大踏步前行。今天，我们对智能世界的所有想象都是保守的。数字化是全世界、全行业的共同机遇。在数字经济新时代，联接从人到物和机器，从家庭走向工厂，从地面走进地下矿道，未来更丰富的行业场景要求联接无处不在、无时不快。算力将成为新生产力，数字化浪潮中，个人、家庭、企业需要算力像水和电一样，触手可及、随取随用。数字化、智能化使能绿色低碳， 创造新的社会价值 使能行业数字化转型，共创商业价值与社会价值。当 前，数字化、智能化成为经济社会发展的关键驱动力， 引领新一轮产业变革。对于华为而言，我们致力于用技 术创新服务于全社会、全人类，在为客户创造商业价值 的同时，努力创造新的社会价值。 过去一年，华为和运营商、合作伙伴一起，在港口、制 造、煤矿、钢铁、化工等 20 多个行业实践数字化转型 升级，累计签署了超过 3,000 个 5G 行业应用商用合同。 在港口，通过 5G+ 高清视频，实现龙门吊的远程控制， 操作人员不再需要爬上爬下，风吹日晒，以及每天 10 到 12 个小时的长时间俯视作业，大幅改善了工作环境 与体验，吸引更多年轻人加入龙门吊司机的行列；在 炼钢车间，5G 远程操控让工人脱离嘈杂、高温的工作 环境，在提升生产效率的同时，让工人享有安全、舒适 的工作环境；在面临易燃、易爆、高温等挑战的化工行 业，通过 5G 机器人巡检、5G 移动作业监控等手段对 厂区的全要素进行视频采集和全天候分析，使得安全预 判效率得到巨大提升。 数字化、智能化使能行业转型升级与经济发展，不仅体 现在生产效率提升与商业价值创造，也凸显出较大社会 价值，让生产更安全，让劳动者更舒适。 绿色 ICT 助力环境保护，促进绿色低碳发展。绿色发 展已成为全球共识。数字化、智能化正前所未有地渗透 到人们生产生活的方方面面，促进更加低碳的生活、工 作与生产方式。未来，数字化和绿色低碳将进一步融 合，带来巨大发展机遇的同时，也将推动社会进步。 华为相信，数字技术是使能绿色发展的基本要素，我 们致力于通过数字技术创新，帮助客户和全社会实现 低碳发展。首先是围绕 ICT 产业自身的节能减排和绿色 发展。一直以来，华为通过节能技术创新，不断提升 ICT 产品的能效。我们与全球运营商一起，打造极简、 绿色、智能的 ICT 基础设施。截至 2021 年底，华为已 支持 100 多个国家的运营商，部署了绿色站点解决方 案，助力全球运营商节省约 842 亿度电，减少约 4,000 万吨二氧化碳排放。同时，华为致力于把 ICT 作为一种 使能技术助力其他行业转型升级，实现绿色发展。在瑞 士，数字技术与传统农业相结合，实施杂草精准清除， 节省约 90% 的除草剂使用，减少了农药对于环境的影 响。在中国，通过智慧供暖实现按需供热，哈尔滨道外 区把平均能耗降低了 10%。 此外，华为数字能源创新融合了数字技术、电力电子技 术、储能技术与热技术，实现比特管理瓦特，助力客户 节能减排。截至 2021 年底，已助力客户累计实现绿色 发电 4,829 亿度，节约用电约 142 亿度，减少二氧化 碳排放近 2.3 亿吨，相当于种植 3.2 亿棵树。 跨越数字鸿沟，推动数字人才培养。数字经济时代，数 字人才是助力数字化转型、推动经济增长的关键。华为 助力各国构建数字人才生态，持续推进 TECH4ALL 数 字包容倡议，实现技术普惠，让人人享受数字生活的便 利，促进数字经济的发展。 从 2008 年开始，华为发起“未来种子”项目，陆续推 出和赞助了各类人才培养、竞赛等项目，包括未来种子 奖学金、ICT 学院、开发者学院、云开发者学堂等，致 力于支持业务所在国培养 ICT 人才，累计投入超过 1.5 亿美元，覆盖 150 多个国家，受益人数超过 154 万人。 我们将持续为数字人才培养做贡献。2021 年，华为发 布了“未来种子 2.0”计划，承诺在未来五年投入 1.5 亿美元用于数字化人才培养，新增受益人数预计将超过 300 万人。我们希望，更多的个人、组织加入华为的数 字包容倡议，共同推进数字人才的发展计划。 2021 年年度报告 09 开放合作，持续创新，坚持全球运 营，与合作伙伴共同成长 科技创新是科技公司对人类社会的重要贡献之一。华为 公司努力探索科学技术的前沿，与世界开放合作，坚持 基础研究、坚持开放创新。当前是华为发展的关键时 期，我们仍将进一步加大在研究与创新领域投资，为各 行各业创造增量价值，让每个人、家庭与组织都从技 术进步中受益。2021 年，华为研发费用支出为人民币 1,427 亿元，约占全年收入的 22.4%。过去十年，我们 在研发领域累计投资超过人民币 8,450 亿元，近几年， 每年在基础研究上的投资超过人民币 200 亿元。 我们建立了 86 个基础技术实验室，扎根基础研究，建 立相关的核心技术体系；我们提出了 AI 时代可信计算 的新架构模型，实现多语言、跨平台运行，实现数据处 理量提升 3 个数量级；我们与合作伙伴一起共同促进欧 拉和鸿蒙开源生态的繁荣，共同打造覆盖未来计算场景 的开源操作系统，促进算力和数据流通，早日实现算力 普惠，让算力像电力一样非常方便地按需而用。 同时，我们持续开放合作，积极融入全球学术组织，加 强与全球高校、科研机构开展合作。通过与科学家碰 撞、分享难题与知识，共同应对世界级的挑战和难题， 促进科技进步。华为将用最大的诚意，最热烈的呼唤拥 抱和欢迎全世界科学家和优秀人才。过去几年，我们在 北京、上海、深圳等地建立了“黄大年茶思屋”和“九 章院”，希望通过提供宽松、宁静的创新环境，让科学 家们心无旁骛、潜心研究，真正面向未来、开创未来， 为人类社会发展做贡献。 华为坚持全球运营，并实施多元化的供应策略，构建长 期、持续、稳定的供应能力，保障供应连续性和面向未 来的可持续发展。我们与全球上万家供应商和合作伙伴 广泛合作，建立了长期合作关系，加强开放合作，在开 放合作中解决发展中的问题和挑战。华为有信心、有能 力继续与全球合作伙伴共同奋斗，秉持“合作共赢、共 同发展”理念，打造安全、可靠、有竞争力的健康产业 链，持续为全球客户提供优质的产品与服务。 完善公司治理，坚持合规运营， 更好地为全球客户服务 良好的公司治理是公司稳健发展的重要基石。华为坚持 以客户为中心、以奋斗者为本，持续优化公司治理架 构、组织、流程和考核机制，使公司长期保持有效增 长。2021 年，持股员工代表会举行了两次会议，审议 了一系列治理文件：通过监事会制度，进一步完善公司 监事会的定位和职权，规范监事会运作；逐步明确集团 董事会与子公司董事会的相互关系。同时，面向一线组 织，采用数字化的方式，构建疫情下的新型业务管理体 系和平台，持续加强数字化建设与运营，快速响应，实 现“多打粮食，增加土地肥力”。 华为坚持以法律遵从的确定性，应对国际政治的不确定 性。坚持诚信经营、恪守商业道德、致力于遵从业务所 在国适用的法律法规是华为管理层一直秉持的核心理 念。公司的基本政策要求全球各子公司、各部门遵从所 在国家和地区适用的法律法规。我们重视并持续营造诚 信文化，要求每一位员工遵守商业行为准则。 同时，作为全球化公司，华为持续积极与外界开放沟 通，进一步开放透明，让所有客户、合作伙伴和利益 相关人更加深入了解华为。我们也欢迎各国政府、媒 体、专家学者等各界人士来访公司，了解和认识真实的 华为。 没有退路就是胜利之路。华为的选择注定是一条布满荆 棘，但同时也是值得为之奋斗的漫漫长路。未来，尽管 面临更多不确定性，华为将坚持开放合作，保持战略定 力，保证公司的生存与发展。无论前路多么坎坷，我们 将始终坚守愿景与使命：把数字世界带入每个人、每个 家庭、每个组织，构建万物互联的智能世界。请总结以上内容主题", + "3294": "智能手表（Smart Watch）是安装有嵌入式系统、用于增强基于报时等功能的腕部手表，其功能相似于一台个人数码助理。智能手表除指示时间之外，还应具有提醒、导航、校准、监测、交互等其中一种或者多种功能；显示方式包括指针、数字、图像等。《2023-2027年智能手表市场调查研究报告》研究显示：随着科技的进步和发展，智能穿戴设备市场从2012年至今得到了快速增长，在这10年间，有越来越多的玩家加入，智能穿戴设备产品不管是在技术上、还是使用体验上，都有了很大提升和改善。智能手表是智能穿戴的主要代表之一，在健康监测、记步、拨打电话、定位、与智能家居联动等功能的加持下，广受欢迎。智能手表“智能”的地方体现在，它拥有一套独立的嵌入式操作系统，有一个数据处理中心，需要调用各类传感器收集到的信息，还要有屏幕、存储器、电池、电源管理系统、无线射频系统等，在内部芯片用料和结构设计上与智能手机较为相似。2017年我国智能手表市场规模为131.23亿元，2021年为317.58亿元，同比2020年增长16.60%。脱硝催化剂泛指选择性催化还原（SCR）脱硝催化剂，也叫SCR脱硝催化剂。SCR脱硝催化剂是指应用于选择性催化还原（SCR）脱硝系统上的催化剂，是在选择性催化还原（SCR）脱硝技术基础上发展起来的。《脱硝催化剂行业数据深度调研分析与发展战略规划报告》研究显示:早在2011年，环保部颁布了《火电厂大气污染物排放标准》，开启中国火电环保风暴，各火电厂纷纷加快升级改造步伐。将火电厂脱硝催化剂市场拆分为二部分：新建机组的初装需求以及已投运机组的补装（更换）需求。2021年新建机组配套催化剂需求为3.89万m3，已投运的脱硝机组催化剂更换需求为24.05万m3，合计27.94万m3。酸枣（学名：Ziziphus jujuba Mill. var. spinosa（Bunge）Hu ex H. F. Chow）鼠李科枣属植物，是枣的变种。又名棘、棘子、野枣、山枣、葛针等，原产中国华北，中南各省亦有分布。多野生，常为灌木，也有的为小乔木。树势较强。枝、叶、花的形态与普通枣相似，但枝条节间较短，托刺发达，除生长枝各节均具托刺外，结果枝托叶也成尖细的托刺。叶小而密生，果小、多圆或椭圆形、果皮厚、光滑、紫红或紫褐色，肉薄，味大多很酸，核圆或椭圆形，核面较光滑，内含种子1至2枚，种仁饱满可作中药。其适应性较普通枣强，花期很长，可为蜜源植物。果皮红色或紫红色，果肉较薄、疏松，味酸甜。《酸枣行业现状分析及发展战略研究报告》研究显示：随着生活节奏的不断加快，出现睡眠障碍的人群不断增多，身心健康状况也逐渐下降。酸枣常被用来治疗心烦失眠、多梦、盗汗、易惊等病。通过深入挖掘酸枣助眠机理，开发相应产品，可改善人们的睡眠质量及健康状况，为健康中国助力。2017年，中国酸枣行业市场规模为17.67亿元，2021年为33.24亿元，同比增长4.25%。丙烯酸树脂（acrylic resin）是丙烯酸、甲基丙烯酸及其衍生物聚合物的总称。丙烯酸树脂涂料就是以（甲基）丙烯酸酯、苯乙烯为主体，同其他丙烯酸酯共聚所得丙烯酸树脂制得的热塑性或热固性树脂涂料或丙烯酸辐射涂料。含羟基丙烯酸树脂是指加入含羟基的丙烯酸酯和甲基丙烯酸酯等单体，使之与其他（甲基）丙烯酸类单体一起共聚而制成的带羟基官能团的丙烯酸树脂。一般情况下，羟基丙烯酸树脂和热固性树脂含有-OH基团，部分热塑性树脂也含有少量-OH。羟基是丙烯酸树脂侧链上作为后期交联的活性官能团，作用是增大丙烯酸树脂的极性，改良漆膜的附着力和润湿分散性。因此，丙烯酸树脂中羟基含量越大，使得漆膜交联密度变大，制备出的涂料漆膜物理性、化学性、机械性能和漆膜外观越好。同时，羟基含量越大时，可加快-OH基团与-NCO基团的活性反应，提高涂料的固化交联速度和干燥时间。《含羟基丙烯酸树脂行业数据深度调研分析与发展战略规划报告》研究显示：随着世界各国对环保问题的逐渐重视，涂料行业正在逐渐由传统的有机溶剂型涂料向环保型涂料转型，为响应新型涂料发展的需要，制备“高性能、低污染、长寿命的环保型涂料产品具有极高的社会意义。而限制环保型涂料发展的最重要原因是环保型树脂的升级，水性树脂作为环保型树脂中最重要的品类之一，是“油转水”转型中最重要的一环；而含羟基丙烯酸树脂具有“高硬度、高光泽、高保护”的使用性能，在工业领域中也得到广泛应用。2017年我国含羟基丙烯酸树脂行业产能为62.45万吨，2021年产能为85.82万吨，同比2020年增长7.25%。城市轨道交通是城市公共交通的骨干，具有节能、省地、运量大、全天候、无污染（或少污染）又安全等特点，属绿色环保交通体系，特别适应于大中城市。进入新世纪，尤其是2008年以后，我国通过扩大内需，促进经济平稳增长的一揽子计划，带动了国内基础建设的发展，同时我国大型城市逐渐面临交通拥堵的问题，加快了城市轨道交通建设。《城市轨道交通行业数据深度调研分析与发展战略规划报告》研究显示：根据城市轨道交通协会发布的数据显示，我国城市轨道交通运营里程稳步攀升，并且我国已经成为了全球城市轨道运营里程数第一，远超德国、俄罗斯、美国等发达国家。2021年中国开通轨道交通的城市数量从2015年的26个增长到51个，在6年期间开通轨道交通的城市数量几乎翻备。2021年，中国城市轨道交通运营里程8708公里，较2015年的3618公里，是2015年3618公里的2.4倍。别墅电梯即安装在私人住宅中，仅供单一家庭成员使用的电梯。它也可以安装在非单一家庭使用的建筑物内，作为单一家庭进入其住所的工具，但是建筑物内的公众或其他居住者无法进入和使用。《别墅电梯行业现状分析及发展战略研究报告》研究显示：作为基础设施配套工程的重要组成部分，电梯与国家经济建设尤其是房地产的发展以及人民生活质量的提高密切相关。近些年，随着全球人口增长、城市化进程加快以及人们对便捷生活要求的提高，别墅电梯得到越来越广泛的使用。目前，国际别墅电梯市场呈现发达国家和地区需求稳步增长、新兴市场需求快速增长的特征。2017年全球别墅电梯行业产能规模为4.79万台，2021年为7.89万台，同比2020年增长12.55%。物质、能量和信息是构成自然界的基本要素。“能源”这一术语，过去人们谈论得很少，正是两次石油危机使它成了人们议论的热点。关于能源的定义，约有20种。例如：说：“能源是可从其获得热、光和动力之类能量的资源”；《大英百科全书》说：“能源是一个包括着所有燃料、流水、阳光和风的术语，人类用适当的转换手段便可让它为自己提供所需的能量”；《日本大百科全书》说：“在各种生产活动中，我们利用热能、机械能、光能、电能等来作功，可利用来作为这些能量源泉的自然界中的各种载体，称为能源”；我国的《能源百科全书》说：“能源是可以直接或经转换提供人类所需的光、热、动力等任一形式能量的载能体资源。”可见，能源是一种呈多种形式的，且可以相互转换的能量的源泉。确切而简单地说，能源是自然界中能为人类提供某种形式能量的物质资源。能源亦称能量资源或能源资源。是指可产生各种能量（如热量、电能、光能和机械能等）或可作功的物质的统称。是指能够直接取得或者通过加工、转换而取得有用能的各种资源，包括煤炭、原油、天然气、煤层气、水能、核能、风能、太阳能、地热能、生物质能等一次能源和电力、热力、成品油等二次能源，以及其他新能源和可再生能源。能源（Energy Source）亦称能量资源或能源资源，是国民经济的重要物质基础，未来国家命运取决于能源的掌控。能源的开发和有效利用程度以及人均消费量是生产技术和生活水平的重要标志。（中国大百科全书·机械工程卷）在《中华人民共和国节约能源法》中所称能源，是指煤炭、石油、天然气、生物质能和电力、热力以及其他直接或者通过加工、转换而取得有用能的各种资源。《能源领域信息化与IT应用行业数据深度调研分析与发展战略规划报告》研究显示：“互联网+智慧能源”的理念推动着能源行业的改革，传统企业需要思考如何将重资产的使用效率提升，进一步拥抱数字化，而运维服务也是企业对外输出的核心竞争力之一。在数字化时代，企业数据上云成为趋势，云计算和云存储成为重要的数字化能力。然而对于关系国计民生的能源行业来说，大力推进绿色低碳科技创新将成为碳中和发展的主要路径，推动绿色低碳技术重大突破，加快能源行业的数字化转型，变得尤为重要。能源云是紧密结合能源行业业务，用云计算、大数据、移动、物联网、人工智能等新一代的技术架构和体系为能源企业提供的云服务平台，它涵盖行业云服务、领域云服务、云工具和云平台，覆盖能源企业运营中的采购、生产、营销、数据、财务、人力、社交与协同办公等多个业务场景和领域，它可以与传统软件融合，让企业的业务自然延伸到云端应用。采用以公有云、专属云、混合云等云服务模式为主，行业解决方案为辅的多种服务形式，为能源企业互联网化提供坚实的支撑和保障。公有云通常指第三方提供商为用户提供的能够使用的云，公有云一般可通过Internet使用，可能是免费或成本低廉的，公有云的核心属性是共享资源服务。花青素（anthocyan），又称花色素，是自然界一类广泛存在于植物中的水溶性天然色素，是花色苷（anthocyains）水解而得的有颜色的苷元。水果、蔬菜、花卉中的主要呈色物质大部分与之有关。在植物细胞液泡不同的PH值条件下，花青素使花瓣呈现五彩缤纷的颜色。已知花青素有20多种，食物中重要的有6种，即天竺葵色素、矢车菊色素、飞燕草色素、芍药色素、牵牛花色素和锦葵色素。自然状态的花青素都以糖苷形式存在，称为花色苷，很少有游离的花青素存在。《花青素市场现状分析及行业前景预测报告》研究显示：我国花青素开发较晚，且主要以粗制品为主，2011年之前，我国对花青素的提取基本上是空白。2011年臻多美研发成功的花青素提取和保存两项专利技术，填补了国内花青素技术的空白，打破了之前国内花青素精制品90%靠进口的局面，并大量销往国外。随着人们健康意识的加强，以及我国与国际规范的接轨，对花青素的需求逐渐凸显。2017年我国花青素市场规模为3.44亿元，2021年为6.93亿元，同比2020年增长17.20%。发动机（Engine）是一种能够把其它形式的能转化为机械能的机器，包括如内燃机（往复活塞式发动机）、外燃机（斯特林发动机、蒸汽机等）、喷气发动机、电动机等。如内燃机通常是把化学能转化为机械能。发动机既适用于动力发生装置，也可指包括动力装置的整个机器（如：汽油发动机、航空发动机）。从字面上理解，道路发动机就是用于驱动在公共道路上行驶的车辆的发动机，而非道路发动机就是所有这个用途以外的发动机，比如船用发动机、工程机械发动机、发电用发动机、航空发动机等等。《非道路发动机行业数据深度调研分析与发展战略规划报告》研究显示：近年来，全球非道路发动机市场呈现稳定态势，到2019年市场产能增长至4726.31万台；2020年，新冠疫情爆发，非道路发动机产能下降至4526.39万台；2021年产能增速恢复至10.78%，总体产能达5014.27万台。发动机是一种能够把其它形式的能转化为机械能的机器，发动机既适用于动力发生装置，也可指包括动力装置的整个机器，农用发动机就是农业机械用发动机。农业机械是指在作物种植业和畜牧业生产过程中，以及农、畜产品初加工和处理过程中所使用的各种机械。《农用发动机行业现状分析及发展战略研究报告》研究显示：随着农业供给侧结构性改革的推进，农用发动机行业进入了深度调整期，技术升级，结构优化也在同步进行中。农业是支撑国民经济建设与发展的基础产业，而农业机械化是建设现代农业的重要物质基础，也是实现农业现代化的重要标志与内容。近年来，在国家政策补贴支持及机械化率稳步增长等因素的推动下，农用发动机行业规模不断增长。市场规模方面，据统计，近年来我国农用发动机市场规模不断增长，截至2021年我国农用发动机市场规模为429.61亿元，同比增长27.60%。为适应文明演进的新趋势和新要求，人类必须从根本上解决文明前行的动力困扰，实现能源的安全、稳定、清洁和永续利用。智慧能源就是充分开发人类的智力和能力，通过不断技术创新和制度变革，在能源开发利用、生产消费的全过程和各环节融汇人类独有的智慧，建立和完善符合生态文明和可持续发展要求的能源技术和能源制度体系，从而呈现出的一种全新能源形式。简而言之，智慧能源就是指拥有自组织、自检查、自平衡、自优化等人类大脑功能，满足系统、安全、清洁和经济要求的能源形式。《智慧能源行业现状分析及发展战略研究报告》研究显示：电力行业是智慧能源行业最大的市场，随着我国经济的快速发展，智慧能源行业作为电力行业产业链中的重要一环也随之蓬勃发展，尤其近几年我国经济发展中面临能源、电力紧张的瓶颈性问题，国家不断加大对智慧能源行业的投资，使得该行业步入了飞跃发展期，2021年中国智能电网市场规模已经达到854.6亿元。请从上述信息中总结出最佳的投资方向并给出理由：" +} diff --git a/benchmark/chatglm2-6b/run-chatglm2.sh b/benchmark/chatglm2-6b/run-chatglm2.sh new file mode 100755 index 0000000..9743bb3 --- /dev/null +++ b/benchmark/chatglm2-6b/run-chatglm2.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# On different systems, different models and differnt modes, OMP_NUM_THREADS need to be ajusted. +output=$(numactl -H) +first_line=$(echo "$output" | head -n 1) +nodes=$(echo "$first_line" | awk '{print $2}') + +echo "memory node number: $nodes" +if [ "$nodes" -eq 16 ]; then +#HBM SNC-4 mode, Confirm that there are 8 HBM memory nodes and 8 DRAM memory nodes through "numactl -H" +#0-7 is DRAM memory node, 8-15 is HBM node + echo "HBM SNC4 mode" +# Run chatglm2 on 1 socket HBM SNC4 mode + OMP_NUM_THREADS=12 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -m 8 -N 0 sh chatglm2-6b.sh : \ + -n 1 numactl -m 9 -N 1 sh chatglm2-6b.sh : \ + -n 1 numactl -m 10 -N 2 sh chatglm2-6b.sh : \ + -n 1 numactl -m 11 -N 3 sh chatglm2-6b.sh +# -n 1 numactl -m 12 -N 4 sh chatglm2-6b.sh : \ +# -n 1 numactl -m 13 -N 5 sh chatglm2-6b.sh : \ +# -n 1 numactl -m 14 -N 6 sh chatglm2-6b.sh : \ +# -n 1 numactl -m 15 -N 7 sh chatglm2-6b.sh + +elif [ "$nodes" -eq 4 ]; then +#HBM Quad-mode, Confirm that there are 2 HBM memory nodes and 2 DRAM memory nodes through "nuamctl -H" + echo "HBM Quad mode" +# Run chatglm2 on 1 socket HBM Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 2 sh chatglm2-6b.sh : \ + -n 1 numactl -N 1 -m 3 sh chatglm2-6b.sh + +elif [ "$nodes" -eq 2 ]; then +#SPR Quad-mode, Confirm that there are 2 DRAM memory nodes through "nuamctl -H" + echo "SPR Quad mode" +# Run chatglm2 on 1 socket SPR Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 sh chatglm2-6b.sh + +else + echo "Please double check the memory nodes" +fi + + diff --git a/benchmark/llama-13b/llama-13b.sh b/benchmark/llama-13b/llama-13b.sh new file mode 100755 index 0000000..eb6ebe2 --- /dev/null +++ b/benchmark/llama-13b/llama-13b.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "FP16 Performance " +python "${SCRIPT_DIR}"/../benchmark.py \ + --token_path /data/llama-13b \ + --model_path /data/llama-13b/cpu \ + --prompt_path "${SCRIPT_DIR}"/prompt_pool.json \ + --model_name "Llama-13B" \ + --dtype fp16 \ + --token_in 32 \ + --token_out 32 --beam_width 1 --iteration 100 + +# In this benchmark case, token_in only can be "demo","32","64","128","256","512","1024","2016" +# "32" means the token length is 32, if needs more test, add it into input_token.py + diff --git a/benchmark/llama-13b/prompt_pool.json b/benchmark/llama-13b/prompt_pool.json new file mode 100644 index 0000000..8f4af58 --- /dev/null +++ b/benchmark/llama-13b/prompt_pool.json @@ -0,0 +1,10 @@ +{ + "demo": "Once upon a time, there was a little girl who loved adventures very much.", + "32": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small", + "64": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small village surrounded by a lush green forest. Although the village was peaceful and quiet, Alice always felt a strong longing for excitement and exploration. Her", + "128": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small village surrounded by a lush green forest. Although the village was peaceful and quiet, Alice always felt a strong longing for excitement and exploration. Her curious mind often wandered beyond the familiar landscapes of her home. Every day, Alice would sneak off to the forest, where she would create imaginary worlds filled with magical creatures and thrilling quests. In these enchanting realms, she could be anything", + "256": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small village surrounded by a lush green forest. Although the village was peaceful and quiet, Alice always felt a strong longing for excitement and exploration. Her curious mind often wandered beyond the familiar landscapes of her home. Every day, Alice would sneak off to the forest, where she would create imaginary worlds filled with magical creatures and thrilling quests. In these enchanting realms, she could be anything she desired—a fearless explorer, a brave knight, or even a daring pirate sailing the seas. One fine morning, as the sun painted the sky with hues of gold, Alice stumbled upon an ancient-looking book hidden among the foliage. The book seemed to call out to her, as if it held the key to unlocking the very mysteries she yearned to unravel. With trembling hands, she opened its weathered pages and found herself entranced by the tales of heroes and heroines who embarked on extraordinary journeys.", + "512": "It is done, and submitted. You can play 'Survival of the Tastiest' on Android, and on the web. Playing on the web works, but you have to simulate multiple touch for table moving and that can be a bit confusing. There is a lot I'd like to talk about. I will go through every topic, insted of making the typical what went right/wrong list. Concept Working over the theme was probably one of the hardest tasks which I had to face. Originally, I had an idea of what kind of game I wanted to develop, gameplay wise - something with a lot of enemies/actors, simple graphics, maybe set in space, controlled from a top-down view. I was confident that I could fit any theme around it. In the end, the problem with a theme like 'Evolution' in a game is that evolution is unassisted. It happens through several seemingly random mutations over time, with the most apt permutation surviving. This genetic car simulator is, in my opinion, a great example of actual evolution of a species facing a challenge. But is it a game? In a game, you need to control something to reach an objective. That control goes against what evolution is supposed to be like. If you allow the user to pick how to evolve something, it's not evolution anymore - it's the equivalent of intelligent design, the fable invented by creationists to combat the idea of evolution. Being agnostic and a Pastafarian, that's not something that rubbed me the right way. Hence, my biggest dillema when deciding what to create was not with what I wanted to create, but with what I did not. I didn't want to create an 'intelligent design' simulator and wrongly call it evolution. This is a problem, of course, every other contestant also had to face. And judging by the entries submitted, not many managed to work around it. I'd say the only real solution was through the use of artificial selection, somehow. So far, I haven't seen any entry using this at its core gameplay. Alas, this is just a fun competition and after a while I decided not to be as strict with the game idea, and allowed myself to pick whatever I thought would work out. My initial idea was to create", + "1024": "It is done, and submitted. You can play 'Survival of the Tastiest' on Android, and on the web. Playing on the web works, but you have to simulate multiple touch for table moving and that can be a bit confusing. There is a lot I'd like to talk about. I will go through every topic, insted of making the typical what went right/wrong list. Concept Working over the theme was probably one of the hardest tasks which I had to face. Originally, I had an idea of what kind of game I wanted to develop, gameplay wise - something with a lot of enemies/actors, simple graphics, maybe set in space, controlled from a top-down view. I was confident that I could fit any theme around it. In the end, the problem with a theme like 'Evolution' in a game is that evolution is unassisted. It happens through several seemingly random mutations over time, with the most apt permutation surviving. This genetic car simulator is, in my opinion, a great example of actual evolution of a species facing a challenge. But is it a game? In a game, you need to control something to reach an objective. That control goes against what evolution is supposed to be like. If you allow the user to pick how to evolve something, it's not evolution anymore - it's the equivalent of intelligent design, the fable invented by creationists to combat the idea of evolution. Being agnostic and a Pastafarian, that's not something that rubbed me the right way. Hence, my biggest dillema when deciding what to create was not with what I wanted to create, but with what I did not. I didn't want to create an 'intelligent design' simulator and wrongly call it evolution. This is a problem, of course, every other contestant also had to face. And judging by the entries submitted, not many managed to work around it. I'd say the only real solution was through the use of artificial selection, somehow. So far, I haven't seen any entry using this at its core gameplay. Alas, this is just a fun competition and after a while I decided not to be as strict with the game idea, and allowed myself to pick whatever I thought would work out. My initial idea was to create something where humanity tried to evolve to a next level, but had some kind of foe trying to stop them from doing so. I kind of had this image of human souls flying in space towards a monolith or a space baby (all based in 2001: A Space Odyssey of course) but I couldn't think of compelling (read: serious) mechanics for that. Borgs were my next inspiration, as their whole hypothesis fit pretty well into the evolution theme. But how to make it work? Are you the borg, or fighting the Borg? The third and final idea came to me through my girlfriend, who somehow gave me the idea of making something about the evolution of Pasta. The more I thought about it the more it sounded like it would work, so I decided to go with it. Conversations with my inspiring co-worker Roushey (who also created the 'Mechanical Underdogs' signature logo for my intros) further matured the concept, as it involved into the idea of having individual pieces of pasta flying around and trying to evolve until they became all-powerful. A secondary idea here was that the game would work to explain how the Flying Spaghetti Monster came to exist - by evolving from a normal dinner table. So the idea evolved more or less into this: you are sitting a table. You have your own plate, with is your 'base'. There are 5 other guests at the table, each with their own plate. Your plate can spawn little pieces of pasta. You do so by 'ordering' them through a menu. Some pastas are better than others; some are faster, some are stronger. They have varying 'costs', which are debited from your credits (you start with a number of credits). Once spawned, your pastas start flying around. Their instinct is to fly to other plates, in order to conquer them (the objective of the game is having your pasta conquer all the plates on the table). But they are really autonomous, so after being spawned, you have no control over your pasta (think DotA or LoL creeps). Your pasta doesn't like other people's pasta, so if they meet, they shoot sauce at each other until one dies. You get credits for other pastas your own pasta kill. ", + "2016": "It is done, and submitted. You can play 'Survival of the Tastiest' on Android, and on the web. Playing on the web works, but you have to simulate multiple touch for table moving and that can be a bit confusing. There is a lot I'd like to talk about. I will go through every topic, insted of making the typical what went right/wrong list. Concept Working over the theme was probably one of the hardest tasks which I had to face. Originally, I had an idea of what kind of game I wanted to develop, gameplay wise - something with a lot of enemies/actors, simple graphics, maybe set in space, controlled from a top-down view. I was confident that I could fit any theme around it. In the end, the problem with a theme like 'Evolution' in a game is that evolution is unassisted. It happens through several seemingly random mutations over time, with the most apt permutation surviving. This genetic car simulator is, in my opinion, a great example of actual evolution of a species facing a challenge. But is it a game? In a game, you need to control something to reach an objective. That control goes against what evolution is supposed to be like. If you allow the user to pick how to evolve something, it's not evolution anymore - it's the equivalent of intelligent design, the fable invented by creationists to combat the idea of evolution. Being agnostic and a Pastafarian, that's not something that rubbed me the right way. Hence, my biggest dillema when deciding what to create was not with what I wanted to create, but with what I did not. I didn't want to create an 'intelligent design' simulator and wrongly call it evolution. This is a problem, of course, every other contestant also had to face. And judging by the entries submitted, not many managed to work around it. I'd say the only real solution was through the use of artificial selection, somehow. So far, I haven't seen any entry using this at its core gameplay. Alas, this is just a fun competition and after a while I decided not to be as strict with the game idea, and allowed myself to pick whatever I thought would work out. My initial idea was to create something where humanity tried to evolve to a next level, but had some kind of foe trying to stop them from doing so. I kind of had this image of human souls flying in space towards a monolith or a space baby (all based in 2001: A Space Odyssey of course) but I couldn't think of compelling (read: serious) mechanics for that. Borgs were my next inspiration, as their whole hypothesis fit pretty well into the evolution theme. But how to make it work? Are you the borg, or fighting the Borg? The third and final idea came to me through my girlfriend, who somehow gave me the idea of making something about the evolution of Pasta. The more I thought about it the more it sounded like it would work, so I decided to go with it. Conversations with my inspiring co-worker Roushey (who also created the 'Mechanical Underdogs' signature logo for my intros) further matured the concept, as it involved into the idea of having individual pieces of pasta flying around and trying to evolve until they became all-powerful. A secondary idea here was that the game would work to explain how the Flying Spaghetti Monster came to exist - by evolving from a normal dinner table. So the idea evolved more or less into this: you are sitting a table. You have your own plate, with is your 'base'. There are 5 other guests at the table, each with their own plate. Your plate can spawn little pieces of pasta. You do so by 'ordering' them through a menu. Some pastas are better than others; some are faster, some are stronger. They have varying 'costs', which are debited from your credits (you start with a number of credits). Once spawned, your pastas start flying around. Their instinct is to fly to other plates, in order to conquer them (the objective of the game is having your pasta conquer all the plates on the table). But they are really autonomous, so after being spawned, you have no control over your pasta (think DotA or LoL creeps). Your pasta doesn't like other people's pasta, so if they meet, they shoot sauce at each other until one dies. You get credits for other pastas your own pasta kill. Once a pasta is in the vicinity of a plate, it starts conquering it for its team. It takes around 10 seconds for a plate to be conquered; less if more pasta from the same team are around. If pasta from other team are around, though, they get locked down in their attempt, unable to conquer the plate, until one of them die (think Battlefield's standard 'Conquest' mode). You get points every second for every plate you own. Over time, the concept also evolved to use an Italian bistro as its main scenario. Carlos, Carlos' Bistro's founder and owner Setup No major changes were made from my work setup. I used FDT and Starling creating an Adobe AIR (ActionScript) project, all tools or frameworks I already had some knowledge with. One big change for me was that I livestreamed my work through a twitch.tv account. This was a new thing for me. As recommended by Roushey, I used a program called XSplit and I got to say, it is pretty amazing. It made the livestream pretty effortless and the features are awesome, even for the free version. It was great to have some of my friends watch me, and then interact with them and random people through chat. It was also good knowing that I was also recording a local version of the files, so I could make a timelapse video later. Knowing the video was being recorded also made me a lot more self-conscious about my computer use, as if someone was watching over my shoulder. It made me realize that sometimes I spend too much time in seemingly inane tasks (I ended up wasting the longest time just to get some text alignment the way I wanted - it'll probably drive someone crazy if they watch it) and that I do way too many typos where writing code. I pretty much spend half of the time writing a line and the other half fixing the crazy characters in it. My own stream was probably boring to watch since I was coding for the most time. But livestreaming is one of the cool things to do as a spectator too. It was great seeing other people working - I had a few tabs opened on my second monitor all the time. It's actually a bit sad, because if I could, I could have spent the whole weekend just watching other people working! But I had to do my own work, so I'd only do it once in a while, when resting for a bit. Design Although I wanted some simple, low-fi, high-contrast kind of design, I ended up going with somewhat realistic (vector) art. I think it worked very well, fitting the mood of the game, but I also went overboard. For example: to know the state of a plate (who owns it, who's conquering it and how much time they have left before conquering it, which pasta units are in the queue, etc), you have to look at the plate's bill. The problem I realized when doing some tests is that people never look at the bill! They think it's some kind of prop, so they never actually read its details. Plus, if you're zoomed out too much, you can't actually read it, so it's hard to know what's going on with the game until you zoom in to the area of a specific plate. One other solution that didn't turn out to be as perfect as I thought was how to indicate who a plate base belongs to. In the game, that's indicated by the plate's decoration - its color denotes the team owner. But it's something that fits so well into the design that people never realized it, until they were told about it. In the end, the idea of going with a full physical metaphor is one that should be done with care. Things that are very important risk becoming background noise, unless the player knows its importance. Originally, I wanted to avoid any kind of heads-up display in my game. In the end, I ended up adding it at the bottom to indicate your credits and bases owned, as well as the hideous out-of-place-and-still-not-obvious 'Call Waiter' button. But in hindsight, I should have gone with a simple HUD from the start, especially one that indicated each team's colors and general state of the game without the need for zooming in and out. Development Development went fast. But not fast enough. Even though I worked around 32+ hours " +} \ No newline at end of file diff --git a/benchmark/llama-13b/run-llama-13b.sh b/benchmark/llama-13b/run-llama-13b.sh new file mode 100755 index 0000000..adb9afe --- /dev/null +++ b/benchmark/llama-13b/run-llama-13b.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# On different systems, different models and differnt modes, OMP_NUM_THREADS need to be ajusted. +output=$(numactl -H) +first_line=$(echo "$output" | head -n 1) +nodes=$(echo "$first_line" | awk '{print $2}') + +echo "memory node number: $nodes" +if [ "$nodes" -eq 16 ]; then +#HBM SNC-4 mode, Confirm that there are 8 HBM memory nodes and 8 DRAM memory nodes through "numactl -H" +#0-7 is DRAM memory node, 8-15 is HBM node + echo "HBM SNC4 mode" +# Run Llama-13B on 1 socket HBM SNC4 mode + OMP_NUM_THREADS=12 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -m 8 -N 0 sh llama-13b.sh : \ + -n 1 numactl -m 9 -N 1 sh llama-13b.sh : \ + -n 1 numactl -m 10 -N 2 sh llama-13b.sh : \ + -n 1 numactl -m 11 -N 3 sh llama-13b.sh : \ + -n 1 numactl -m 12 -N 4 sh llama-13b.sh : \ + -n 1 numactl -m 13 -N 5 sh llama-13b.sh : \ + -n 1 numactl -m 14 -N 6 sh llama-13b.sh : \ + -n 1 numactl -m 15 -N 7 sh llama-13b.sh + +elif [ "$nodes" -eq 4 ]; then +#HBM Quad-mode, Confirm that there are 2 HBM memory nodes and 2 DRAM memory nodes through "nuamctl -H" + echo "HBM Quad mode" +# Run Llama-13B on 1 socket HBM Quad mode,Flat mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 2 sh llama-13b.sh : \ + -n 1 numactl -N 1 -m 3 sh llama-13b.sh + +elif [ "$nodes" -eq 2 ]; then +#SPR Quad-mode, Confirm that there are 2 DRAM memory nodes through "nuamctl -H" + echo "SPR Quad mode" +# Run Llama-13B on 1 socket SPR Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 sh llama-13b.sh : \ + -n 1 numactl -N 1 -m 1 sh llama-13b.sh + +else + echo "Please double check the memory nodes" +fi + + diff --git a/benchmark/llama-7b/llama-7b.sh b/benchmark/llama-7b/llama-7b.sh new file mode 100755 index 0000000..bd4ba1b --- /dev/null +++ b/benchmark/llama-7b/llama-7b.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "FP16 Performance " +python "${SCRIPT_DIR}"/../benchmark.py \ + --token_path /data/llama-7b \ + --model_path /data/llama-7b/cpu \ + --prompt_path "${SCRIPT_DIR}"/prompt_pool.json \ + --model_name "Llama-7B" \ + --dtype fp16 \ + --token_in 32 \ + --token_out 32 --beam_width 1 --batch_size 2 --iteration 100 --padding=False + +# In this benchmark case, token_in only can be "demo","32","64","128","256","512","1024","2016" +# "32" means the token length is 32, if needs more test, add it into input_token.py + diff --git a/benchmark/llama-7b/prompt_pool.json b/benchmark/llama-7b/prompt_pool.json new file mode 100644 index 0000000..8f4af58 --- /dev/null +++ b/benchmark/llama-7b/prompt_pool.json @@ -0,0 +1,10 @@ +{ + "demo": "Once upon a time, there was a little girl who loved adventures very much.", + "32": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small", + "64": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small village surrounded by a lush green forest. Although the village was peaceful and quiet, Alice always felt a strong longing for excitement and exploration. Her", + "128": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small village surrounded by a lush green forest. Although the village was peaceful and quiet, Alice always felt a strong longing for excitement and exploration. Her curious mind often wandered beyond the familiar landscapes of her home. Every day, Alice would sneak off to the forest, where she would create imaginary worlds filled with magical creatures and thrilling quests. In these enchanting realms, she could be anything", + "256": "Once upon a time, there was a little girl who loved adventures very much. This little girl's name was Alice. She lived in a small village surrounded by a lush green forest. Although the village was peaceful and quiet, Alice always felt a strong longing for excitement and exploration. Her curious mind often wandered beyond the familiar landscapes of her home. Every day, Alice would sneak off to the forest, where she would create imaginary worlds filled with magical creatures and thrilling quests. In these enchanting realms, she could be anything she desired—a fearless explorer, a brave knight, or even a daring pirate sailing the seas. One fine morning, as the sun painted the sky with hues of gold, Alice stumbled upon an ancient-looking book hidden among the foliage. The book seemed to call out to her, as if it held the key to unlocking the very mysteries she yearned to unravel. With trembling hands, she opened its weathered pages and found herself entranced by the tales of heroes and heroines who embarked on extraordinary journeys.", + "512": "It is done, and submitted. You can play 'Survival of the Tastiest' on Android, and on the web. Playing on the web works, but you have to simulate multiple touch for table moving and that can be a bit confusing. There is a lot I'd like to talk about. I will go through every topic, insted of making the typical what went right/wrong list. Concept Working over the theme was probably one of the hardest tasks which I had to face. Originally, I had an idea of what kind of game I wanted to develop, gameplay wise - something with a lot of enemies/actors, simple graphics, maybe set in space, controlled from a top-down view. I was confident that I could fit any theme around it. In the end, the problem with a theme like 'Evolution' in a game is that evolution is unassisted. It happens through several seemingly random mutations over time, with the most apt permutation surviving. This genetic car simulator is, in my opinion, a great example of actual evolution of a species facing a challenge. But is it a game? In a game, you need to control something to reach an objective. That control goes against what evolution is supposed to be like. If you allow the user to pick how to evolve something, it's not evolution anymore - it's the equivalent of intelligent design, the fable invented by creationists to combat the idea of evolution. Being agnostic and a Pastafarian, that's not something that rubbed me the right way. Hence, my biggest dillema when deciding what to create was not with what I wanted to create, but with what I did not. I didn't want to create an 'intelligent design' simulator and wrongly call it evolution. This is a problem, of course, every other contestant also had to face. And judging by the entries submitted, not many managed to work around it. I'd say the only real solution was through the use of artificial selection, somehow. So far, I haven't seen any entry using this at its core gameplay. Alas, this is just a fun competition and after a while I decided not to be as strict with the game idea, and allowed myself to pick whatever I thought would work out. My initial idea was to create", + "1024": "It is done, and submitted. You can play 'Survival of the Tastiest' on Android, and on the web. Playing on the web works, but you have to simulate multiple touch for table moving and that can be a bit confusing. There is a lot I'd like to talk about. I will go through every topic, insted of making the typical what went right/wrong list. Concept Working over the theme was probably one of the hardest tasks which I had to face. Originally, I had an idea of what kind of game I wanted to develop, gameplay wise - something with a lot of enemies/actors, simple graphics, maybe set in space, controlled from a top-down view. I was confident that I could fit any theme around it. In the end, the problem with a theme like 'Evolution' in a game is that evolution is unassisted. It happens through several seemingly random mutations over time, with the most apt permutation surviving. This genetic car simulator is, in my opinion, a great example of actual evolution of a species facing a challenge. But is it a game? In a game, you need to control something to reach an objective. That control goes against what evolution is supposed to be like. If you allow the user to pick how to evolve something, it's not evolution anymore - it's the equivalent of intelligent design, the fable invented by creationists to combat the idea of evolution. Being agnostic and a Pastafarian, that's not something that rubbed me the right way. Hence, my biggest dillema when deciding what to create was not with what I wanted to create, but with what I did not. I didn't want to create an 'intelligent design' simulator and wrongly call it evolution. This is a problem, of course, every other contestant also had to face. And judging by the entries submitted, not many managed to work around it. I'd say the only real solution was through the use of artificial selection, somehow. So far, I haven't seen any entry using this at its core gameplay. Alas, this is just a fun competition and after a while I decided not to be as strict with the game idea, and allowed myself to pick whatever I thought would work out. My initial idea was to create something where humanity tried to evolve to a next level, but had some kind of foe trying to stop them from doing so. I kind of had this image of human souls flying in space towards a monolith or a space baby (all based in 2001: A Space Odyssey of course) but I couldn't think of compelling (read: serious) mechanics for that. Borgs were my next inspiration, as their whole hypothesis fit pretty well into the evolution theme. But how to make it work? Are you the borg, or fighting the Borg? The third and final idea came to me through my girlfriend, who somehow gave me the idea of making something about the evolution of Pasta. The more I thought about it the more it sounded like it would work, so I decided to go with it. Conversations with my inspiring co-worker Roushey (who also created the 'Mechanical Underdogs' signature logo for my intros) further matured the concept, as it involved into the idea of having individual pieces of pasta flying around and trying to evolve until they became all-powerful. A secondary idea here was that the game would work to explain how the Flying Spaghetti Monster came to exist - by evolving from a normal dinner table. So the idea evolved more or less into this: you are sitting a table. You have your own plate, with is your 'base'. There are 5 other guests at the table, each with their own plate. Your plate can spawn little pieces of pasta. You do so by 'ordering' them through a menu. Some pastas are better than others; some are faster, some are stronger. They have varying 'costs', which are debited from your credits (you start with a number of credits). Once spawned, your pastas start flying around. Their instinct is to fly to other plates, in order to conquer them (the objective of the game is having your pasta conquer all the plates on the table). But they are really autonomous, so after being spawned, you have no control over your pasta (think DotA or LoL creeps). Your pasta doesn't like other people's pasta, so if they meet, they shoot sauce at each other until one dies. You get credits for other pastas your own pasta kill. ", + "2016": "It is done, and submitted. You can play 'Survival of the Tastiest' on Android, and on the web. Playing on the web works, but you have to simulate multiple touch for table moving and that can be a bit confusing. There is a lot I'd like to talk about. I will go through every topic, insted of making the typical what went right/wrong list. Concept Working over the theme was probably one of the hardest tasks which I had to face. Originally, I had an idea of what kind of game I wanted to develop, gameplay wise - something with a lot of enemies/actors, simple graphics, maybe set in space, controlled from a top-down view. I was confident that I could fit any theme around it. In the end, the problem with a theme like 'Evolution' in a game is that evolution is unassisted. It happens through several seemingly random mutations over time, with the most apt permutation surviving. This genetic car simulator is, in my opinion, a great example of actual evolution of a species facing a challenge. But is it a game? In a game, you need to control something to reach an objective. That control goes against what evolution is supposed to be like. If you allow the user to pick how to evolve something, it's not evolution anymore - it's the equivalent of intelligent design, the fable invented by creationists to combat the idea of evolution. Being agnostic and a Pastafarian, that's not something that rubbed me the right way. Hence, my biggest dillema when deciding what to create was not with what I wanted to create, but with what I did not. I didn't want to create an 'intelligent design' simulator and wrongly call it evolution. This is a problem, of course, every other contestant also had to face. And judging by the entries submitted, not many managed to work around it. I'd say the only real solution was through the use of artificial selection, somehow. So far, I haven't seen any entry using this at its core gameplay. Alas, this is just a fun competition and after a while I decided not to be as strict with the game idea, and allowed myself to pick whatever I thought would work out. My initial idea was to create something where humanity tried to evolve to a next level, but had some kind of foe trying to stop them from doing so. I kind of had this image of human souls flying in space towards a monolith or a space baby (all based in 2001: A Space Odyssey of course) but I couldn't think of compelling (read: serious) mechanics for that. Borgs were my next inspiration, as their whole hypothesis fit pretty well into the evolution theme. But how to make it work? Are you the borg, or fighting the Borg? The third and final idea came to me through my girlfriend, who somehow gave me the idea of making something about the evolution of Pasta. The more I thought about it the more it sounded like it would work, so I decided to go with it. Conversations with my inspiring co-worker Roushey (who also created the 'Mechanical Underdogs' signature logo for my intros) further matured the concept, as it involved into the idea of having individual pieces of pasta flying around and trying to evolve until they became all-powerful. A secondary idea here was that the game would work to explain how the Flying Spaghetti Monster came to exist - by evolving from a normal dinner table. So the idea evolved more or less into this: you are sitting a table. You have your own plate, with is your 'base'. There are 5 other guests at the table, each with their own plate. Your plate can spawn little pieces of pasta. You do so by 'ordering' them through a menu. Some pastas are better than others; some are faster, some are stronger. They have varying 'costs', which are debited from your credits (you start with a number of credits). Once spawned, your pastas start flying around. Their instinct is to fly to other plates, in order to conquer them (the objective of the game is having your pasta conquer all the plates on the table). But they are really autonomous, so after being spawned, you have no control over your pasta (think DotA or LoL creeps). Your pasta doesn't like other people's pasta, so if they meet, they shoot sauce at each other until one dies. You get credits for other pastas your own pasta kill. Once a pasta is in the vicinity of a plate, it starts conquering it for its team. It takes around 10 seconds for a plate to be conquered; less if more pasta from the same team are around. If pasta from other team are around, though, they get locked down in their attempt, unable to conquer the plate, until one of them die (think Battlefield's standard 'Conquest' mode). You get points every second for every plate you own. Over time, the concept also evolved to use an Italian bistro as its main scenario. Carlos, Carlos' Bistro's founder and owner Setup No major changes were made from my work setup. I used FDT and Starling creating an Adobe AIR (ActionScript) project, all tools or frameworks I already had some knowledge with. One big change for me was that I livestreamed my work through a twitch.tv account. This was a new thing for me. As recommended by Roushey, I used a program called XSplit and I got to say, it is pretty amazing. It made the livestream pretty effortless and the features are awesome, even for the free version. It was great to have some of my friends watch me, and then interact with them and random people through chat. It was also good knowing that I was also recording a local version of the files, so I could make a timelapse video later. Knowing the video was being recorded also made me a lot more self-conscious about my computer use, as if someone was watching over my shoulder. It made me realize that sometimes I spend too much time in seemingly inane tasks (I ended up wasting the longest time just to get some text alignment the way I wanted - it'll probably drive someone crazy if they watch it) and that I do way too many typos where writing code. I pretty much spend half of the time writing a line and the other half fixing the crazy characters in it. My own stream was probably boring to watch since I was coding for the most time. But livestreaming is one of the cool things to do as a spectator too. It was great seeing other people working - I had a few tabs opened on my second monitor all the time. It's actually a bit sad, because if I could, I could have spent the whole weekend just watching other people working! But I had to do my own work, so I'd only do it once in a while, when resting for a bit. Design Although I wanted some simple, low-fi, high-contrast kind of design, I ended up going with somewhat realistic (vector) art. I think it worked very well, fitting the mood of the game, but I also went overboard. For example: to know the state of a plate (who owns it, who's conquering it and how much time they have left before conquering it, which pasta units are in the queue, etc), you have to look at the plate's bill. The problem I realized when doing some tests is that people never look at the bill! They think it's some kind of prop, so they never actually read its details. Plus, if you're zoomed out too much, you can't actually read it, so it's hard to know what's going on with the game until you zoom in to the area of a specific plate. One other solution that didn't turn out to be as perfect as I thought was how to indicate who a plate base belongs to. In the game, that's indicated by the plate's decoration - its color denotes the team owner. But it's something that fits so well into the design that people never realized it, until they were told about it. In the end, the idea of going with a full physical metaphor is one that should be done with care. Things that are very important risk becoming background noise, unless the player knows its importance. Originally, I wanted to avoid any kind of heads-up display in my game. In the end, I ended up adding it at the bottom to indicate your credits and bases owned, as well as the hideous out-of-place-and-still-not-obvious 'Call Waiter' button. But in hindsight, I should have gone with a simple HUD from the start, especially one that indicated each team's colors and general state of the game without the need for zooming in and out. Development Development went fast. But not fast enough. Even though I worked around 32+ hours " +} \ No newline at end of file diff --git a/benchmark/llama-7b/run-llama-7b.sh b/benchmark/llama-7b/run-llama-7b.sh new file mode 100755 index 0000000..ed1c471 --- /dev/null +++ b/benchmark/llama-7b/run-llama-7b.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# On different systems, different models and differnt modes, OMP_NUM_THREADS need to be ajusted. +output=$(numactl -H) +first_line=$(echo "$output" | head -n 1) +nodes=$(echo "$first_line" | awk '{print $2}') + +echo "memory node number: $nodes" +if [ "$nodes" -eq 16 ]; then +#HBM SNC-4 mode, Confirm that there are 8 HBM memory nodes and 8 DRAM memory nodes through "numactl -H" +#0-7 is DRAM memory node, 8-15 is HBM node + echo "HBM SNC4 mode" +# Run Llama-7B on 1 socket HBM SNC4 mode + OMP_NUM_THREADS=12 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -m 8 -N 0 sh llama-7b.sh : \ + -n 1 numactl -m 9 -N 1 sh llama-7b.sh : \ + -n 1 numactl -m 10 -N 2 sh llama-7b.sh : \ + -n 1 numactl -m 11 -N 3 sh llama-7b.sh +# -n 1 numactl -m 12 -N 4 sh llama-7b.sh : \ +# -n 1 numactl -m 13 -N 5 sh llama-7b.sh : \ +# -n 1 numactl -m 14 -N 6 sh llama-7b.sh : \ +# -n 1 numactl -m 15 -N 7 sh llama-7b.sh + +elif [ "$nodes" -eq 4 ]; then +#HBM Quad-mode, Confirm that there are 2 HBM memory nodes and 2 DRAM memory nodes through "nuamctl -H" + echo "HBM Quad mode" +# Run Llama-7B on 1 socket HBM Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 2 sh llama-7b.sh : \ + -n 1 numactl -N 1 -m 3 sh llama-7b.sh + +elif [ "$nodes" -eq 2 ]; then +#SPR Quad-mode, Confirm that there are 2 DRAM memory nodes through "nuamctl -H" + echo "SPR Quad mode" +# Run Llama-7B on 1 socket SPR Quad mode + OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 sh llama-7b.sh + +else + echo "Please double check the memory nodes" +fi + + diff --git a/ci_build b/ci_build new file mode 100755 index 0000000..6f3b24d --- /dev/null +++ b/ci_build @@ -0,0 +1,51 @@ +#!/bin/bash +set -e -x + +# todo(marvin): move oneccl deps into cmake. +pushd 3rdparty/ +sh prepare_oneccl.sh +source ./oneCCL/build/_install/env/setvars.sh +popd + +# Define functions for build, UT, and model +build() { + echo "Running build function with arguments: $@" + rm -rf build && mkdir build && cd build && cmake .. && make -j +} + +ut() { + echo "Running UT function with arguments: $@" + cd tests/ut + # rm -rf build && mkdir build && cd build && cmake .. && make -j +} + +model() { + numactl -H + core_count=$(lscpu | grep "Core(s) per socket" | awk '{print $NF}') + + echo "Running model function with arguments: $@" + # DATASETS_LIST=( 'Llama-2-7b' 'chatglm-6b' 'chatglm2-6b' 'llama-13b' 'llama-7b' 'opt-1.3b' 'opt-13b' 'opt-30b' ) + DATASETS_LIST=( 'llama-2-7b-chat' 'llama-2-13b-chat' 'chatglm2-6b' 'llama-13b' 'llama-7b' ) + DATATYPE_LIST=( 'fp16' 'bf16' 'int8' 'bf16_fp16' 'bf16_int8' ) + for DATASET in ${DATASETS_LIST[@]}; do + for DATATYPE in ${DATATYPE_LIST[@]}; do + # always run one socket + OMP_NUM_THREADS=$core_count numactl -N 0 -m 0 \ + build/example --model=/data/${DATASET}-cpu/ --token=/data/${DATASET}-hf/tokenizer.model \ + --dtype=${DATATYPE} --loop=3 --input_len=18 --output_len=32 --no_stream + done + done +} + +# Check if a function with the given name exists and call it with provided arguments +if [ "$#" -ge 1 ]; then + function_name="$1" + shift + if [ "$(type -t $function_name)" = "function" ]; then + $function_name "$@" + else + echo "Function $function_name not found." + fi +else + echo "Usage: ci_build function_name [function_arguments...]" +fi \ No newline at end of file diff --git a/cmake/cmdline.cmake b/cmake/cmdline.cmake new file mode 100644 index 0000000..169f6d9 --- /dev/null +++ b/cmake/cmdline.cmake @@ -0,0 +1,26 @@ +# Copyright (c) Intel Corporation. All rights reserved. +# Licensed under the Apache V2.0 License. + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(cmdline + URL https://github.com/tanakh/cmdline/archive/refs/heads/master.zip + URL_HASH MD5=69f98dc95edcae8c423a62ceccf81644 + TIMEOUT 60 + SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/cmdline + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) +# cmake-format: on diff --git a/cmake/ig.cmake b/cmake/ig.cmake new file mode 100644 index 0000000..3580f7d --- /dev/null +++ b/cmake/ig.cmake @@ -0,0 +1,26 @@ +# Copyright (c) Intel Corporation. All rights reserved. +# Licensed under the Apache V2.0 License. + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(ig_lib + URL https://github.com/intel/xFasterTransformer/releases/download/IntrinsicGemm/ig_v1.1.tar.gz + URL_HASH MD5=47e5a2cd021caad2b1367c0b71dff2e7 + TIMEOUT 60 + SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/ig + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) +# cmake-format: on diff --git a/cmake/jsoncpp.cmake b/cmake/jsoncpp.cmake new file mode 100644 index 0000000..17cd619 --- /dev/null +++ b/cmake/jsoncpp.cmake @@ -0,0 +1,27 @@ +# Copyright (c) Intel Corporation. All rights reserved. +# Licensed under the Apache V2.0 License. + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(jsoncpp_lib + URL https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.8.4.tar.gz + URL_HASH MD5=fa47a3ab6b381869b6a5f20811198662 + TIMEOUT 60 + SOURCE_DIR ./jsoncpp-prefix + BINARY_DIR ./jsoncpp-prefix + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E make_directory "build" && ${CMAKE_COMMAND} -E chdir "build" ${CMAKE_COMMAND} -DCMAKE_CXX_FLAGS=-fPIC -DCMAKE_BUILD_TYPE=release -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=ON -DJSONCPP_WITH_TESTS=OFF -DJSONCPP_WITH_POST_BUILD_UNITTEST=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_SOURCE_DIR}/3rdparty/jsoncpp .. + BUILD_COMMAND ${CMAKE_COMMAND} -E chdir "build" make + INSTALL_COMMAND ${CMAKE_COMMAND} -E chdir "build" make install + TEST_COMMAND "" +) +# cmake-format: on diff --git a/cmake/mklml.cmake b/cmake/mklml.cmake new file mode 100644 index 0000000..cd00a4c --- /dev/null +++ b/cmake/mklml.cmake @@ -0,0 +1,26 @@ +# Copyright (c) Intel Corporation. All rights reserved. +# Licensed under the Apache V2.0 License. + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(mklml + URL https://github.com/oneapi-src/oneDNN/releases/download/v0.21/mklml_lnx_2019.0.5.20190502.tgz + URL_HASH MD5=dfcea335652dbf3518e1d02cab2cea97 + TIMEOUT 60 + SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/mklml + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) +# cmake-format: on diff --git a/cmake/onednn.cmake b/cmake/onednn.cmake new file mode 100644 index 0000000..e0a4375 --- /dev/null +++ b/cmake/onednn.cmake @@ -0,0 +1,26 @@ +# Copyright (c) Intel Corporation. All rights reserved. +# Licensed under the Apache V2.0 License. + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(onednn + GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git + GIT_TAG v3.2 + SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/onednn + BINARY_DIR ${CMAKE_SOURCE_DIR}/3rdparty/onednn + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E make_directory "build" && ${CMAKE_COMMAND} -E chdir "build" ${CMAKE_COMMAND} -DONEDNN_LIBRARY_TYPE=STATIC -DONEDNN_BUILD_TESTS=OFF -DONEDNN_BUILD_EXAMPLES=OFF .. + BUILD_COMMAND ${CMAKE_COMMAND} -E chdir "build" make -j all + INSTALL_COMMAND "" + TEST_COMMAND "" +) +# cmake-format: on diff --git a/cmake/sentencepiece.cmake b/cmake/sentencepiece.cmake new file mode 100644 index 0000000..af9fb31 --- /dev/null +++ b/cmake/sentencepiece.cmake @@ -0,0 +1,27 @@ +# Copyright (c) Intel Corporation. All rights reserved. +# Licensed under the Apache V2.0 License. + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(sentencepiece_lib + URL https://github.com/google/sentencepiece/releases/download/v0.1.99/sentencepiece-0.1.99.tar.gz + URL_HASH MD5=6af04027121d138eb12c458a53df937e + TIMEOUT 60 + SOURCE_DIR ./sentencepiece-prefix + BINARY_DIR ./sentencepiece-prefix + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E make_directory "build" && ${CMAKE_COMMAND} -E chdir "build" ${CMAKE_COMMAND} -DSPM_ENABLE_SHARED=OFF -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0 -DCMAKE_INSTALL_PREFIX=${CMAKE_SOURCE_DIR}/3rdparty/sentencepiece ../sentencepiece + BUILD_COMMAND ${CMAKE_COMMAND} -E chdir "build" make -j + INSTALL_COMMAND ${CMAKE_COMMAND} -E chdir "build" make install + TEST_COMMAND "" +) +# cmake-format: on diff --git a/cmake/xdnn.cmake b/cmake/xdnn.cmake new file mode 100644 index 0000000..a8f2814 --- /dev/null +++ b/cmake/xdnn.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +cmake_minimum_required(VERSION 3.18) + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +project(dependency NONE) + +include(ExternalProject) + +# cmake-format: off +ExternalProject_Add(xdnn_lib + URL https://github.com/intel/xFasterTransformer/releases/download/IntrinsicGemm/xdnn_v1.1.tar.gz + URL_HASH MD5=b49bf8808d66ea75cfba80a406c9a587 + TIMEOUT 60 + SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/xdnn + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) +# cmake-format: on diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 0000000..e924758 --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,57 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc-12 \ + g++-12 \ + make \ + vim \ + wget \ + libnuma-dev \ + numactl \ + git \ + libjsoncpp-dev \ + pkg-config \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 \ + && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 60 + + +# Install python +RUN apt install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt update -y \ + && apt install python3.8 python3.8-dev python3.8-distutils python3.8-venv -y \ + && wget https://bootstrap.pypa.io/get-pip.py \ + && python3.8 get-pip.py \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 60 \ + && update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip3.8 60 \ + && python -m pip install --upgrade pip \ + && rm get-pip.py + +RUN pip install cmake==3.26.1 transformers==4.28.1 sentencepiece==0.1.99 tokenizers==0.13.3 +RUN pip install torch==2.0.1+cpu --index-url https://download.pytorch.org/whl/cpu + +# Install oneCCL +RUN git clone https://github.com/oneapi-src/oneCCL.git /tmp/oneCCL \ + && cd /tmp/oneCCL \ + && git checkout 2021.9 \ + && sed -i 's/cpu_gpu_dpcpp/./g' cmake/templates/oneCCLConfig.cmake.in \ + && mkdir build \ + && cd build \ + && cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/oneCCL \ + && make -j install \ + && cd ~ \ + && rm -rf /tmp/oneCCL + +RUN echo "source /usr/local/oneCCL/env/setvars.sh" >> ~/.bashrc + +WORKDIR /usr/local/ + +RUN wget https://github.com/oneapi-src/oneDNN/releases/download/v0.21/mklml_lnx_2019.0.5.20190502.tgz \ + && tar -xzf mklml_lnx_2019.0.5.20190502.tgz \ + && rm -f mklml_lnx_2019.0.5.20190502.tgz + +RUN echo 'export LD_LIBRARY_PATH=/usr/local/mklml_lnx_2019.0.5.20190502/lib:$LD_LIBRARY_PATH' >> /root/.bashrc + +WORKDIR /root/ \ No newline at end of file diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev new file mode 100644 index 0000000..7925bc2 --- /dev/null +++ b/dockerfiles/Dockerfile.dev @@ -0,0 +1,44 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +ARG BASE_IMG=intel/xfastertransformer +ARG VERSION=dev-ubuntu22.04 + +FROM ${BASE_IMG}:${VERSION} + +ARG USER_NAME +ARG USER_UID +ARG USER_GROUP +ARG USER_GID +ARG USER_PASSWD=qwer1234 +ARG USER_HOME=/home/${USER_NAME} + + +############################# Set same user in container ############################# +RUN apt-get install sudo -y +WORKDIR / +RUN getent group "${USER_GID}" || addgroup --force-badname --gid ${USER_GID} ${USER_GROUP} +RUN getent passwd "${USER_UID}" || adduser --force-badname --gid ${USER_GID} --uid ${USER_UID} \ + --disabled-password --home ${USER_HOME} --quiet ${USER_NAME} +RUN usermod -a -G sudo ${USER_NAME} +RUN echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo + +USER ${USER_UID}:${USER_GID} + +RUN echo ${USER_NAME}:${USER_PASSWD} | sudo chpasswd +RUN whoami + +# Set oneCCL environment +RUN echo "source /usr/local/oneCCL/env/setvars.sh" >> ${USER_HOME}/.bashrc +RUN echo 'export LD_LIBRARY_PATH=/usr/local/mklml_lnx_2019.0.5.20190502/lib:$LD_LIBRARY_PATH' >> ${USER_HOME}/.bashrc \ No newline at end of file diff --git a/dockerfiles/Dockerfile.release b/dockerfiles/Dockerfile.release new file mode 100644 index 0000000..c16b12a --- /dev/null +++ b/dockerfiles/Dockerfile.release @@ -0,0 +1,45 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +ARG BASE_IMG=intel/xfastertransformer +ARG VERSION=dev-ubuntu22.04 + +FROM ${BASE_IMG}:${VERSION} + +ARG TAG=main + +RUN echo 'export LD_PRELOAD=libiomp5.so:$LD_PRELOAD' >> /root/.bashrc + +WORKDIR /root/ +RUN git clone https://github.com/intel/xFasterTransformer.git + +RUN cd xFasterTransformer \ + && git checkout ${TAG} \ + && sed -i 's/source ${CCL_ROOT}\/env\/vars.sh/. ${CCL_ROOT}\/env\/vars.sh/g' /usr/local/oneCCL/env/setvars.sh \ + && . /usr/local/oneCCL/env/setvars.sh \ + && export LD_LIBRARY_PATH=/usr/local/mklml_lnx_2019.0.5.20190502/lib:$LD_LIBRARY_PATH \ + && python setup.py build \ + && python setup.py egg_info bdist_wheel --verbose \ + && pip install dist/* + +RUN mkdir -p /usr/local/xft/lib \ + && cp /root/xFasterTransformer/build/libxfastertransformer.so /usr/local/xft/lib \ + && cp -r /root/xFasterTransformer/include /usr/local/xft/ \ + && mkdir -p /usr/local/include/xft/ \ + && ln -s /usr/local/xft/include /usr/local/include/xft/include + + +RUN rm -rf /root/xFasterTransformer + +RUN echo 'export LD_LIBRARY_PATH=/usr/local/xft/lib:$LD_LIBRARY_PATH' >> /root/.bashrc \ No newline at end of file diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..6e0e13d --- /dev/null +++ b/examples/README.md @@ -0,0 +1,15 @@ +# Examples +xFasterTransformer provides C++, Python(Pytorch) examples to help users learn the API usage. Web demos of some models based on [Gradio](https://www.gradio.app/) are provided. All of the examples and web demo support multi-instance. + +## [C++ example](cpp/README.md) +C++ example support automatic identification model and tokenizer which is implemented by [SentencePiece](https://github.com/google/sentencepiece), excluding Opt model which tokenizer is a hard code. + +## [Python (PyTorch) example](pytorch/README.md) +Python(PyTorch) example achieves end-to-end inference of the model with streaming output combining the transformer's tokenizer. + +## [Web Demo](web_demo/README.md) +A web demo based on [Gradio](https://www.gradio.app/) is provided in repo. +Support list: +- ChatGLM +- ChatGLM2 +- Llama2-chat \ No newline at end of file diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt new file mode 100644 index 0000000..a529809 --- /dev/null +++ b/examples/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (C) 2023-2024 Intel Corporation +cmake_minimum_required(VERSION 3.15.1) + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} EXAMPLE_SCR) + +include(${CMAKE_SOURCE_DIR}/cmake/cmdline.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/sentencepiece.cmake) + +add_executable(example ${EXAMPLE_SCR}) + +target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/3rdparty/cmdline) +target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/3rdparty/sentencepiece/include) + +target_link_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/3rdparty/sentencepiece/${CMAKE_INSTALL_LIBDIR}) + +if(BUILD_WITH_SHARED_LIBS) + target_link_libraries(example PRIVATE xfastertransformer) +else() + target_link_libraries(example PRIVATE xfastertransformer_static) +endif() +target_link_libraries(example PRIVATE sentencepiece -lstdc++fs) + +add_dependencies(example cmdline sentencepiece_lib) diff --git a/examples/cpp/README.md b/examples/cpp/README.md new file mode 100644 index 0000000..b3b5616 --- /dev/null +++ b/examples/cpp/README.md @@ -0,0 +1,32 @@ +# C++ example +C++ example support automatic identification model and tokenizer which is implemented by [SentencePiece](https://github.com/google/sentencepiece), excluding Opt model which tokenizer is a hard code. + +## Step 1: Build binary file +Please refer to [Build from source](../README.md#built-from-source) to build C++ example binary which is built with xFasterTransformer library and under `build` directory named `example`. + +## Step 2: Prepare models +Please refer to [Prepare model](../README.md#prepare-model) + +## Step 3: Run binary +```bash +# Recommend preloading `libiomp5.so` to get a better performance. +# `libiomp5.so` file will be in `3rdparty/mklml/lib` directory after build xFasterTransformer. +SINGLE_INSTANCE=1 LD_PRELOAD=libiomp5.so ./example -m ${MODEL_PATH} -t ${TOKEN_PATH} + +# run multi-instance like +OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun \ + -n 1 numactl -N 0 -m 0 ./example -m ${MODEL_PATH} -t ${TOKEN_PATH} : \ + -n 1 numactl -N 1 -m 1 ./example -m ${MODEL_PATH} -t ${TOKEN_PATH} +``` +More parameter options settings: +- `-?`, `-h`, `--help` Help information +- `-m`, `--model` directory path of xft format model. +- `-t`, `--token` path of tokenizer file(name like tokenizer.model), invalid for Opt model. +- `-i`, `--input` input prompt, invalid for Opt model. Default use `Once upon a time, there existed a little girl who liked to have adventures.` +- `-d`, `--dtype` data type, default `fp16`, should be one of `["fp16", "bf16", "int8", "bf16_fp16", "bf16_int8"]` +- `-l`, `--input_len` input token size. Input token ids will ben expand to this size if it greater than input prompt's size. +- `-n`, `--num_beams` number of beam size, default 1. +- `-b`, `--batch_size` batch size, default 1. If greater than 1, input prompt will be duplicated this times. +- `--output_len` max tokens can generate excluded input, default 100. +- `--loop` number of loop, default 10. +- `--no_stream` disable streaming output. \ No newline at end of file diff --git a/examples/cpp/example.cpp b/examples/cpp/example.cpp new file mode 100644 index 0000000..bb7a1fd --- /dev/null +++ b/examples/cpp/example.cpp @@ -0,0 +1,309 @@ +#include +#include +#include +#include +#include +#include + +#include "INIReader.h" +#include "cmdline.h" +#include "sentencepiece_processor.h" +#include "timer.h" +#include "xfastertransformer.h" + +extern const char *vocab_opt[]; + +class TokenizerBase { +public: + TokenizerBase() {} + TokenizerBase(std::string &tokenPath) { + std::filesystem::path filePath(tokenPath); + + if (!(std::filesystem::exists(filePath) && std::filesystem::is_regular_file(filePath))) { + std::cout << "[ERROR] " << filePath << " isn't a file or not existed." << std::endl; + exit(-1); + } + + const auto status = processor.Load(tokenPath); + if (!status.ok()) { + std::cout << status.ToString() << std::endl; + std::cout << "[ERROR] Fail to load tokenizer file." << std::endl; + exit(-1); + } + vocabSize = processor.GetPieceSize(); + }; + + virtual std::vector encode(std::string &input) { + std::vector output; + processor.Encode(input, &output); + addSpecialTokenIds(output); + return output; + } + + void addSpecialTokenIds(std::vector &input) { + input.insert(input.begin(), prefixTokenIds.begin(), prefixTokenIds.end()); + input.insert(input.end(), suffixTokenIds.begin(), suffixTokenIds.end()); + } + + virtual std::string decode(std::vector &ids) { + std::string text; + processor.Decode(ids, &text); + return text; + } + virtual std::string decode(int id) { return processor.IdToPiece(id); } + + void printResult(std::vector &ids, int batchSize, int numBeams) { + if (batchSize * numBeams > 2) { + printf("[%d]%s [%d]%s ... [%d]%s\n", ids[0], decode(ids[0]).c_str(), ids[1], decode(ids[1]).c_str(), + ids[batchSize * numBeams - 1], decode(ids[batchSize * numBeams - 1]).c_str()); + } else if (batchSize * numBeams > 1) { + printf("[%d]%s [%d]%s\n", ids[0], decode(ids[0]).c_str(), ids[batchSize * numBeams - 1], + decode(ids[batchSize * numBeams - 1]).c_str()); + } else { + printf("[%d]%s ", ids[0], decode(ids[0]).c_str()); + } + } + + std::vector batchDecode(std::vector &input, int batchSize) { + int seqLen = input.size() / batchSize; + std::vector ret; + for (int i = 0; i < batchSize; ++i) { + std::vector tokens(input.begin() + i * seqLen, input.begin() + (i + 1) * seqLen); + ret.emplace_back(decode(tokens)); + } + return ret; + } + +protected: + std::vector prefixTokens; + std::vector suffixTokens; + std::vector prefixTokenIds; + std::vector suffixTokenIds; + + sentencepiece::SentencePieceProcessor processor; + int vocabSize; +}; + +class ChatGLMTokenizer : public TokenizerBase { +public: + ChatGLMTokenizer(std::string &tokenPath) : TokenizerBase(tokenPath) { + suffixTokens = {"[gMASK]", ""}; + suffixTokenIds = {processor.PieceToId("[gMASK]"), processor.PieceToId("")}; + } +}; + +class ChatGLM2Tokenizer : public TokenizerBase { +public: + ChatGLM2Tokenizer(std::string &tokenPath) : TokenizerBase(tokenPath) { + // ChatGLM2's special tokens is not included in sentencepiece. ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + prefixTokens = {"[gMASK]", "sop"}; + prefixTokenIds = {vocabSize + 1, vocabSize + 3}; + } + std::string decode(std::vector &ids) override { + ids.erase(std::remove_if(ids.begin(), ids.end(), [this](int value) { return value >= vocabSize; }), ids.end()); + std::string text; + processor.Decode(ids, &text); + return text; + } + + std::string decode(int id) override { + if (id > vocabSize) { + return ""; + } else { + return processor.IdToPiece(id); + } + } +}; + +class LlamaTokenizer : public TokenizerBase { +public: + LlamaTokenizer(std::string &tokenPath) : TokenizerBase(tokenPath) { processor.SetEncodeExtraOptions("bos"); } +}; + +class OptTokenizer : public TokenizerBase { +public: + OptTokenizer(std::string &tokenPath) { vocabSize = 50265; } + + std::vector encode(std::string &input) override { + return std::vector({2, 11475, 2115, 10, 86, 6, 89, 13412, 10, 410, 1816, 54, 6640, 7, 33, 18848, 4}); + } + + std::string decode(std::vector &ids) override { + if (ids.size() == 1) { return decode(ids[0]); } + std::string text(""); + for (int id : ids) { + if (id < vocabSize) { + text += vocab_list[id]; + text += " "; + } else { + text += "(null) "; + } + } + return text; + } + std::string decode(int id) override { + if (id < vocabSize) { + return vocab_list[id]; + } else { + return "(null)"; + } + } + +private: + const char **vocab_list = vocab_opt; +}; + +TokenizerBase *getTokenizer(std::string &modeltype, std::string &tokenPath) { + if (modeltype == "gpt") { + return new OptTokenizer(tokenPath); + } else if (modeltype == "llama") { + return new LlamaTokenizer(tokenPath); + } else if (modeltype == "chatglm") { + return new ChatGLMTokenizer(tokenPath); + } else if (modeltype == "chatglm2") { + return new ChatGLM2Tokenizer(tokenPath); + } else { + std::cout << "[Error] Token list of loaded model is unsupported yet.\n" << std::endl; + exit(-1); + } +} + +std::map dataTypeMap + = {{"fp16", xft::DataType::fp16}, {"bf16", xft::DataType::bf16}, {"int8", xft::DataType::int8}, + {"bf16_fp16", xft::DataType::bf16_fp16}, {"bf16_int8", xft::DataType::bf16_int8}}; + +std::string getModelType(std::string &modelPath) { + std::string configPath = modelPath + "/config.ini"; + INIReader reader = INIReader(configPath); + if (reader.ParseError() < 0) { + printf("[Error] Could not load model config.ini.\n"); + exit(-1); + } + std::string modeltype = *reader.Sections().begin(); + return modeltype; +} + +int main(int argc, char **argv) { + cmdline::parser args; + + args.add("model", 'm', "path of xft format model", true); + args.add("token", 't', "path of tokenizer", true); + args.add("input", 'i', "input prompt, invalid for Opt model.", false, + "Once upon a time, there existed a little girl who liked to have adventures."); + args.add("dtype", 'd', "weight data type", false, "fp16", + cmdline::oneof("fp16", "bf16", "int8", "bf16_fp16", "bf16_int8")); + args.add("input_len", 'l', "input token size", false, -1); + args.add("output_len", '\0', "max tokens can generate excluded input.", false, 100, cmdline::range(1, 4096)); + args.add("num_beams", 'n', "number of beam size.", false, 1, cmdline::range(1, 32)); + args.add("batch_size", 'b', "batch size.", false, 1, cmdline::range(1, 32)); + args.add("loop", '\0', "number of loop.", false, 10); + args.add("no_stream", '\0', "disable streaming output"); + args.parse_check(argc, argv); + + std::string modelPath = args.get("model"); + std::string tokenPath = args.get("token"); + + bool streamingOutput = !args.exist("no_stream"); + + std::string dtype_name = args.get("dtype"); + xft::DataType dtype = xft::DataType::fp16; + + auto it = dataTypeMap.find(dtype_name); + if (it != dataTypeMap.end()) { + dtype = it->second; + } else { + std::cout << "[Error] Unsupport dtype index: " << dtype_name << std::endl; + return 0; + } + + int inputSize = args.get("input_len"); + int outputLen = args.get("output_len"); + int numBeams = args.get("num_beams"); + int batchSize = args.get("batch_size"); + int loop = args.get("loop"); + + std::string modeltype = getModelType(modelPath); + + auto *tokenizer = getTokenizer(modeltype, tokenPath); + // std::string inputPrompt("Once upon a time, there existed a little girl who liked to have adventures."); + std::string inputPrompt = args.get("input"); + std::vector input = tokenizer->encode(inputPrompt); + + xft::AutoModel model(modelPath, dtype); + bool isMaster = (model.getRank() == 0); + + // Need longer prompt + if (inputSize > 0 && inputSize > input.size()) { + input.reserve(inputSize); + std::vector fakeTokens(inputSize - input.size(), input[2]); + input.insert(input.begin() + 2, fakeTokens.begin(), fakeTokens.end()); + } else if (inputSize > 0) { + printf("[Warning] Do not support token size of %d, use %ld instead.\n", inputSize, input.size()); + } + int maxLen = input.size() + outputLen; + + if (batchSize > 1) { + int len = input.size(); + input.resize(len * batchSize); + for (int i = 1; i < batchSize; i++) { + std::copy(input.begin(), input.begin() + len, input.begin() + i * len); + } + } + + if (isMaster) { + std::cout << "[INFO] Model path is " << modelPath << std::endl; + std::cout << "[INFO] Token path is " << tokenPath << std::endl; + std::cout << "[INFO] Data type is " << dtype_name << std::endl; + std::cout << "[INFO] inputSize is " << inputSize << std::endl; + std::cout << "[INFO] outputLen is " << outputLen << std::endl; + std::cout << "[INFO] num_beams is " << numBeams << std::endl; + std::cout << "[INFO] batch_size is " << batchSize << std::endl; + std::cout << "[INFO] loop is " << loop << std::endl; + std::cout << "[INFO] Input prompt is :" << inputPrompt << std::endl; + std::cout << "[INFO] Input Token Ids is :"; + for (auto x : input) { + std::cout << x << " "; + } + std::cout << std::endl; + } + + for (int i = 0; i < loop; ++i) { + model.config(maxLen, numBeams); + model.input(input, batchSize); + + std::vector fisrtIds; + std::vector seconedIds; + + { + Timer t(isMaster, "[INFO] Fisrt token"); + fisrtIds = model.generate(); + } + + if (!model.isDone()) { + Timer t(isMaster, "[INFO] Second token"); + seconedIds = model.generate(); + } + + if (isMaster && streamingOutput) { + tokenizer->printResult(fisrtIds, batchSize, numBeams); + tokenizer->printResult(seconedIds, batchSize, numBeams); + } + + while (!model.isDone()) { + auto nextIds = model.generate(); + if (isMaster && streamingOutput) { tokenizer->printResult(nextIds, batchSize, numBeams); } + } + auto result = model.finalize(); + + if (isMaster) { + std::cout << "\n[INFO] Finalzie output is:" << std::endl; + std::vector sent = tokenizer->batchDecode(result, batchSize); + for (auto str : sent) { + std::cout << "==============================================" << std::endl; + std::cout << str << std::endl; + } + } + } + + return 0; +} diff --git a/examples/cpp/timer.h b/examples/cpp/timer.h new file mode 100644 index 0000000..994fb5d --- /dev/null +++ b/examples/cpp/timer.h @@ -0,0 +1,35 @@ +#ifndef __TIMER_H_ +#define __TIMER_H_ +#include + +#include +#include + +class Timer { +public: + Timer(bool _auto_print = false, std::string _name = "Timer") : auto_print(_auto_print), name(_name) { + gettimeofday(&start, NULL); + } + + ~Timer() { + if (auto_print) { + gettimeofday(&end, NULL); + float duration = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_usec - start.tv_usec) / 1000.0f; + std::cout << name << " time: " << duration << " ms" << std::endl; + } + } + + float getTime() { + gettimeofday(&end, NULL); + float duration = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_usec - start.tv_usec) / 1000.0f; + return duration; + } + +private: + bool auto_print; + std::string name; + struct timeval start; + struct timeval end; +}; + +#endif diff --git a/examples/cpp/vocab_opt.cpp b/examples/cpp/vocab_opt.cpp new file mode 100644 index 0000000..2cd7e7c --- /dev/null +++ b/examples/cpp/vocab_opt.cpp @@ -0,0 +1,50267 @@ +const char *vocab_opt[] = { + "", + "", + "", + "", + ".", + "the", + ",", + "to", + "and", + "of", + "a", + "in", + "-", + "for", + "that", + "on", + "is", + "âĢ", + "'s", + "with", + "The", + "was", + "\"", + "at", + "it", + "as", + "said", + "Ļ", + "be", + "s", + "by", + "from", + "are", + "have", + "has", + ":", + "(", + "he", + "I", + "his", + "will", + "an", + "this", + ")", + "âĢ", + "not", + "Ŀ", + "you", + "ľ", + "their", + "or", + "they", + "we", + "but", + "who", + "more", + "had", + "been", + "were", + "about", + ",\"", + "which", + "up", + "its", + "can", + "one", + "out", + "also", + "$", + "her", + "all", + "after", + ".\"", + "/", + "would", + "'t", + "year", + "when", + "first", + "she", + "two", + "over", + "people", + "A", + "our", + "It", + "time", + "than", + "into", + "there", + "t", + "He", + "new", + "âĢĶ", + "last", + "just", + "In", + "other", + "so", + "what", + "I", + "like", + "a", + "some", + "S", + "ë", + "them", + "years", + "'", + "do", + "your", + "-", + "1", + "\"", + "if", + "could", + "?", + "no", + "i", + "m", + "get", + "U", + "now", + "him", + "back", + "But", + "âĢĵ", + "my", + "'", + "only", + "three", + ";", + "2", + "The", + "1", + "percent", + "against", + "before", + "company", + "o", + "Trump", + "how", + "because", + "any", + "most", + "being", + "make", + "where", + "during", + "through", + "while", + "000", + "This", + "million", + "ing", + "3", + "made", + "well", + "10", + "down", + "off", + "says", + "me", + "B", + "going", + "team", + "We", + "those", + "government", + "way", + "We", + "many", + "then", + "work", + "told", + "com", + "2", + "game", + "And", + "in", + "year", + "p", + "very", + "day", + "home", + "take", + "week", + "since", + "New", + "may", + "even", + "season", + "see", + "2017", + "state", + "5", + "ed", + "should", + "around", + "2018", + "second", + "us", + "still", + "much", + "4", + "good", + "think", + "%", + "S", + "these", + "market", + "D", + "th", + "go", + "'re", + "such", + "know", + "including", + "don", + "y", + "next", + "P", + "did", + "under", + "say", + "en", + "L", + "between", + "per", + "K", + "C", + "6", + "world", + "part", + "N", + "right", + "want", + "four", + "),", + "high", + "need", + "re", + "e", + "It", + "help", + "5", + "3", + "country", + "R", + "police", + "A", + "long", + "They", + "end", + "er", + "T", + "M", + "u", + "both", + "here", + "an", + "on", + "7", + "de", + "She", + "business", + "report", + "j", + "ers", + "really", + "President", + "ar", + "G", + "Friday", + "F", + "best", + "same", + "another", + "set", + "old", + "That", + "as", + "n", + "come", + "family", + "public", + "For", + "As", + "0", + "H", + "8", + "20", + "five", + "es", + "Tuesday", + "n", + "Thursday", + "quarter", + "h", + "top", + "got", + "life", + "Monday", + "found", + "use", + "W", + "4", + "Wednesday", + "own", + "according", + "play", + "show", + "St", + "man", + "left", + "United", + "12", + "place", + "If", + "lot", + "former", + "0", + ").", + "support", + "ie", + "billion", + "t", + "shares", + "!", + "z", + "k", + "State", + "points", + "group", + "school", + "information", + "2016", + "al", + "r", + "win", + "news", + "used", + "put", + "city", + "J", + "There", + "number", + "C", + "'ve", + "each", + "too", + "won", + "ly", + "month", + "is", + "added", + "look", + "better", + "every", + "&", + "days", + "9", + "took", + "night", + "e", + "11", + "os", + "few", + "or", + "North", + "You", + "third", + "great", + "called", + "On", + "past", + "came", + "months", + "Saturday", + "15", + "big", + "E", + "US", + "things", + "O", + "d", + "start", + "B", + "stock", + "30", + "women", + "South", + "May", + "never", + "president", + "Sunday", + "without", + "man", + "8", + "didn", + "local", + "6", + "something", + "case", + "All", + "it", + "7", + "So", + "children", + "away", + "little", + "six", + "City", + "County", + "data", + "at", + "already", + "d", + "money", + "early", + "across", + "expected", + "run", + "later", + "am", + "price", + "games", + "Mr", + "b", + "might", + "different", + "reported", + "deal", + "media", + "growth", + "community", + "China", + "'m", + "c", + "went", + "No", + "able", + "making", + "area", + "far", + "statement", + "House", + "working", + "M", + "k", + "seen", + "companies", + "today", + "members", + "until", + "full", + "again", + "half", + "share", + "le", + "always", + "court", + "l", + "and", + "change", + "find", + "9", + "system", + "V", + "York", + "American", + "head", + "players", + "does", + "health", + "m", + "power", + "point", + "hit", + ".", + "--", + "free", + ".,", + "lead", + "several", + "recent", + "call", + "N", + "law", + "keep", + "open", + "News", + "give", + "ia", + "March", + "D", + "National", + "At", + "times", + "future", + "R", + "14", + "June", + "officials", + "18", + "important", + "f", + "final", + "13", + "One", + "P", + "following", + "car", + "least", + "water", + "event", + "line", + "move", + "services", + "having", + "When", + "students", + "Police", + "el", + "am", + "Z", + "side", + "story", + "due", + "meeting", + "K", + "must", + "States", + "likely", + "G", + "continue", + "ago", + "party", + "major", + "industry", + "less", + "30", + "un", + "hard", + "service", + "16", + "looking", + "held", + "ve", + "whether", + "July", + "taken", + "along", + "asked", + "started", + "become", + "forward", + "research", + "office", + "political", + "to", + "together", + "getting", + "plan", + "25", + "T", + "among", + "coming", + "decision", + "video", + "2015", + "g", + "After", + "security", + "L", + "care", + "given", + "available", + "âĢĶ", + "s", + "West", + "'ll", + "pay", + "near", + "saying", + "announced", + "program", + "April", + "real", + "University", + "With", + "AP", + "social", + "close", + "et", + "current", + "why", + "F", + "To", + "Twitter", + "though", + "17", + "taking", + "Inc", + "men", + "w", + "comes", + "ley", + "doing", + "process", + "John", + "ch", + "00", + "financial", + "low", + "enough", + "While", + "further", + "post", + "feel", + "st", + "person", + "Facebook", + "World", + "within", + "ad", + "done", + "the", + "late", + "tax", + "doesn", + "thing", + "national", + "job", + "using", + "However", + "ic", + "campaign", + "record", + "behind", + "://", + "Department", + "p", + "others", + "January", + "order", + "[", + "sales", + "yet", + "Ä", + "small", + "series", + "face", + "What", + "50", + "ever", + "earlier", + "love", + "up", + "rights", + "An", + "ist", + "morning", + "Washington", + "young", + "latest", + "India", + "trying", + "fire", + "led", + "strong", + "return", + "level", + "O", + "average", + "period", + "experience", + "ak", + "possible", + "believe", + "include", + "oil", + "recently", + "once", + "known", + "lost", + "sure", + "us", + "weeks", + "food", + "reports", + "rating", + "Minister", + "woman", + "provide", + "project", + "issue", + "live", + "10", + "clear", + "he", + "cost", + "played", + "released", + "coach", + "v", + "24", + "seven", + "plans", + "development", + "ur", + "ĺ", + "increase", + "This", + "policy", + "cent", + "based", + "E", + "il", + "December", + "global", + "trade", + "hours", + "higher", + "goal", + "H", + "Al", + "100", + "minutes", + "election", + "America", + "rate", + "Ch", + "21", + "...", + "White", + "director", + "position", + "shot", + "large", + "c", + "b", + "]", + "issues", + "death", + "building", + "total", + "often", + "v", + "countries", + "history", + "outside", + "federal", + "19", + "fact", + "High", + "career", + "im", + "international", + "November", + "front", + "kind", + "key", + "ra", + "San", + "short", + "name", + "According", + "course", + "re", + "wanted", + "W", + "September", + "interest", + "role", + "results", + "economic", + "2014", + "chance", + "October", + "special", + "official", + "needs", + "um", + "l", + "products", + "non", + "@", + "Bank", + "ahead", + "house", + "U", + "board", + "old", + "saw", + "lower", + "European", + "control", + "Russia", + "eight", + "release", + "potential", + "thought", + "investigation", + "online", + "based", + "technology", + "Donald", + "id", + "body", + "risk", + "ian", + "capital", + "staff", + "action", + "League", + "playing", + "makes", + "almost", + "performance", + "22", + "g", + "film", + "nearly", + "Center", + "visit", + "Group", + "bank", + "bit", + "received", + "August", + "military", + "His", + "ine", + "chief", + "School", + "bring", + "Court", + "(@", + "means", + "Sh", + "fans", + "se", + "40", + "20", + "\".", + "V", + "cut", + "killed", + "#", + "prices", + "gave", + "Street", + "ir", + "Y", + "currently", + "f", + "ay", + "ne", + "te", + "try", + "Park", + "ĥ", + "J", + "question", + "hand", + "economy", + "investors", + "able", + "player", + "By", + "David", + "loss", + "ab", + "below", + "wrote", + "co", + "ate", + "running", + "un", + "began", + "single", + "field", + "23", + "leader", + "w", + "California", + "fourth", + "actually", + "list", + "ll", + "couple", + "study", + "teams", + "He", + "ah", + "Canada", + "la", + "result", + "access", + "vote", + "More", + "February", + "revenue", + "offer", + "let", + "ier", + "buy", + "attack", + "black", + "r", + "areas", + "stop", + "impact", + "match", + "investment", + "customers", + "leaders", + "ies", + "member", + "child", + "road", + "ul", + "value", + "shows", + "Dr", + "De", + "ant", + "London", + "room", + "music", + "production", + "anything", + "firm", + "biggest", + "air", + "problem", + "general", + "wasn", + "i", + "private", + "especially", + "administration", + "additional", + "Co", + "opportunity", + "hold", + "&", + "matter", + "senior", + "club", + "someone", + "Ã", + "East", + "2019", + ".'", + "needed", + "James", + "time", + "however", + "everything", + "everyone", + "died", + "involved", + "friends", + "isn", + "worth", + "ik", + "Cup", + "showed", + "There", + "28", + "meet", + "26", + "27", + "Y", + "region", + "Press", + "Now", + "son", + "space", + "leading", + "states", + "weekend", + "£", + "mother", + "previous", + "UK", + "Michael", + "leave", + "est", + "em", + "z", + "Some", + "ors", + "out", + "15", + "war", + "website", + "star", + "X", + "ro", + "target", + "himself", + "turn", + "Europe", + "worked", + "energy", + "scored", + "*", + "soon", + "ball", + "TV", + "annual", + "2013", + "race", + "International", + "'d", + "Market", + "conference", + "io", + "o", + "changes", + "ig", + "officers", + "inside", + "form", + "published", + "phone", + "co", + "legal", + "executive", + "fight", + "ings", + "hope", + "summer", + "officer", + "football", + "property", + "@", + "book", + "parents", + "costs", + "ac", + "manager", + "create", + "age", + "email", + "markets", + "main", + "human", + "sent", + "management", + "Day", + "ton", + "cash", + "focus", + "expect", + "training", + "became", + "whose", + "events", + "round", + "Le", + "fell", + "above", + "analysts", + "talk", + "situation", + "ri", + "ated", + "ke", + "wants", + "ag", + "lives", + "om", + "al", + "demand", + "safety", + "rest", + "Council", + "personal", + "site", + "Russian", + "mid", + "nothing", + "whole", + "bill", + "sold", + "British", + "se", + "remain", + "12", + "foreign", + "shooting", + "stay", + "50", + "ang", + "hospital", + "bad", + "address", + "Korea", + "happened", + "charges", + "white", + "31", + "If", + "earnings", + "break", + "light", + "terms", + "Chinese", + "Senate", + "ana", + "idea", + "ap", + "of", + "nine", + "compared", + "build", + "ard", + "In", + "similar", + "gas", + "victory", + "2012", + "debt", + "Mar", + "arrested", + "comment", + "increased", + "medical", + "29", + "Jan", + "groups", + "despite", + "fall", + "tell", + "workers", + "town", + "é", + "wife", + "questions", + "continued", + "heart", + "met", + "brought", + "helped", + "Congress", + "step", + "father", + "moment", + "product", + "probably", + "largest", + "vehicle", + "England", + "allow", + "starting", + "kids", + "incident", + "net", + "rates", + "Read", + "pressure", + "included", + "read", + "issued", + "ol", + "either", + "efforts", + "includes", + "Republican", + "ish", + "âĢ¦", + "goals", + "aj", + "en", + "x", + "raised", + "au", + "longer", + "ut", + "watch", + "Texas", + "You", + "range", + "nd", + "funds", + "remains", + "Mark", + "60", + "que", + "sh", + "interview", + "rather", + "residents", + "growing", + "pre", + "paid", + "cases", + "Reuters", + "difficult", + "sign", + "Google", + "https", + "Paul", + "living", + "day", + "Q", + "iz", + "Red", + "land", + "They", + "Road", + "_", + "These", + "view", + "agency", + "reason", + "allowed", + "Australia", + "az", + "Re", + "turned", + "11", + "nation", + "ready", + "press", + "budget", + "daily", + "Chief", + "families", + "significant", + "First", + "themselves", + "j", + "runs", + "accused", + "takes", + "spent", + "via", + "ot", + "ina", + "25", + "land", + "example", + "authorities", + "date", + "ended", + "all", + "Reuters", + "businesses", + "ans", + "details", + "ground", + "pretty", + "Apple", + "ation", + "Smith", + "Company", + "Florida", + "drug", + "response", + "one", + "education", + "mean", + "league", + "anyone", + "minister", + "title", + "adding", + "problems", + "opening", + "conditions", + "red", + "decided", + "Å", + "posted", + "term", + "amount", + "EU", + "success", + "evidence", + "Obama", + "addition", + "provided", + "Los", + "agreement", + "stage", + "ens", + "relationship", + "General", + "sector", + "student", + "ating", + "test", + "\",", + "winning", + "felt", + "source", + "Z", + "seems", + "cause", + "schools", + "drive", + "ensure", + "huge", + "My", + "Health", + "scene", + "giving", + "center", + "positive", + "yards", + "jobs", + "account", + "heard", + "quality", + "ways", + "immediately", + "employees", + "are", + "pass", + "CEO", + "receive", + "looks", + "Africa", + "throughout", + "led", + "related", + "sell", + "Union", + "Photo", + "ter", + "quickly", + "How", + "various", + "reach", + "pick", + "charged", + "quite", + "ent", + "q", + "ins", + "photo", + "understand", + "âĢ¢", + "reached", + "track", + "uk", + "effort", + "ville", + "central", + "daughter", + "contract", + "injury", + "opened", + "($", + "straight", + "17", + "credit", + "Indian", + "sexual", + "works", + "easy", + "18", + "closed", + "h", + "happen", + "force", + "ler", + "happy", + "shared", + "overall", + "moving", + "á", + "projects", + "Black", + "concerns", + "class", + "tried", + "appeared", + "content", + "District", + "term", + "instead", + "Office", + "continues", + "levels", + "afternoon", + "fund", + "sale", + "driver", + "ask", + "cannot", + "ner", + "end", + "Here", + "field", + "store", + "www", + "certain", + "self", + "dollar", + "Her", + "popular", + "follow", + "spending", + "by", + "moved", + "goes", + "created", + "stand", + "operations", + "looked", + "treatment", + "ov", + "district", + "signed", + "hands", + "model", + "Angeles", + "y", + "border", + "income", + "Last", + "charge", + "driving", + "Japan", + "rise", + "talks", + "followed", + "previously", + "users", + "funding", + "Johnson", + "", + "ou", + "ai", + "named", + "friend", + "Nov", + "defense", + "Britain", + "entire", + "trading", + "failed", + "El", + "claims", + "comments", + "beat", + "ib", + "basis", + "Jones", + "present", + "Be", + "double", + "rose", + "ite", + "ability", + "original", + "dead", + "Commission", + "Me", + "competition", + "2011", + "knew", + "material", + "av", + "France", + "score", + "sense", + "serious", + "confirmed", + "anti", + "violence", + "improve", + "son", + "ó", + "AP", + "sh", + "host", + "Mike", + "patients", + "NFL", + "crisis", + "revealed", + "ach", + "Prime", + "built", + "Not", + "rules", + "else", + "department", + "itself", + "ise", + "500", + "complete", + "ion", + "trial", + "Bay", + "Dec", + "attention", + "travel", + "Central", + "ry", + "agreed", + "mind", + "Mc", + "70", + "contact", + "ari", + "Times", + "spot", + "French", + "gets", + "op", + "brand", + "calls", + "banks", + "design", + "safe", + "offers", + "practice", + "Of", + "á", + "ling", + "true", + "off", + "numbers", + "fun", + "learn", + "multiple", + "Is", + "res", + "als", + "common", + "ized", + "challenge", + "committee", + "Our", + "base", + "ani", + "Association", + "ung", + "network", + "Brown", + "approach", + "16", + "finished", + "review", + "required", + "app", + "Man", + "âĢ¦", + "twitter", + "Democratic", + "13", + "evening", + "Tom", + "ä", + "Associated", + "Canadian", + "college", + "spokesman", + "article", + "towards", + "Chicago", + "movie", + "14", + "ity", + "forces", + "Chris", + "Democrats", + "features", + "hearing", + "X", + "Also", + "message", + "age", + "noted", + "Super", + "thousands", + "aw", + "Bill", + "Ar", + "La", + "ip", + "/", + "During", + "note", + ".)", + "wrong", + "if", + "passed", + "Two", + "die", + ",'", + "Don", + "Germany", + "letter", + "described", + "Iran", + "Williams", + "particularly", + "add", + "conversation", + "Se", + "highest", + "be", + "homes", + "sports", + "gone", + "Ad", + "el", + "opportunities", + "words", + "leaving", + "Christmas", + "As", + "Government", + "simply", + "husband", + "Research", + "Mexico", + "ates", + "ale", + "Green", + "$", + "od", + "Hall", + "natural", + "operating", + "les", + "ations", + "Kim", + "gold", + "ok", + "provides", + "(", + "ell", + "begin", + "Party", + "back", + "Amazon", + "19", + "majority", + "Even", + "check", + "weather", + "organization", + "stories", + "Car", + "forced", + "George", + "walk", + "ong", + "filed", + "Justice", + "launched", + "offered", + "www", + "construction", + "Ben", + "served", + "...", + "parts", + "cancer", + "guys", + "Reporting", + "ash", + "less", + "leadership", + "Committee", + "regular", + "council", + "cars", + "Director", + "judge", + "victims", + "Daily", + "kept", + "effect", + "beyond", + "pm", + "talking", + "considered", + "ore", + "Advertisement", + "st", + "ED", + "middle", + "raise", + "we", + "claimed", + "ino", + "alleged", + "Pro", + "Scott", + "Oct", + "consider", + "Share", + "traffic", + "African", + "couldn", + "toward", + "search", + "But", + "launch", + "injured", + "That", + "although", + "activities", + "changed", + "sources", + "missing", + "u", + "35", + "cover", + "ised", + "|", + "ow", + "ES", + "decades", + "ich", + "caused", + "elections", + "ane", + "IS", + "feet", + "Bar", + "version", + "grow", + "vehicles", + "options", + "individual", + "environment", + "Robert", + "Valley", + "From", + "per", + "ara", + "systems", + "protect", + "King", + "injuries", + "finally", + "nuclear", + "40", + "ratio", + "gun", + "Pakistan", + "Management", + "Air", + "ce", + "opposition", + "ment", + "ick", + "pro", + "act", + "platform", + "lack", + "pair", + "500", + "calling", + "ary", + "programs", + "scheduled", + "fast", + "joined", + "War", + "Editing", + "Since", + "Ryan", + "Mac", + "Big", + "Lake", + "digital", + "When", + "ue", + "assets", + "seeing", + "Act", + "partner", + "Board", + "beginning", + "supply", + "miles", + "prison", + "ons", + "Americans", + "ub", + "Or", + "me", + "benefits", + "benefit", + "measures", + "hear", + "parties", + "successful", + "Just", + "victim", + "block", + "limited", + "trip", + "People", + "serve", + "art", + "ism", + "wide", + "Sch", + "80", + "Thomas", + "90", + "stocks", + "girl", + "Asia", + "seeking", + "certainly", + "Services", + "College", + "communities", + "extra", + "2010", + "ness", + "holding", + "ous", + "tough", + "ade", + "mobile", + "owns", + "Do", + "Fire", + "spoke", + "returned", + "size", + "criminal", + "Instagram", + "offering", + "God", + "Service", + "page", + "her", + "deep", + "wood", + "crime", + "Sports", + "ile", + "Global", + "proposed", + "ain", + "session", + "Federal", + "Syria", + "ch", + "threat", + "allegations", + "Republicans", + "German", + "strategy", + "commercial", + "ING", + "Secretary", + "Q", + "reporters", + "100", + "Capital", + "Both", + "Post", + "Israel", + "save", + "ts", + "ill", + "drop", + "reserved", + "Many", + "avoid", + "200", + "iv", + "damage", + "condition", + "dropped", + "door", + "planning", + "ire", + "card", + "designed", + "reduce", + "AN", + "Un", + "ford", + "Then", + "pic", + "Copyright", + "rain", + "Martin", + "domestic", + "45", + "ge", + "murder", + "speech", + "line", + "helping", + "planned", + "feature", + "ud", + "type", + "ham", + "Public", + "ja", + "insurance", + "attacks", + "Corp", + "forecast", + "resources", + "ma", + "?\"", + "Am", + "Sept", + "push", + "attorney", + "23", + "emergency", + "winner", + "blood", + "north", + "Feb", + "baby", + "floor", + "spend", + "ex", + "dollars", + "unit", + "Hill", + "der", + "About", + "alone", + "ization", + "presidential", + "activity", + "THE", + "ee", + "ber", + "Other", + "owner", + "hour", + "cities", + "answer", + "ide", + "fully", + "ek", + "ists", + "coverage", + "vs", + "figure", + "population", + "org", + "snow", + "becoming", + "Sam", + "Carolina", + "join", + "profit", + "items", + "index", + "analysis", + "tournament", + "stake", + "perfect", + "way", + "band", + "girls", + "option", + "plays", + "oc", + "providing", + "ÃŃ", + "24", + "wouldn", + "ones", + "declined", + "written", + "voters", + "candidate", + "suspect", + "policies", + "peace", + "ast", + "particular", + "for", + "hopes", + "station", + "Most", + "speak", + "River", + "asking", + "statements", + "fifth", + "ha", + "Nigeria", + "af", + "explained", + "bar", + "housing", + "Santa", + "identified", + "simple", + "critical", + "Club", + "Security", + "Like", + "starts", + "art", + "street", + "reality", + "heavy", + "progress", + "showing", + "challenges", + "ban", + "committed", + "35", + "»", + "directly", + "aren", + "claim", + "Western", + "ind", + "gives", + "Saudi", + "choice", + "Th", + "approved", + "located", + "arrived", + "22", + "caught", + "professional", + "missed", + "culture", + "Year", + "Ohio", + "Ltd", + "Another", + "seem", + "believes", + "believed", + "character", + "Aug", + "red", + "fine", + "prior", + "thinking", + "http", + "+", + "zone", + "putting", + "crash", + "Australian", + "Ab", + "focused", + "REUTERS", + "Fox", + "Sp", + "traditional", + "analyst", + "wait", + "IT", + "request", + "ru", + "ians", + "ize", + "finish", + "laws", + "ran", + "ER", + "south", + "speed", + "movement", + "assault", + "exchange", + "appear", + "Sun", + "le", + "maybe", + "losing", + "subject", + "ive", + "mer", + "Business", + "Bl", + "appears", + "advantage", + "Lee", + "ada", + "Under", + "prevent", + "respect", + "sex", + "centre", + "Joe", + "ado", + "table", + "equipment", + "fair", + "tour", + "32", + "Financial", + "county", + "devices", + "customer", + "infrastructure", + "expectations", + "facing", + "upon", + "cross", + "Open", + "AL", + "quick", + "attempt", + "completed", + "facility", + "confidence", + "Supreme", + "piece", + "our", + "places", + "sometimes", + "poor", + "storm", + "hot", + "affected", + "na", + "abuse", + "Ms", + "word", + "over", + "brother", + "necessary", + "eventually", + "Star", + "send", + "boy", + "Rs", + "remember", + "21", + "climate", + "capacity", + "responsible", + "Matt", + "month", + "suffered", + "%.", + "og", + "Peter", + ",", + "feeling", + "ze", + "buying", + "oy", + "ij", + "bought", + "actions", + "owned", + "___", + "physical", + "specific", + "battle", + "Energy", + "picture", + "active", + "individuals", + "guy", + "regional", + "bond", + "ows", + "Toronto", + "rule", + "develop", + "crowd", + "guilty", + "female", + "selling", + "Follow", + "myself", + "ata", + "device", + "reasons", + "records", + "fighting", + "ON", + "ities", + "Home", + "status", + "plant", + "drugs", + "Church", + "completely", + "disease", + "highly", + "Paris", + "decade", + "owners", + "wall", + "camp", + "Steve", + "reporting", + "earned", + "Images", + "existing", + "Sen", + "concern", + "hundreds", + "song", + "knows", + "unique", + "lose", + "Kh", + "approximately", + "haven", + "park", + "independent", + "Although", + "Andrew", + "paper", + "developed", + "rising", + "direct", + "purchase", + "exactly", + "q", + "massive", + "box", + "champion", + "Clinton", + "voice", + "arrest", + "Korean", + "learning", + "Virginia", + "sa", + "par", + "chairman", + "agencies", + "healthy", + "Those", + "powerful", + "45", + "difference", + "Jackson", + "enforcement", + "dividend", + "qu", + "enjoy", + "ruling", + "ongoing", + "software", + "ks", + "location", + "mostly", + "candidates", + "men", + "broke", + "What", + "Br", + "2008", + "consumer", + "discuss", + "di", + "primary", + "En", + "green", + "concerned", + "image", + "Premier", + "Meanwhile", + "fired", + "Boston", + "ann", + "camera", + "traded", + "hasn", + "excited", + "increasing", + "Despite", + "citizens", + "euro", + "reportedly", + "minute", + "Will", + "LLC", + "sp", + "Michigan", + "stopped", + "eye", + "denied", + "modern", + "Wall", + "definitely", + "point", + "lines", + "politics", + "hotel", + "retail", + "stated", + "Over", + "grew", + "broadcast", + "legislation", + "fresh", + "bid", + "managed", + "society", + "scoring", + "Get", + "intelligence", + "holiday", + "governor", + "estimated", + "experts", + "Jeff", + "struck", + "hits", + "carry", + "placed", + "stores", + "expressed", + "valued", + "ad", + "twice", + "ala", + "display", + "usually", + "responded", + "dog", + "AS", + "Fed", + "2009", + "documents", + "normal", + "train", + "fl", + "shown", + "Ed", + "sort", + "allegedly", + "shots", + "ka", + "accounts", + "yesterday", + "creating", + "church", + "bus", + "award", + "equity", + "photos", + "33", + "fiscal", + "je", + "consumers", + "Manchester", + "no", + "Kevin", + "gain", + "corporate", + "civil", + "Middle", + "ally", + "sound", + "English", + "IC", + "winds", + "worst", + "Grand", + "effective", + "Island", + "drivers", + "fan", + "pe", + "sides", + "Go", + "clean", + "âĢĵ", + "television", + "Jr", + "allows", + "My", + "greater", + "ance", + "decisions", + "restaurant", + "Hospital", + "Tr", + "balance", + "mph", + "keeping", + "seconds", + "weapons", + "ert", + "pain", + "ass", + "steps", + "ger", + "Brexit", + "remaining", + "bringing", + "ure", + "weight", + "And", + "writing", + "Photo", + "Christian", + "ob", + "sport", + "figures", + "trust", + "skills", + "seat", + "faces", + "ck", + "born", + "super", + "fuel", + "del", + "meant", + "ica", + "justice", + "spring", + "killing", + "negative", + "Richard", + "und", + "factors", + "signs", + "learned", + "Game", + "audience", + "deliver", + "illegal", + "blue", + "screen", + "remained", + "announcement", + "IN", + "waiting", + "thanks", + "immigration", + "FBI", + "warned", + "measure", + "draw", + "positions", + "debut", + "Media", + "allowing", + "air", + "hen", + "mark", + "ys", + "prepared", + "Vegas", + "ep", + "ice", + "2018", + "defensive", + "60", + "Beach", + "pulled", + "£", + "lawyer", + "cast", + "solution", + "eyes", + "marketing", + "Foundation", + "risks", + "Today", + "za", + "draft", + "ice", + "26", + "Har", + "Executive", + "truck", + "ions", + "Your", + "Ireland", + "Jim", + "ha", + "fear", + "36", + "UR", + "Ford", + "watching", + "ien", + "style", + "Good", + "wearing", + "Houston", + "onto", + "boost", + "application", + "Dan", + "spread", + "Davis", + "strike", + "els", + "wind", + "interested", + "guard", + "mission", + "yourself", + "operation", + "larger", + "She", + "seasons", + "28", + "27", + "respond", + "ci", + "Centre", + "Our", + "names", + "flight", + "quarterback", + "standard", + "so", + "suggested", + "Mal", + "older", + "ini", + "perhaps", + "ont", + "Institute", + "millions", + "mental", + "ÃĤ", + "ga", + "clients", + "please", + "loan", + "aware", + "ft", + "int", + "75", + "05", + "AY", + "Out", + "hair", + "ied", + "seemed", + "ene", + "ty", + "NYSE", + "offensive", + "taxes", + "initial", + "ren", + "separate", + "la", + "Miami", + "AC", + "clearly", + "fit", + "Coast", + "firms", + "partners", + "upcoming", + "cold", + "proposal", + "AT", + "shut", + "Community", + "nature", + "Sal", + "bottom", + "ting", + "Click", + "nice", + "ets", + "hurt", + "itt", + "ama", + "carried", + "Con", + "rd", + "estate", + "Las", + "Law", + "ng", + "protection", + "produce", + "currency", + "happens", + "Per", + "ney", + "Long", + "fellow", + "cuts", + "reading", + "ano", + "proud", + "ost", + "UN", + "Arizona", + "AD", + "helps", + "winter", + "finding", + "Gold", + "att", + "Why", + "basketball", + "lin", + "Can", + "Bowl", + "ial", + "Alex", + "200", + "AM", + "presence", + "produced", + "developing", + "regarding", + "debate", + "vice", + "Italy", + "su", + "its", + "ator", + "34", + "complex", + "presented", + "researchers", + "slow", + "ya", + "sanctions", + "loved", + "seek", + "responsibility", + "admitted", + "album", + "solutions", + "facilities", + "ett", + "Gu", + "Well", + "lawmakers", + "miss", + "ful", + "Nick", + "'.", + "feels", + "prime", + "knowledge", + "deals", + "Taylor", + "survey", + "Francisco", + "joint", + "whom", + "sit", + "01", + "tr", + "organizations", + "Avenue", + "Their", + "Tim", + "rally", + "game", + "bigger", + "lawsuit", + "recorded", + "favorite", + "yard", + "transaction", + "qu", + "oh", + "interesting", + "inflation", + "ath", + "stuff", + "industrial", + "ico", + "TS", + "speaking", + "losses", + "ID", + "Stadium", + "stars", + "Women", + "Blue", + "wins", + "des", + "competitive", + "ters", + "pounds", + "direction", + "innings", + "Best", + "actor", + "dangerous", + "require", + "plus", + "solid", + "generation", + "strength", + "Mary", + "For", + "plenty", + "Team", + "influence", + "faced", + "es", + "Islamic", + "let", + "Development", + "path", + "youth", + "commitment", + "beautiful", + "Jack", + "ort", + "ten", + "attend", + "ars", + "ón", + "views", + "euros", + "author", + "core", + "supporters", + "iPhone", + "fashion", + "smaller", + "elected", + "university", + "picked", + "wa", + "ordered", + "Sc", + "Å", + "largely", + "+", + "Attorney", + "paying", + "AR", + "connection", + "setting", + "na", + "Rock", + "recovery", + "ew", + "serving", + "surprise", + "occurred", + "division", + "telling", + "margin", + "2020", + "sister", + "NBA", + "voted", + "con", + "By", + "49", + "foot", + "ü", + "Turkey", + "amazing", + "combined", + "appearance", + "easily", + "DAY", + "notes", + "Start", + "language", + "extremely", + "cloudy", + "Let", + "delivered", + "improved", + "collection", + "PM", + "estimates", + "boys", + "izing", + "text", + "closer", + "protest", + "province", + "shop", + "smart", + "de", + "Sheriff", + "EN", + "corner", + "panel", + "books", + "supported", + "mentioned", + "ver", + "Ministry", + "Prince", + "USA", + "receiving", + "choose", + "IN", + "Spain", + "section", + "considering", + "Cor", + "wish", + "welcome", + "Conference", + "ere", + "Officer", + "hoping", + "portfolio", + "standards", + "grand", + "Real", + "secure", + "Corporation", + "Rep", + "Kelly", + "streets", + "sitting", + "slightly", + "Investment", + "99", + "ond", + "units", + "votes", + "segment", + "championship", + "squad", + "iting", + "ron", + "®", + "em", + "touch", + "38", + "ceremony", + "decide", + "approval", + "So", + "Port", + "sub", + "sc", + "rep", + "Week", + "upper", + "agree", + "ny", + "matches", + "ics", + "tweeted", + "heat", + "Great", + "penalty", + "mass", + "alongside", + "herself", + "berg", + "science", + "entered", + "appeal", + "Pr", + "file", + "che", + "Report", + "Three", + "Northern", + "Jordan", + "amid", + "pace", + "jail", + "finance", + "Young", + "32", + "willing", + "conduct", + "Par", + "established", + "returns", + "aid", + "internet", + "IA", + "29", + "meetings", + "warning", + "Cl", + "campus", + "Most", + "Fund", + "William", + "Japanese", + "consensus", + "brain", + "!\"", + "poll", + "tech", + "trend", + "potentially", + "reduced", + "Show", + "37", + "happening", + "Brazil", + "pl", + "Cal", + "covered", + "enter", + "TV", + "catch", + "foot", + "union", + "expansion", + "Singapore", + "Detroit", + "attended", + "ats", + "newspaper", + "Division", + "news", + "cap", + "removed", + "48", + "Royal", + "window", + "parking", + "dark", + "standing", + "update", + "agent", + "transfer", + "Army", + "uses", + "80", + "Te", + "introduced", + "male", + "Southern", + "ratings", + "island", + "Miller", + "teachers", + "advice", + "familiar", + "uf", + "sought", + "por", + "Eric", + "da", + "ideas", + "uh", + "sixth", + "talent", + "Image", + "ering", + "run", + "ments", + "conducted", + "300", + "urged", + "discovered", + "pl", + "understanding", + "offense", + "secretary", + "sk", + "loans", + "Gr", + "applications", + "crude", + "go", + "Instead", + "opinion", + "doubt", + "ey", + "dis", + "31", + "experienced", + "leg", + "Cleveland", + "ven", + "failure", + "market", + "ack", + "decline", + "changing", + "300", + "defence", + "Brian", + "delivery", + "married", + "declared", + "pull", + "limit", + "MORE", + "defeat", + "expand", + "Colorado", + "Rob", + "iss", + "worse", + "perform", + "ising", + "2007", + "Del", + "surgery", + "easier", + "maintain", + "Ex", + "tied", + "east", + "user", + "ola", + "programme", + "manufacturing", + "hitting", + "x", + "skin", + "artist", + "tells", + "nearby", + "Daniel", + "Power", + "determined", + "actual", + "treated", + "lived", + "computer", + "cool", + "oo", + "Pl", + "effects", + "environmental", + "Morgan", + "flow", + "achieve", + "Bell", + "testing", + "Bob", + "whatever", + "Because", + "US", + "Hollywood", + "conflict", + "walking", + "Judge", + "Alabama", + "aircraft", + "te", + "well", + "goods", + "identify", + "associated", + "Ver", + "Education", + "airport", + "IL", + "falling", + "giant", + "Ma", + "Medical", + "ride", + "den", + "º", + "Jose", + "west", + "Pacific", + "visitors", + "Watch", + "Nations", + "gains", + "schedule", + "34", + "Exchange", + "payments", + "II", + "70", + "No", + "Syrian", + "Adam", + "ne", + "partnership", + "bl", + "Georgia", + "sites", + "models", + "degree", + "determine", + "Wilson", + "contest", + "professor", + "Chelsea", + "meaning", + "Games", + "Trust", + "Asian", + "33", + "link", + "Up", + "holds", + "Top", + "Italian", + "ord", + "Kansas", + "farmers", + "extended", + "birth", + "reform", + "relations", + "write", + "supporting", + "55", + "ita", + "notice", + "ster", + "animals", + "Jersey", + "arm", + "Foreign", + "Life", + "truly", + "Once", + "Mayor", + "Free", + "Agency", + "Wood", + "passing", + "DA", + "52", + "moves", + "com", + "house", + "Its", + "marijuana", + "ines", + "veteran", + "variety", + "ki", + "ff", + "amb", + "listed", + "pushed", + "volume", + "increasingly", + "kick", + "rock", + "ank", + "fees", + "enable", + "images", + "truth", + "ministry", + "rare", + "Dallas", + "Minnesota", + "contributed", + "Charles", + "percentage", + "technical", + "App", + "assistant", + "interests", + "immediate", + "38", + "Town", + "closing", + "Anthony", + "southern", + "ase", + "Putin", + "Force", + "ba", + "refused", + "Still", + "ix", + "Col", + "materials", + "structure", + "driven", + "patient", + "broken", + "radio", + "scale", + "replace", + "39", + "Land", + "deputy", + "und", + "color", + "OS", + "roads", + "corruption", + "Rose", + "employee", + "Water", + "seats", + "walked", + "ec", + "cents", + "chain", + "payment", + "Android", + "eb", + "commission", + "throw", + "count", + "accident", + "expensive", + "ered", + "Yes", + "Louis", + "studies", + "investigating", + "century", + "discussion", + "inter", + "DAQ", + "Before", + "initially", + "*", + "investments", + "multi", + "tight", + "confident", + "counter", + "Qu", + "governments", + "armed", + "suit", + "row", + "locations", + "episode", + "itch", + "younger", + "festival", + "pitch", + "OF", + "talked", + "ca", + "protests", + "targets", + "90", + "originally", + "singer", + "journey", + "ug", + "apply", + "teacher", + "chances", + "):", + "deaths", + "isation", + "Stephen", + "code", + "Championship", + "Jason", + "AT", + "accept", + "Series", + "values", + "bed", + "Harry", + "flat", + "tools", + "publicly", + "37", + "pointed", + "Golden", + "ps", + "unable", + "ants", + "estimate", + "warm", + "basic", + "ern", + "raising", + "Related", + "ultimately", + "northern", + "plane", + "Vice", + "Raj", + "Justin", + "anc", + "brings", + "Art", + "OT", + "shift", + "BBC", + "Su", + "BS", + "bag", + "doctor", + "fill", + "downtown", + "possibility", + "Ag", + "est", + "44", + "struggling", + "linked", + "tickets", + "Jay", + "Call", + "stands", + "wedding", + "resident", + "eng", + "leads", + "advance", + "Atlanta", + "tie", + "advanced", + "pt", + "burg", + "Earlier", + "Sw", + "Zealand", + "exercise", + "AM", + "affect", + "possession", + "involving", + "42", + "writer", + "Beijing", + "doctors", + "obviously", + "er", + "Olympic", + "75", + "Khan", + "Fort", + "app", + "like", + "sea", + "ock", + "mix", + "Iraq", + "Muslim", + "Finally", + "continuing", + "pr", + "Ke", + "Joseph", + "expects", + "institutions", + "conservative", + "own", + "Chairman", + "returning", + ".-", + "stood", + "vision", + "ess", + "adults", + "yield", + "prove", + "orders", + "dream", + "36", + "related", + "sl", + "everybody", + "ui", + "represents", + "discussed", + "becomes", + "village", + "CC", + "negotiations", + "Philadelphia", + "celebrate", + "farm", + "ç", + "registered", + "Governor", + "OL", + "Mon", + "filing", + "04", + "SE", + "Assembly", + "actress", + "si", + "thank", + "heading", + "Who", + "famous", + "consecutive", + "marriage", + "ette", + "NAS", + "acks", + "Please", + "Diego", + "baseball", + "Moore", + "ties", + "carrying", + "que", + "turning", + "McC", + "Ken", + "OR", + "Stock", + "buildings", + "49", + "Van", + "39", + "Seattle", + "wild", + "crew", + "route", + "Time", + "tonight", + "moments", + "videos", + "internal", + "Liverpool", + "port", + "chair", + "rival", + "Scotland", + "round", + "ith", + "breaking", + "voting", + "ically", + "producer", + "Love", + "remove", + "PA", + "asset", + "requires", + "signing", + "ages", + "impressive", + "Irish", + "authority", + "ruled", + "aimed", + "captain", + "AG", + "plants", + "Anderson", + "Spanish", + "banking", + "threats", + "suspended", + "tests", + "religious", + "electric", + "READ", + "strategic", + "split", + "ex", + "practices", + "Israeli", + "Arabia", + "Moscow", + "franchise", + "custody", + "Old", + "requirements", + "quarterly", + "comfortable", + "crimes", + "headed", + "newsletter", + "animal", + "regulations", + "long", + "CNN", + "assists", + "shopping", + "Gov", + "Securities", + "assistance", + "nor", + "relatively", + "increases", + "generally", + "55", + "gained", + "41", + "pictures", + "gan", + "pop", + "updates", + "Republic", + "rebounds", + "Patrick", + "relief", + "acting", + "Festival", + "2006", + "boss", + "types", + "65", + "Yet", + "purpose", + "ning", + "matters", + "compete", + "ball", + "Ram", + "sw", + "Following", + "Bush", + "troops", + "supposed", + "freedom", + "featured", + "storage", + "Information", + "Hong", + "golf", + "agents", + "fraud", + "minimum", + "artists", + "eat", + "high", + "Former", + "Kong", + "Josh", + "Delhi", + "showers", + "Academy", + "apartment", + "van", + "fish", + "oe", + "films", + "Bo", + "edge", + "possibly", + "tweet", + "09", + "resolution", + "jo", + "kill", + "44", + "cell", + "scheme", + "th", + "bonds", + "entry", + "secret", + "43", + "ending", + "weren", + "Credit", + "Live", + "retired", + "machine", + "summit", + "sharing", + "acquired", + "era", + "wear", + "ical", + "07", + "exciting", + "li", + "BC", + "Social", + "historic", + "Che", + "Lewis", + "ira", + "stolen", + "Speaking", + "sleep", + "spokeswoman", + "week", + "purchased", + "importance", + "EC", + "ends", + "dress", + "parliament", + "Cruz", + "cards", + "hi", + "Email", + "represent", + "brands", + "Senior", + "participants", + "fly", + "identity", + "Ham", + "Sky", + "ij", + "SA", + "promised", + "trouble", + "suffering", + "leaves", + "suggest", + "Sh", + "busy", + "properties", + "worldwide", + "cloud", + "SEC", + "closely", + "manage", + "numerous", + "background", + "Express", + "65", + "Tony", + "Madrid", + "ev", + "der", + "significantly", + "alternative", + "ship", + "head", + "ators", + "dinner", + "ax", + "SC", + "criticism", + "Mah", + "Min", + "rie", + "Tour", + "bench", + "adds", + "seriously", + "star", + "Journal", + "Di", + "ali", + "sentence", + "Several", + "mayor", + "ati", + "suggests", + "behavior", + "stronger", + "Food", + "client", + "not", + "Price", + "targeted", + "Singh", + "Network", + "prosecutors", + "directed", + "Democrat", + "bl", + "ues", + "Family", + "connected", + "Champions", + "roughly", + "absolutely", + "08", + "passengers", + "ö", + "Special", + "coast", + "complaint", + "400", + "Em", + "ves", + "dogs", + "handle", + "otherwise", + "sees", + "ticket", + "Award", + "All", + "task", + "songs", + "Among", + "dedicated", + "steel", + "looking", + "shortly", + "tackle", + "ative", + "minor", + "â", + "provider", + "vers", + "use", + "ives", + "typically", + "arms", + "Ant", + "IS", + "jump", + "©", + "47", + "aff", + "monthly", + "Microsoft", + "CBS", + "threatened", + "honor", + "Mo", + "42", + "inning", + "pool", + "healthcare", + "Story", + "Tennessee", + "promote", + "EL", + "emotional", + "pe", + "factor", + "investigators", + "Ľ", + "Back", + "Project", + "cu", + "side", + "messages", + "TH", + "eg", + "experiences", + "causing", + "joining", + "package", + "bodies", + "lots", + "Harris", + "cl", + "Internet", + "free", + "performed", + "pieces", + "buy", + "caption", + "web", + "contracts", + "At", + "attempted", + "unlikely", + "click", + "invest", + "IM", + "View", + "neighborhood", + "ring", + "Four", + "ail", + "46", + "One", + "native", + "CH", + "OM", + "alcohol", + "Val", + "characters", + "Pat", + "politicians", + "Mag", + "begins", + "Ak", + "los", + "personnel", + "enjoyed", + "Technology", + "sun", + "IT", + "document", + "deficit", + "coalition", + "memory", + "pushing", + "any", + "ified", + "founder", + "2000", + "2017", + "visited", + "Though", + "ph", + "soft", + "flag", + "mom", + "inch", + "Samsung", + "apps", + "touchdown", + "Care", + "Mrs", + "redistributed", + "encourage", + "ched", + "tend", + "regions", + "pp", + "IP", + "br", + "ush", + "argued", + "junior", + "BA", + "severe", + "NIGHT", + "def", + "surrounding", + "48", + "engine", + "filled", + "seventh", + "battery", + "Allen", + "guidance", + "roll", + "rural", + "expert", + "convicted", + "likes", + "Ro", + "grown", + "retirement", + "intended", + "mis", + "army", + "dance", + "Thank", + "ent", + "outlook", + "para", + "dry", + "TO", + "era", + "waste", + "faster", + "Eagles", + "TA", + "Frank", + "Ã", + "LE", + "ura", + "ko", + "ao", + "distribution", + "improvement", + "playoff", + "acquisition", + "CH", + "tomorrow", + "struggle", + "Human", + "newly", + "oon", + "Ne", + "con", + "sc", + "unless", + "transition", + "ten", + "Inter", + "equal", + "rec", + "appointed", + "wake", + "Earth", + "ose", + "Eastern", + "soldiers", + "Parliament", + "sets", + "attempts", + "Illinois", + "revenues", + "Wil", + "heads", + "prepare", + "priority", + "PS", + "Jo", + "NBC", + "therefore", + "yn", + "initiative", + "ct", + "coffee", + "Fair", + "43", + "den", + "form", + "ova", + "appropriate", + "Play", + "accepted", + "creative", + "follows", + "rescue", + "tree", + "With", + "Netflix", + "Football", + "surprised", + "lowest", + "800", + "amp", + "worried", + "mar", + "ran", + "visiting", + "selected", + "Music", + "Ann", + "explain", + "ging", + "widely", + "square", + "trends", + "improving", + "Head", + "Queen", + "Society", + "cutting", + "GOP", + "03", + "',", + "ET", + "Drive", + "oll", + "ato", + "Sea", + "jury", + "Rights", + "investor", + "ABC", + "tool", + "Are", + "rejected", + "emerging", + "counts", + "nations", + "false", + "treat", + "va", + "weak", + "Highway", + "down", + "struggled", + "MP", + "guests", + "gender", + "houses", + "rit", + "Wild", + "streak", + "uc", + "Reserve", + "Ratings", + "alt", + "greatest", + "lawyers", + "reaching", + "temperatures", + "To", + "outstanding", + "passes", + "faith", + "inc", + "cr", + "informed", + "oz", + "trees", + "sending", + "150", + "bo", + "wine", + "ros", + "suspected", + "repeatedly", + "hat", + "shape", + "Wh", + "assist", + "stress", + "feed", + "ark", + "ored", + "watched", + "incredible", + "cl", + "nt", + "entertainment", + "ih", + "beauty", + "bi", + "Local", + "sat", + "41", + "broad", + "heavily", + "engaged", + "specifically", + "Men", + "Ross", + "2005", + "ST", + "95", + "download", + "400", + "sentenced", + "Catholic", + "Oklahoma", + "threw", + "worry", + "imp", + "drove", + "colleagues", + "agenda", + "64", + "Each", + "fee", + "New", + "ium", + "spokesperson", + "bills", + "47", + "Afghanistan", + "invited", + "YouTube", + "anniversary", + "dozen", + "ram", + "Only", + "employment", + "Getty", + "gap", + "sweet", + "Little", + "inf", + "ying", + "glass", + "classes", + "coal", + "Sub", + "duty", + "CA", + "coaches", + "Â", + "anna", + "Sk", + "46", + "ison", + "ille", + "ST", + "ric", + "participate", + "equ", + "rich", + "respectively", + "expenses", + "combination", + "right", + "shareholders", + "turns", + "earn", + "51", + "ured", + "drink", + "Kar", + "Shares", + "Mid", + "Getty", + "bridge", + "lo", + "inspired", + "surface", + "gift", + "ence", + "challenging", + "offices", + "suspects", + "Finance", + "ab", + "bound", + "momentum", + "backed", + "parent", + "crucial", + "ave", + "dealing", + "regulatory", + "apparently", + "Mat", + "apart", + "port", + "ole", + "beach", + "cultural", + "institutional", + "beating", + "Iowa", + "Ali", + "67", + "je", + "ays", + "weekly", + "birthday", + "pipeline", + "knee", + "solar", + "Pe", + "category", + "Area", + "ky", + "ures", + "06", + "Ball", + "semi", + "Hamilton", + "hip", + "Ph", + "Next", + "athletes", + "ii", + "movies", + "han", + "net", + "plastic", + "behalf", + "gen", + "findings", + "stretch", + "Sa", + "officially", + "Sarah", + "privacy", + "Mad", + "none", + "gh", + "On", + "drama", + "Fl", + "ika", + "Arsenal", + "violent", + "UN", + "called", + "59", + "hate", + "relationships", + "granted", + "Jon", + "listen", + "season", + "fewer", + "GA", + "Labour", + "remarks", + "Jonathan", + "Ros", + "sey", + "Ontario", + "Thompson", + "Night", + "ranked", + "Ukraine", + "immigrants", + "degrees", + "Ge", + "labor", + "umb", + "YORK", + "allies", + "sp", + "hed", + "sw", + "tariffs", + "SP", + "classic", + "awards", + "ents", + "fix", + "soccer", + "concert", + "ust", + "adult", + "output", + "managing", + "02", + "promise", + "awareness", + "gross", + "entering", + "po", + "oj", + "metal", + "exit", + "excellent", + "clubs", + "hold", + "replaced", + "Class", + "scientists", + "primarily", + "Mer", + "ão", + "circumstances", + "ades", + "supplies", + "aker", + "Sand", + "scandal", + "settlement", + "Wisconsin", + "Warriors", + "Austin", + "journalists", + "ening", + "reflect", + "Buy", + "Awards", + "selection", + "Bel", + "bury", + "technologies", + "%,", + "ime", + "Ä", + "Administration", + "channel", + "Star", + "transport", + "awarded", + "ena", + "motor", + "orn", + "kin", + "featuring", + "phones", + "AND", + "relevant", + "See", + "winners", + "dad", + "Source", + "Check", + "aut", + "Far", + "opponents", + "outcome", + "doors", + "suicide", + "ima", + "jumped", + "perspective", + "transportation", + "thinks", + "Mor", + "deadline", + "53", + "Deputy", + "ery", + "detailed", + "uch", + "Bur", + "trades", + "Greg", + "zero", + "erson", + "Children", + "du", + "66", + "mixed", + "Barack", + "54", + "territory", + "ac", + "concept", + "Add", + "ourselves", + "reaction", + "Sydney", + "ink", + "consistent", + "boat", + "room", + "dozens", + "effectively", + "but", + "motion", + "alive", + "Key", + "weight", + "exports", + "operate", + "regime", + "Authority", + "och", + "CR", + "leg", + "forget", + "American", + "bs", + "thoughts", + "Sign", + "Patriots", + "brief", + "Oregon", + "Bal", + "mine", + "citing", + "magazine", + "more", + "ERS", + "Ber", + "ua", + "ox", + "Main", + "instance", + "tr", + "restaurants", + "ora", + "harassment", + "\",\"", + "Ł", + "silver", + "Mueller", + "Senator", + "Every", + "footage", + "ms", + "opposed", + "Link", + "ver", + "pleased", + "ame", + "ending", + "rivals", + "ida", + "ike", + "ta", + "Cook", + "headquarters", + "ear", + "aggressive", + "courts", + "Museum", + "im", + "Holdings", + "communication", + "phase", + "yl", + "powers", + "proved", + "carbon", + "aside", + "Olympics", + "gathered", + "Pennsylvania", + "smartphone", + "Met", + "Hurricane", + "protected", + "communications", + "emerged", + "aim", + "stable", + "ides", + "GB", + "entirely", + "missile", + "Gen", + "unclear", + "electricity", + "ology", + "away", + "license", + "Pittsburgh", + "cameras", + "musical", + "managers", + "57", + "scores", + "profile", + "hel", + "¼", + "shouldn", + "RA", + ");", + "permanent", + "ome", + "et", + "mar", + "favor", + "maker", + "discussions", + "ory", + "sharp", + "pleaded", + "passenger", + "quarter", + "dem", + "versus", + "mainly", + "eighth", + "Airport", + "Cross", + "million", + "Nas", + "cited", + "56", + "yes", + "Below", + "arn", + "Turkish", + "Sl", + "stepped", + "producers", + "overnight", + "sounds", + "52", + "64", + "54", + "58", + "Clark", + "Rick", + "gr", + "Mont", + "beer", + "une", + "reporter", + "charity", + "eating", + "extend", + "guess", + "NA", + "hedge", + "encouraged", + "owned", + "Mel", + "Kentucky", + "ace", + "lineup", + "hosts", + "capable", + "PR", + "Arts", + "controversial", + "hosted", + "ries", + "roster", + "fixed", + "Walker", + "ged", + "disaster", + "dispute", + "Denver", + "Trade", + "ute", + "ese", + "cy", + "grant", + "Max", + "distance", + "isc", + "editor", + "Dave", + "performances", + "lay", + "vulnerable", + "Murray", + "âĤ¬", + "mining", + "2004", + "level", + "ability", + "auto", + "fake", + "attacked", + "ona", + "ups", + "ened", + "fallen", + "stations", + "Contact", + "itz", + "incidents", + "complaints", + "operates", + "refugees", + "essential", + "Test", + "demands", + "roles", + "yr", + "acts", + "usual", + "ring", + "handed", + "Matthew", + "hour", + "industries", + "shoot", + "Authorities", + "probe", + "Utah", + "RBI", + "AD", + "prospect", + "outs", + "Uber", + "bright", + "mention", + "savings", + "Miss", + "ONDON", + "1990", + "arm", + "Ten", + "These", + "explains", + "minute", + "85", + "maximum", + "ro", + "rookie", + "studio", + "Cam", + "Gal", + "defend", + "hand", + "53", + "Oil", + "serves", + "sn", + "ios", + "Defense", + "AB", + "hired", + "supports", + "premium", + "ef", + "failing", + "Indiana", + "exp", + "objective", + "affordable", + "Com", + "Thanks", + "anywhere", + "confirm", + "ited", + "representing", + "witness", + "69", + "claiming", + "violation", + "historical", + "med", + "preparing", + "Tech", + "posts", + "OC", + "Graham", + "Gl", + "Lions", + "ales", + "ID", + "correct", + "Antonio", + "advertising", + "eastern", + "OW", + "holdings", + "polls", + "SH", + "executives", + "Jewish", + "Gary", + "prize", + "Commissioner", + "cells", + "ify", + "lunch", + "democracy", + "Er", + "regularly", + "resulted", + "Ave", + "Partners", + "rewritten", + "lo", + "cooperation", + "Gulf", + "smoke", + "Memorial", + "wave", + "fears", + "kid", + "Giants", + "recovered", + "row", + "Radio", + "Barcelona", + "wonderful", + "Dow", + "stream", + "Simon", + "detail", + "volunteers", + "Ind", + "forms", + "mann", + "Ray", + "oor", + "Take", + "represented", + "het", + "blow", + "aged", + "RE", + "Missouri", + "covering", + "profits", + "concluded", + "thus", + "Columbia", + "ode", + "Zimbabwe", + "disclosed", + "lifted", + "Sean", + "Harvey", + "Plus", + "ces", + "Greece", + "Lady", + "delay", + "kitchen", + "Index", + "bear", + "puts", + "new", + "88", + "Ash", + "Å¡", + "performing", + "law", + "Part", + "indicated", + "announce", + "compensation", + "ka", + "Science", + "ris", + "recommendations", + "Second", + "lights", + "temporary", + "urs", + "western", + "stone", + "68", + "Disney", + "playoffs", + "judges", + "engineering", + "Pen", + "Pal", + "obvious", + "Bridge", + "End", + "Arab", + "except", + "hole", + "class", + "causes", + "connect", + "AI", + "An", + "chose", + "Elizabeth", + "min", + "proper", + "NHL", + "races", + "innovation", + "sugar", + "600", + "Modi", + "illa", + "trillion", + "Sar", + "Affairs", + "impossible", + "guide", + "captured", + "Sales", + "species", + "51", + "ar", + "master", + "stayed", + "iro", + "Economic", + "vast", + "ili", + "pet", + "ye", + "77", + "keeps", + "Phil", + "EPS", + "Regional", + "sectors", + "desire", + "Stanley", + "¾", + "unknown", + "pot", + "PR", + "knowing", + "flying", + "Treasury", + "iers", + "enn", + "ably", + "sick", + "manner", + "manufacturers", + "champions", + "gy", + "Part", + "ister", + "Mountain", + "imagine", + "portion", + "Camp", + "chemical", + "ible", + "Analy", + "Bureau", + "pm", + "updated", + "etc", + "Field", + "iles", + "obtained", + "stick", + "cat", + "har", + "marked", + "medium", + "Des", + "People", + "wealth", + "ores", + "Baltimore", + "tip", + "dismissed", + "Victoria", + "Brad", + "Ch", + "56", + "stadium", + "eth", + "thunder", + "tested", + "drawn", + "counsel", + "ld", + "spirit", + "uss", + "theme", + "my", + "necessarily", + "elements", + "collected", + "Res", + "Maryland", + "Enter", + "founded", + "ae", + "pilot", + "shoulder", + "PC", + "argument", + "yen", + "receiver", + "harm", + "ET", + "protesters", + "72", + "Aaron", + "ed", + "expecting", + "\":\"", + "bike", + "Äĩ", + "luxury", + "half", + "Barbara", + "foundation", + "ill", + "submitted", + "deeply", + "hospitals", + "BJP", + "shock", + "platforms", + "summary", + "Where", + "celebration", + "iff", + "veterans", + "achieved", + "fl", + "activists", + "Manager", + "formal", + "formed", + "investigate", + "Kyle", + ":", + "Ra", + "ovic", + "drinking", + "networks", + "Alexander", + "Os", + ")", + "bomb", + "recalled", + "ito", + "ient", + "representatives", + "Christ", + "Way", + "deadly", + "investing", + "Russell", + "consumption", + "harder", + "bail", + "critics", + "danger", + "drew", + "Sol", + "copyright", + "Henry", + "buyers", + "residential", + "maintenance", + "pr", + "marks", + "ages", + "covers", + "ton", + "titles", + "PS", + "Evans", + "migrants", + "flights", + "monitoring", + "addressed", + "vital", + "controlled", + "weapon", + "inches", + "reduction", + "urban", + "coaching", + "reducing", + "ila", + "realize", + "meat", + "ref", + "overseas", + "blame", + "terrorist", + "stuck", + "Us", + "esh", + "pro", + "58", + "ough", + "exposure", + "Abu", + "state", + "providers", + "fore", + "jet", + "bar", + "ownership", + "ret", + "upset", + "facts", + "purchasing", + "reforms", + "river", + "somebody", + "guest", + "iy", + "auction", + "Reading", + "consequences", + "representative", + "appointment", + "add", + "collaboration", + "Tesla", + "Cohen", + "engagement", + "speaks", + "EST", + "exposed", + "maintained", + "rs", + "dating", + "Program", + "board", + "racing", + "pension", + "ign", + "iti", + "Five", + "extensive", + "Ha", + "Point", + "Mexican", + "expanded", + "totally", + "investigations", + "Orleans", + "cycle", + "ESPN", + "ifying", + "cup", + "Az", + "Investors", + "engage", + "reg", + "fought", + "terrorism", + "blocked", + "OK", + "Äį", + "72", + "destroyed", + "«", + "staying", + "afford", + "appearances", + "Hills", + "crore", + "strategies", + "tips", + "Sm", + "Fr", + "banned", + "Son", + "ask", + "limits", + "recognition", + "eligible", + "Gar", + "volatility", + "laid", + "nes", + "grade", + "RE", + "Hart", + "57", + "oma", + "uncertainty", + "recognized", + "PC", + "chosen", + "uz", + "adviser", + "una", + "assessment", + "reveal", + "mo", + "After", + "Bro", + "Off", + "peak", + "referred", + "SC", + "2003", + "ification", + "shutdown", + "Officials", + "ias", + "extreme", + "flood", + "hockey", + "wage", + "Net", + "damaged", + "replacement", + "Maria", + "creation", + "guns", + "aci", + "worker", + "do", + "viewers", + "seed", + "sts", + "touchdowns", + "mistake", + "ray", + "ull", + "pricing", + "strongly", + "aims", + "Navy", + "Egypt", + "ker", + "ve", + "Steven", + "res", + "ational", + "requests", + "emissions", + "Arena", + "uma", + "Atlantic", + "hr", + "AFP", + "Square", + "contribute", + "function", + "dec", + "Nelson", + "89", + "referendum", + "Pre", + "applied", + "GMT", + "Iranian", + "Nigerian", + "Any", + "NG", + "acknowledged", + "referring", + "venture", + "imports", + "blog", + "futures", + "OU", + "UFC", + "neither", + "extension", + "hes", + "Med", + "76", + "sustainable", + "ains", + "reputation", + "Vancouver", + "basically", + "acy", + "sad", + "Francis", + "Kennedy", + "Nevada", + "Lu", + "ras", + "Av", + "rear", + "Ho", + "properly", + "abe", + "Hotel", + "opinions", + "under", + "Station", + "FOR", + "ops", + "adopted", + "Swiss", + "Country", + "Ter", + "Andy", + "Me", + "Cooper", + "Tigers", + "Creek", + "gay", + "iner", + "AN", + "bird", + "lla", + "Kate", + "Pet", + "ni", + "prospects", + "ater", + "ites", + "escape", + "lam", + "ake", + "1980", + "Lag", + "successfully", + "districts", + "ministers", + "aries", + "frame", + "ON", + "Euro", + "Markets", + "register", + "defeated", + "developments", + "ninth", + "quiet", + "generated", + "valuable", + "recommended", + "Theatre", + "Cap", + "bed", + "reference", + "ease", + "oring", + "66", + "improvements", + "elsewhere", + "Hillary", + "defender", + "Right", + "zy", + "comprehensive", + "spotted", + "Oakland", + "Ok", + "System", + "ique", + "persons", + "exist", + "broader", + "clinical", + "2001", + "oul", + "securities", + "ghan", + "shelter", + "ero", + "ATED", + "hosting", + "select", + "Kavanaugh", + "restrictions", + "osa", + "yields", + "LA", + "59", + "wonder", + "absence", + "ür", + "ÅĤ", + "DP", + "electronic", + "illegally", + "micro", + "NEW", + "hall", + "aged", + "temperature", + "cast", + "atic", + "legacy", + "affairs", + "ji", + "Resources", + "gang", + "winning", + "attending", + "aro", + "friendly", + "aine", + "cannabis", + "airline", + "noting", + "professionals", + "FREE", + "RC", + "financing", + "independence", + "ved", + "resulting", + "steady", + "Winter", + "uring", + "hoped", + "98", + "presentation", + "aya", + "rated", + "osh", + "Analysis", + "=", + "donations", + "IR", + "combat", + "Howard", + "anda", + "79", + "invested", + "expanding", + "omb", + "ress", + "ble", + "journalist", + "Woods", + "centers", + "ott", + "streaming", + "terror", + "sustained", + "WWE", + "pre", + "ÅŁ", + "ait", + "arrival", + "residence", + "extent", + "arrive", + "2002", + "establish", + "74", + "Argentina", + "Dem", + "inn", + "aud", + "NCAA", + "questioned", + "ballot", + "min", + "landscape", + "horse", + "opponent", + "iel", + "prompted", + "atory", + "lift", + "association", + "cher", + "defending", + "tiny", + "poverty", + "Safety", + "petition", + "Limited", + "CA", + "FC", + "Ãł", + "oni", + "monitor", + "ÃŃa", + "MA", + "answers", + "Mitchell", + "bo", + "Shah", + "sm", + "medal", + "Civil", + "recognize", + "key", + "pregnant", + "spots", + "ante", + "academic", + "initiatives", + "secured", + "CL", + "ils", + "anticipated", + "involvement", + "Make", + "insisted", + "Wales", + "clothing", + "tracks", + "symptoms", + "plate", + "NY", + "retailers", + "Pan", + "fled", + "quoted", + "saved", + "Carter", + "teaching", + "Tokyo", + "Cr", + "Six", + "Picture", + "recover", + "comedy", + "ree", + "strikes", + "Sanders", + "sel", + "graduate", + "pending", + "St", + "warrant", + "honest", + "GM", + "noticed", + "Galaxy", + "ider", + "proposals", + "wore", + "indeed", + "EM", + "Channel", + "ances", + "Brady", + "86", + "gotten", + "throwing", + "Leader", + "Video", + "71", + "welcomed", + "NEW", + "fairly", + "promises", + "Silver", + "rape", + "opener", + "ares", + "Sir", + "making", + "cur", + "rooms", + "73", + "amounts", + "Industry", + "Dar", + "62", + "ted", + "abroad", + "Maybe", + "readers", + "oke", + "publication", + "Jean", + "operator", + "Having", + "Mil", + "life", + "generate", + "Craig", + "Mass", + "Bh", + "requested", + "crazy", + "Space", + "copy", + "export", + "context", + "br", + "62", + "Robinson", + "cyber", + "ENT", + "BI", + "arg", + "speaker", + "dramatic", + "Ol", + "Mill", + "trained", + "editing", + "salary", + "directors", + "explore", + "lucky", + "prominent", + "brothers", + "neck", + "icht", + "Watson", + "born", + "proven", + "principal", + "edition", + "Ed", + "switch", + "maker", + "relative", + "mi", + "Bruce", + "ho", + "Scottish", + "water", + "Sport", + "Kings", + "Collins", + "adi", + "celebrated", + "clothes", + "sunny", + "Charlotte", + "ees", + "scenes", + "Data", + "wounded", + "unusual", + "realized", + "Plan", + "Trans", + "FC", + "letters", + "alerts", + "Warren", + "DS", + "oss", + "pping", + "suspension", + "benchmark", + "Acc", + "alert", + "passion", + "Est", + "latter", + "stability", + "arts", + "pursue", + "Season", + "fields", + "method", + "63", + "folks", + "exclusive", + "crews", + "sessions", + "Major", + "Mount", + "map", + "=", + "situations", + "Berlin", + "rey", + "dates", + "sheet", + "Lo", + "fighters", + "Mart", + "atmosphere", + "illness", + "competing", + "Christopher", + "Roy", + "mm", + "iano", + "ge", + "Rams", + "conversations", + "Pa", + "Tel", + "appreciate", + "78", + "Total", + "low", + "Stone", + "opposite", + "barrel", + "developers", + "express", + "highs", + "which", + "par", + "Vietnam", + "blocks", + "recording", + "adjusted", + "ret", + "AR", + "militants", + "innovative", + "Ghana", + "FR", + "fantastic", + "mortgage", + "ando", + "Lane", + "ises", + "Â", + "homeless", + "Kal", + "approached", + "rounds", + "margins", + "ament", + "Motor", + "encouraging", + "ÂŃ", + "uru", + "handling", + "Massachusetts", + "planet", + "Spring", + "Bon", + "gu", + "Beat", + "drawing", + "Phoenix", + "very", + "aid", + "Ste", + "Entertainment", + "Ron", + "assigned", + "SA", + "News", + "interviews", + "Oh", + "media", + "vel", + "permission", + "transactions", + "traders", + "solo", + "provincial", + "suggesting", + "¡", + "diverse", + "67", + "ranks", + "Fre", + "favourite", + "63", + "differences", + "targeting", + "actors", + "76", + "icated", + "collect", + "akes", + "war", + "contained", + "ches", + "library", + "segments", + "Line", + "ê", + "ual", + "bags", + "factory", + "ear", + "somewhat", + "rail", + "UP", + "ula", + "Niger", + "las", + "implementation", + "emails", + "kel", + "wing", + "advised", + "--", + "istic", + "depth", + "shoes", + "Jennifer", + "venue", + "contain", + "highlights", + "capabilities", + "processes", + "tradition", + "contacted", + "producing", + "trail", + "rem", + "600", + "68", + "AA", + "Ba", + "Such", + "Tyler", + "ipp", + "survived", + "ami", + "Continue", + "capture", + "bi", + "61", + "96", + "threatening", + "keen", + "dale", + "trailer", + "stages", + "Gordon", + "finishing", + "legislative", + "useful", + "Greek", + "ald", + "grounds", + "Du", + "storms", + "ills", + "expense", + "detained", + "Today", + "diet", + "wood", + "Cameron", + "thrown", + "cricket", + "ideal", + "with", + "teammates", + "ours", + "projected", + "personally", + "Boy", + "rom", + "Philippines", + "win", + "ges", + "counties", + "Baker", + "prosecutor", + "roof", + "met", + "partly", + "Moon", + "eman", + "focusing", + "fishing", + "than", + "Jeremy", + "Bad", + "ais", + "controls", + "tonnes", + "shall", + "61", + "gathering", + "ERA", + "presidency", + "85", + "Gas", + "scenario", + "quarters", + "ang", + "settled", + "Commerce", + "anybody", + "garden", + "Library", + "bet", + "topic", + "olo", + "intense", + "87", + "links", + "med", + "AG", + "flooding", + "Murphy", + "PM", + "finds", + "sensitive", + "pped", + "completion", + "minority", + "von", + "striking", + "rich", + "bars", + "efficient", + "contributions", + "visits", + "attract", + "Malaysia", + "REL", + "opens", + "essentially", + "reasonable", + "sentiment", + "Melbourne", + "fitness", + "frequently", + "Rangers", + "museum", + "DNA", + "contrast", + "Adams", + "Win", + "falls", + "imposed", + "250", + "ood", + "Rio", + "choices", + "yellow", + "rin", + "ben", + "Staff", + "Indonesia", + "carries", + "tourism", + "UM", + "Orange", + "sell", + "resolve", + "Mumbai", + "pan", + "implement", + "midfielder", + "OP", + "tensions", + "800", + "Lord", + "Light", + "lies", + "és", + "participation", + "tries", + "sheriff", + "degree", + "congressional", + "mode", + "regulation", + "Jacob", + "Crown", + "bowl", + "Mississippi", + "theft", + "Kingdom", + "resort", + "royal", + "unemployment", + "PP", + "nomination", + "TR", + "behaviour", + "bank", + "Forest", + "WASHINGTON", + "Others", + "slowly", + "menu", + "vo", + "Sy", + "Metro", + "Lisa", + "registration", + "While", + "Jesus", + "250", + "processing", + "monetary", + "ape", + "ener", + "Systems", + "disappointed", + "print", + "uy", + "ħ", + "demanding", + "incredibly", + "play", + "surveillance", + "Standard", + "periods", + "writes", + "Luke", + "Palestinian", + "walks", + "riding", + "waters", + "Sox", + "traveling", + "tap", + "organized", + "resource", + "angry", + "timing", + "empty", + "milk", + "therapy", + "Brandon", + "mon", + "nationwide", + "novel", + "Storm", + "iet", + "Bre", + "begun", + "diplomatic", + "ads", + "DC", + "Ob", + "Montreal", + "Down", + "Milwaukee", + "meal", + "Puerto", + "Mas", + "joy", + "departure", + "Wright", + "spoken", + "style", + "Action", + "Comey", + "delivering", + "toll", + "midnight", + "Revenue", + "firing", + "stunning", + "kicked", + "Ottawa", + "efficiency", + "Lincoln", + "taste", + "ez", + "Weather", + "Morning", + "hadn", + "diversity", + "ily", + "Ay", + "argue", + "error", + "taught", + "che", + "occasion", + "inc", + "Orlando", + "Online", + "legs", + "Nation", + "uck", + "widespread", + "Ocean", + "constantly", + "Latin", + "comfort", + "rely", + "uff", + "Card", + "aring", + "humans", + "Thomson", + "aka", + "BIT", + "Review", + "po", + "ú", + "trucks", + "forecasts", + "view", + "longtime", + "Constitution", + "reserves", + "bit", + "stressed", + "contribution", + "chicken", + "DE", + "fat", + "Oscar", + "criticized", + "testimony", + "apparent", + "constant", + "cabinet", + "Duke", + "aspects", + "lic", + "Vol", + "wing", + "reb", + "Sessions", + "Smart", + "car", + "Im", + "operational", + "regulators", + "Jimmy", + "eter", + "nobody", + "Marc", + "literally", + "resistance", + "Kam", + "sexually", + "69", + "uth", + "viewed", + "picks", + "din", + "talented", + "tennis", + "strengthen", + "gl", + "Protection", + "installed", + "ways", + "Campbell", + "Portland", + "intent", + "Palace", + "secondary", + "locked", + "PA", + "landed", + "length", + "boosted", + "purchases", + "command", + "Asked", + "spaces", + "iconic", + "recommend", + "duties", + "seized", + "delayed", + "FA", + "AND", + "daq", + "hiring", + "occur", + "DC", + "Mus", + "ag", + "hopefully", + "Penn", + "ards", + "striker", + "rent", + "Ty", + "Buffalo", + "Ky", + "hike", + "pper", + "120", + "op", + "wheel", + "Ian", + "chart", + "tt", + "volunteer", + "IG", + "person", + "ight", + "Book", + "unt", + "Technologies", + "Now", + "favour", + "Gh", + "Qatar", + "Dutch", + "Grant", + "Ban", + "rel", + "agreements", + "educational", + "worth", + "Ward", + "700", + "anymore", + "repair", + "operators", + "Li", + "ots", + "Louisiana", + "Whether", + "odds", + "noon", + "Str", + "fail", + "iser", + "forever", + "recall", + "Po", + "Hot", + "designer", + "ido", + "LL", + "Control", + "survive", + "iam", + "organisation", + "Work", + "wider", + "tank", + "work", + "AS", + "posting", + "suddenly", + "MC", + "AL", + "Professor", + "Coach", + "rushed", + "afraid", + "activist", + "that", + "Film", + "backing", + "household", + "signal", + "accurate", + "str", + "Thread", + "Bears", + "ATION", + "Alliance", + "McDonald", + "Venezuela", + "ogg", + "Windows", + "makers", + "utility", + "rapidly", + "attractive", + "pa", + "Larry", + "misconduct", + "freshman", + "qualified", + "cleared", + "crashed", + "participating", + "pages", + "highlight", + "dialogue", + "Alberta", + "ca", + "witnesses", + "ables", + "followers", + "ensuring", + "promoting", + "searching", + "remote", + "clash", + "firefighters", + "teen", + "Place", + "Note", + "regardless", + "ult", + "oney", + "ander", + "ional", + "ining", + "demanded", + "Communications", + "consideration", + "TC", + "Southeast", + "aga", + "Garden", + "inger", + "ht", + "branch", + "mouth", + "audio", + "raw", + "coordinator", + "exact", + "Han", + "delays", + "Wal", + "Wells", + "ng", + "handful", + "girlfriend", + "typical", + "Wayne", + "Franklin", + "constitutional", + "Chance", + "blamed", + "rim", + "preliminary", + "lie", + "da", + "Capitol", + "routine", + "NASA", + "tre", + "Golf", + "sight", + "Der", + "reserve", + "150", + "speculation", + "competitors", + "Macron", + "ony", + "overtime", + "71", + "depending", + "Warner", + "accusations", + "ius", + "predicted", + "Charlie", + "everywhere", + "cable", + "Saint", + "Region", + "hero", + "Emb", + "kinds", + "starter", + "solve", + "Guard", + "loves", + "Douglas", + "funded", + "Brent", + "Anyone", + "substantial", + "Marine", + "Michelle", + "celebrating", + "offset", + "button", + "gg", + "medicine", + "uri", + "somewhere", + "PD", + "mon", + "fires", + "final", + "oth", + "ined", + "underway", + "mistakes", + "grateful", + "cheap", + "È", + "95", + "violations", + "arr", + "surprising", + "ob", + "NATO", + "controversy", + "Sweden", + "funeral", + "reviews", + "promotion", + "TY", + "liberal", + "promising", + "SP", + "How", + "memories", + "breast", + "zi", + "ights", + "pattern", + "outdoor", + "Mu", + "rush", + "Theresa", + "Pol", + "describe", + "Band", + "Stewart", + "1999", + "Raiders", + "mp", + "procedures", + "plot", + "hire", + "used", + "1970", + "picking", + "Sim", + "regard", + "inal", + "backs", + "Hard", + "Low", + "Ac", + "Is", + "guarantee", + "Given", + "beta", + "Tre", + "trans", + "retailer", + "purposes", + "Hol", + "enjoying", + "brown", + "Perry", + "plea", + "MS", + "Dakota", + "Parker", + "commit", + "Lawrence", + "Morris", + "ended", + "virtual", + "ÃĹ", + "fruit", + "84", + "Has", + "ishing", + "dominated", + "FA", + "channels", + "understood", + "citizen", + "checks", + "Kenya", + "disabled", + "SD", + "protecting", + "tweets", + "sparked", + "CO", + "§", + "ori", + "GDP", + "Ser", + "Visit", + "MS", + "barely", + "sand", + "ap", + "aging", + "rel", + "Perhaps", + "Mourinho", + "Jets", + "disclosure", + "highlighted", + "implemented", + "compliance", + "AB", + "Assistant", + "Cape", + "funny", + "leverage", + "machines", + "ranging", + "fastest", + "Roberts", + "Policy", + "gar", + "collapse", + "Through", + "robbery", + "Hay", + "elite", + "Digital", + "Fun", + "Alan", + "ement", + "mit", + "spin", + "listening", + "Doug", + "Saints", + "interior", + "enhance", + "Cardinals", + "ever", + "robust", + "inform", + "suffer", + "book", + "Muslims", + "agriculture", + "km", + "divers", + "ñ", + "Reg", + "equivalent", + "craft", + "settle", + "contains", + "Mack", + "Dis", + "Fore", + "Sudan", + "Mail", + "Brooklyn", + "izer", + "bn", + "hundred", + "exhibition", + "Have", + "vin", + "civilians", + "Cincinnati", + "Some", + "SE", + "bat", + "Ins", + "calm", + "tone", + "normally", + "seeks", + "Ass", + "membership", + "annually", + "employers", + "CO", + "complicated", + "headlines", + "Labor", + "lifestyle", + "Ren", + "Rich", + "cent", + "ude", + "awesome", + "paint", + "rolling", + "walls", + "lab", + "tourists", + "care", + "gear", + "izz", + "cream", + "Tro", + "ices", + "pack", + "diseases", + "Speaker", + "Officers", + "sky", + "83", + "BE", + "categories", + "indicate", + "ru", + "Sony", + "Dun", + "ocks", + "concrete", + "Madison", + "Sab", + "IV", + "observed", + "ria", + "interim", + "encounter", + "ista", + "anger", + "rapid", + "mail", + "destination", + "ĩ", + "breaks", + "rell", + "Chase", + "attorneys", + "rolled", + "Springs", + "Village", + "TO", + "HS", + "campaigns", + "ologist", + "Tax", + "III", + "teach", + "provision", + "rem", + "shirt", + "deployed", + "guidelines", + "av", + "zer", + "rushing", + "94", + "place", + "Man", + "divided", + "Gun", + "windows", + "components", + "aba", + "Switzerland", + "election", + "Tampa", + "Ari", + "ás", + "highway", + "acres", + "crown", + "known", + "inquiry", + "url", + "expertise", + "praised", + "yer", + "conclusion", + "abortion", + "lady", + "tribute", + "unveiled", + "beaten", + "TE", + "Mot", + "unk", + "triple", + "forcing", + "Tickets", + "uit", + "iron", + "scientific", + "IP", + "diagnosed", + "ocean", + "wide", + "Cowboys", + "LC", + "methods", + "Find", + "Dean", + "fundamental", + "Gill", + "feelings", + "IO", + "hu", + "feedback", + "ote", + "duo", + "fully", + "get", + "proof", + "story", + "longest", + "shops", + "Jong", + "Cro", + "Hawaii", + "91", + "Jake", + "Susan", + "submit", + "rav", + "modest", + "lit", + "attempting", + "sits", + "addressing", + "93", + "Bi", + "lying", + "Organization", + "Oak", + "oli", + "fatal", + "mountain", + "val", + "lu", + "Maine", + "charging", + "resigned", + "illo", + "recommendation", + "party", + "Web", + "Panthers", + "noise", + "Brussels", + "awa", + "ambassador", + "accessible", + "Calgary", + "idd", + "Airlines", + "gr", + "nu", + "roy", + "Mars", + "Poland", + "Jerry", + "ados", + "Rico", + "Mir", + "Fin", + "ious", + "packed", + "insider", + "President", + "Bull", + "Yemen", + "Connecticut", + "73", + "departments", + "organic", + "Summer", + "Bet", + "ste", + "zo", + "rat", + "alliance", + "intervention", + "wan", + "OR", + "defined", + "Ãł", + "Chiefs", + "knocked", + "ared", + "holes", + "pulling", + "Todd", + "Jamie", + "Sher", + "signature", + "Sur", + "gym", + "Vladimir", + "Thailand", + "gaming", + "saving", + "ceive", + "82", + "Bern", + "Did", + "hardware", + "ished", + "conspiracy", + "ANS", + "Intelligence", + "assembly", + "101", + "concise", + "Manhattan", + "belief", + "surge", + "deserve", + "consistently", + "Nor", + "okes", + "ðŁ", + "ME", + "Asset", + "substance", + "prefer", + "burning", + "Nik", + "ook", + "Pinterest", + "boyfriend", + "Hal", + "Merkel", + "introduce", + "LinkedIn", + "Full", + "Farm", + "childhood", + "Transportation", + "terrible", + "du", + "intention", + "seemingly", + "elle", + "foods", + "titled", + "dual", + "import", + "developer", + "UL", + "ington", + "Delta", + "?'", + "iness", + "quit", + "Garcia", + "Sri", + "hip", + "Brazilian", + "elt", + "ively", + "structures", + "labour", + "neighbors", + "till", + "soil", + "dropping", + "nominee", + "meets", + "92", + "rant", + "isa", + "luck", + "aa", + "jet", + "Tor", + "Crime", + "lane", + "flu", + "launching", + "Autom", + "aks", + "universities", + "pollution", + "Advis", + "Mall", + "ls", + "deeper", + "repeated", + "meanwhile", + "chip", + "outlets", + "liked", + "sal", + "welfare", + "ago", + "makers", + "ving", + "fer", + "overcome", + "mb", + "shocked", + "akers", + "nonprofit", + "donated", + "eral", + "resume", + "logo", + "subscription", + "74", + "ela", + "aspect", + "html", + "sorry", + "upgrade", + "stance", + "fr", + "papers", + "attacking", + "meaningful", + "81", + "Weinstein", + "creates", + "honour", + "Reply", + "oph", + "march", + "smile", + "comparison", + "will", + "Sanchez", + "voter", + "theory", + "equally", + "Roger", + "perfectly", + "landing", + "billions", + "Bloomberg", + "permit", + "finals", + "racial", + "pregnancy", + "iled", + "Federation", + "forest", + "tag", + "aul", + "drinks", + "(\"", + "Mobile", + "touched", + "clock", + "reg", + "asylum", + "igan", + "senator", + "99", + "Kumar", + "skill", + "1998", + "pa", + "Af", + "mood", + "ston", + "hang", + "MPs", + "Please", + "Eve", + "documentary", + "personality", + "Cast", + "discount", + "bing", + "Boeing", + "depend", + "crossing", + "EX", + "succeed", + "humanitarian", + "Muhammad", + "wages", + "column", + "external", + "statistics", + "TODAY", + "trips", + "ta", + "penalties", + "writers", + "shipping", + "Indians", + "salt", + "Industrial", + "Yankees", + "Den", + "rough", + "barrels", + "Hor", + "bert", + "Dep", + "resign", + "97", + "balls", + "Jun", + "Bab", + "associate", + "string", + "hub", + "organ", + "Marshall", + "FIFA", + "Mun", + "ency", + "research", + "peers", + "tall", + "Goldman", + "Don", + "parade", + "parks", + "det", + "disappointing", + "reflects", + "Lakers", + "files", + "relatives", + "USD", + "Article", + "custom", + "Carlos", + "tracking", + "maintaining", + "Cur", + "ardo", + "Skip", + "attitude", + "Just", + "institution", + "narrow", + "snap", + "enterprise", + "drives", + "77", + "crop", + "virus", + "celebrity", + "economies", + "ued", + "sum", + "Dubai", + "Insurance", + "Ĺ", + "ury", + "Unfortunately", + "closure", + "ota", + "Philip", + "oms", + "investigated", + "generations", + "ETF", + "Keith", + "Later", + "isk", + "preferred", + "default", + "towns", + "Rod", + "Die", + "integrated", + "acquiring", + "voices", + "ser", + "presents", + "BR", + "Emergency", + "religion", + "HA", + "responding", + "Things", + "beef", + "Without", + "urd", + "Carl", + "administrative", + "Which", + "challenged", + "cooking", + "ivid", + "Fer", + "tremendous", + "Terry", + "iri", + "CS", + "Junior", + "Reddit", + "tea", + "accounting", + "lan", + "detention", + "replied", + "SI", + "Hel", + "ns", + "Prof", + "ramp", + "Conservative", + "attendance", + "specialist", + "Final", + "advertisement", + "acquire", + "WhatsApp", + "workforce", + "Calif", + "speakers", + "EPA", + "conviction", + "hire", + "Fisher", + "Intel", + "bin", + "Was", + "earth", + "vi", + "hurricane", + "holidays", + "assume", + "involve", + "dynamic", + "Gre", + "item", + "pound", + "anxiety", + "Print", + "rop", + "automatically", + "discrimination", + "Lam", + "Coll", + "impressed", + "involves", + "Les", + "Tri", + "Look", + "iOS", + "grab", + "Angel", + "stops", + "Pay", + "ECB", + "bunch", + "letting", + "ele", + "Additionally", + "boards", + "NC", + "tragedy", + "pink", + "gonna", + "ones", + "rev", + "Independent", + "Cambridge", + "Pence", + "prosecution", + "deputies", + "Ahmed", + "lows", + "Amy", + "Building", + "mark", + "smooth", + "sole", + "wanting", + "Heart", + "obtain", + "Bus", + "exchanges", + "friendly", + "label", + "elect", + "Companies", + "owing", + "CB", + "RI", + "Master", + "liquid", + "Danny", + "proceeds", + "Laura", + "card", + "tears", + "exploration", + "depression", + "ken", + "Fe", + "lending", + "Youth", + "ality", + "NS", + "moon", + "Taiwan", + "struggles", + "discovery", + "qualify", + "wireless", + "alia", + "witnessed", + "height", + "Guy", + "left", + "KE", + "foul", + "Mohammed", + "grass", + "Non", + "swim", + "brilliant", + "you", + "Flynn", + "singing", + "eria", + "UT", + "McCain", + "Sep", + "Wars", + "burden", + "pas", + "abandoned", + "int", + "Turner", + "collective", + "Environmental", + "Students", + "offerings", + "resignation", + "explosion", + "Koh", + "ager", + "throws", + "asks", + "light", + "anyway", + "yard", + "carrier", + "waves", + "backed", + "TR", + "oud", + "breach", + "dated", + "dressed", + "Dodgers", + "oles", + "78", + "reads", + "predict", + "Jerusalem", + "PT", + "crack", + "yan", + "nights", + "eline", + "convinced", + "lock", + "carefully", + "Mercedes", + "ultimate", + "dist", + "slight", + "Edwards", + "swing", + "iling", + "knife", + "Nashville", + "IF", + "inder", + "udd", + "senators", + "Further", + "Xi", + "str", + "Od", + "days", + "comm", + "verdict", + "confirmation", + "king", + "CS", + "advocates", + "pride", + "memorial", + "ams", + "erman", + "teenager", + "Neil", + "uts", + "soul", + "see", + "post", + "chest", + "fire", + "Lynch", + "peaceful", + "OND", + "Industries", + "Juan", + "restore", + "reliable", + "ming", + "agan", + "Source", + "Cabinet", + "remarkable", + "Trudeau", + "Es", + "integrity", + "ove", + "fe", + "proceedings", + "connections", + "unprecedented", + "Glen", + "ux", + "earning", + "ingredients", + "nominated", + "Bangladesh", + "made", + "lessons", + "breakfast", + "Relations", + "loose", + "Al", + "upgraded", + "ral", + "Page", + "oto", + "Queensland", + "procedure", + "Small", + "respective", + "pictured", + "Bas", + "preparation", + "Myanmar", + "donation", + "visible", + "iest", + "Broadway", + "rick", + "Schools", + "arrests", + "Jessica", + "Bengal", + "hell", + "announcing", + "mail", + "McG", + "two", + "rest", + "OD", + "Bradley", + "doubled", + "pledged", + "comeback", + "extraordinary", + "slide", + "assess", + "agricultural", + "Kay", + "vendors", + "narrative", + "reviewed", + "Pass", + "inspiration", + "Hunter", + "calendar", + "Diamond", + "removal", + "ners", + "Kap", + "consent", + "visual", + "cheese", + "Ther", + "FR", + "Shanghai", + "iah", + "Cole", + "AK", + "ranking", + "cook", + "halftime", + "Stars", + "routes", + "aim", + "establishment", + "Mug", + "survivors", + "urg", + "Brett", + "unexpected", + "ained", + "rarely", + "Gall", + "advocate", + "Nad", + "911", + "racist", + "erer", + "Rev", + "Section", + "helpful", + "CT", + "agg", + "governance", + "felony", + "optimistic", + "electoral", + "EG", + "town", + "daughters", + "answered", + "thin", + "Classic", + "shareholder", + "Blake", + "Fla", + "parliamentary", + "dy", + "commented", + "tri", + "globe", + "mandate", + "slipped", + "Tower", + "operated", + "gers", + "assured", + "Martinez", + "designs", + "Model", + "stakeholders", + "defended", + "seniors", + "vacation", + "globally", + "ump", + "Not", + "clip", + "articles", + "BR", + "km", + "Front", + "PL", + "adoption", + "sudden", + "framework", + "hanging", + "gl", + "Sel", + "moderate", + "reverse", + "income", + "cor", + "GB", + "physically", + "transparency", + "Electric", + "refugee", + "profile", + "iva", + "ately", + "AC", + "transferred", + "affair", + "Alaska", + "oria", + "Change", + "repeat", + "screening", + "ender", + "Cas", + "Dav", + "focuses", + "commissioner", + "upside", + "Keep", + "Blues", + "ently", + "aut", + "experiencing", + "aman", + "approve", + "mile", + "cheaper", + "Wind", + "Store", + "grabbed", + "sons", + "fighter", + "um", + "Based", + "don", + "constitution", + "finals", + "act", + "¢", + "mill", + "organisations", + "Toyota", + "yuan", + "terrorists", + "forth", + "availability", + "entrance", + "volumes", + "mult", + "plus", + "Columbus", + "Summit", + "babies", + "Mur", + "Gray", + "Char", + "Butler", + "pose", + "Natural", + "Att", + "decrease", + "tens", + "kt", + "minds", + "impacted", + "chapter", + "Op", + "Harrison", + "Rodriguez", + "ethnic", + "travelling", + "Bond", + "ader", + "core", + "gallery", + "founder", + "Vill", + "decent", + "History", + "Int", + "Na", + "Had", + "mainstream", + "Ts", + "bottle", + "sen", + "recession", + "sophomore", + "silence", + "cc", + "qualifying", + "complained", + "Rad", + "actively", + "backs", + "Musk", + "careful", + "meals", + "Dor", + "mess", + "Belgium", + "ke", + "Lopez", + "bow", + "helicopter", + "was", + "stone", + "kins", + "unlike", + "collision", + "Alt", + "HP", + "Mason", + "has", + "climbed", + "indication", + "hotels", + "loud", + "Milan", + "kes", + "badly", + "trials", + "impacts", + "Jane", + "crossed", + "discussing", + "SM", + "popularity", + "Want", + "fall", + "artificial", + "Bu", + "akh", + "dominant", + "gov", + "premier", + "execution", + "gate", + "swimming", + "chat", + "devastating", + "acking", + "reception", + "urt", + "theater", + "gather", + "tear", + "uro", + "democratic", + "rebels", + "lifetime", + "radical", + "uan", + "techniques", + "ache", + "ior", + "camps", + "telephone", + "Dublin", + "Brand", + "Marcus", + "aun", + "Rec", + "82", + "ban", + "safely", + "aku", + "aki", + "bankruptcy", + "FF", + "format", + "attached", + "Fame", + "Edward", + "merger", + "Representatives", + "izes", + "hidden", + "val", + "zz", + "excess", + "scope", + "divorce", + "burn", + "requirement", + "BB", + "Hand", + "cons", + "risen", + "twitter", + "offseason", + "Sometimes", + "Inf", + "Ang", + "uer", + "report", + "dreams", + "700", + "ips", + "Dream", + "gifts", + "somehow", + "Tur", + "Rachel", + "can", + "log", + "Medicaid", + "les", + "tired", + "Arkansas", + "liquidity", + "Phillips", + "BTC", + "hide", + "pun", + "Run", + "lyn", + "UC", + "Design", + "Dev", + "valuation", + "reveals", + "Child", + "other", + "posed", + "lee", + "ships", + "True", + "describes", + "runner", + "bro", + "ankle", + "od", + "Annual", + "CL", + "overhaul", + "ned", + "bold", + "mo", + "Falls", + "employed", + "Gro", + "flash", + "TD", + "nervous", + "integration", + "smartphones", + "movements", + "nie", + "ition", + "Third", + "Ģ", + "metres", + "economist", + "omp", + "teens", + "everyday", + "interviewed", + "briefly", + "],", + "uke", + "FOX", + "underlying", + "Luc", + "courses", + "ss", + "amed", + "°", + "ju", + "Banks", + "outfit", + "illing", + "trafficking", + "urging", + "belt", + "rid", + "CP", + "elderly", + "Growth", + "án", + "Sn", + "surrounded", + "sisters", + "Islam", + "synd", + "Costa", + "di", + "Kl", + "manufacturer", + "holders", + "element", + "load", + "booked", + "accompanied", + "Chamber", + "briefing", + "Oh", + "imi", + "Defence", + "Currently", + "aking", + "handled", + "CD", + "Benjamin", + "pocket", + "Kashmir", + "lighting", + "aps", + "1997", + "ech", + "addiction", + "bases", + "priorities", + "hardly", + "Quebec", + "Earn", + "IES", + "Zach", + "Along", + "MI", + "ins", + "Rogers", + "Kan", + "Future", + "triggered", + "Unit", + "weighed", + "pointing", + "chocolate", + "Browns", + "ISIS", + "goalkeeper", + "saves", + "Andre", + "burn", + "Cont", + "Netherlands", + "politically", + "Ashley", + "Whit", + "aded", + "PH", + "borders", + "ORE", + "ally", + "Trump", + "istan", + "Hunt", + "Cancer", + "Grace", + "Tottenham", + "1960", + "Marg", + "Bryan", + "Again", + "acing", + "arguments", + "Southwest", + "vocal", + "judgment", + "engaging", + "adopt", + "rental", + "linebacker", + "Kardashian", + "episodes", + "..", + "unt", + "vowed", + "79", + "ule", + "transit", + "offshore", + "suppliers", + "arguing", + "satellite", + "Lind", + "Taliban", + "Buy", + "Caribbean", + "Barry", + "authors", + "Wolf", + "viewing", + "Cubs", + "From", + "%", + "currencies", + "Why", + "Broncos", + "trick", + "diesel", + "Liberal", + "FL", + "topics", + "retain", + "Liberty", + "acquisitions", + "ced", + "fre", + "fleet", + "copper", + "Pot", + "jen", + "Elliott", + "Pyongyang", + "object", + "Use", + "mutual", + "MP", + "ev", + "deny", + "Everyone", + "lling", + "pays", + "drought", + "corn", + "workplace", + "rig", + "Mn", + "advisory", + "Cat", + "chronic", + "Steelers", + "boxes", + "Nap", + "demonstrated", + "Tournament", + "symbol", + "Afghan", + "Tan", + "ired", + "Ev", + "Consumer", + "moral", + "Additional", + "websites", + "occasions", + "fate", + "pitcher", + "taxpayers", + "deemed", + "Libya", + "priced", + "distributed", + "Forum", + "rice", + "bloc", + "provisions", + "agh", + "pen", + "attracted", + "Edmonton", + "thousand", + "painting", + "il", + "courtesy", + "eliminate", + "acc", + "meters", + "reflected", + "component", + "Every", + "sells", + "fault", + "burned", + "Kirk", + "Anna", + "appeals", + "eggs", + "frequent", + "trigger", + "revised", + "Angela", + "81", + "singles", + "viral", + "worries", + "Should", + "profit", + "raises", + "Bryant", + "Product", + "tenure", + "diabetes", + "colour", + "azz", + "Girls", + "practical", + "blind", + "ancing", + "pictured", + "finale", + "Election", + "athletic", + "promoted", + "flowers", + "trains", + "ario", + "sufficient", + "IE", + "examples", + "shed", + "birds", + "chaos", + "wound", + "rocket", + "wet", + "sample", + "Nag", + "Oliver", + "scrutiny", + "Seven", + "Roman", + "Fred", + "weird", + "Tam", + "Support", + "Nathan", + "studying", + "introduction", + "tons", + "cer", + "aus", + "ION", + "critic", + "Ah", + "alo", + "pur", + "storms", + "Mission", + "credits", + "grants", + "comp", + "hearts", + "part", + "pin", + "subsequent", + "mad", + "Sacramento", + "woman", + "from", + "outcomes", + "oldest", + "desperate", + "Tal", + "DJ", + "ward", + "audiences", + "importantly", + "Emily", + "sk", + "Heat", + "Type", + "Peace", + "suspicious", + "aly", + "GET", + "CAP", + "dis", + "Iraqi", + "Reed", + "strange", + "Parent", + "900", + "glad", + "Troy", + "Short", + "heritage", + "arriving", + "ingly", + "transformation", + "lease", + "collapsed", + "cha", + "Patrol", + "computers", + "principles", + "sporting", + "Hughes", + "mile", + "Cit", + "drilling", + "Box", + "ÃŁ", + "bre", + "Overall", + "opioid", + "delighted", + "honored", + "Cold", + "unions", + "Cou", + "Circuit", + "blast", + "sson", + "Hernandez", + "Looking", + "legally", + "Walmart", + "bridge", + "mat", + "rad", + "ids", + "dining", + "rebound", + "abad", + "Rom", + "impose", + "Alpha", + "Weekly", + "TER", + "Jam", + "absolute", + "inventory", + "Billy", + "Karen", + "Friends", + "Cent", + "Vikings", + "Much", + "cell", + "ads", + "ph", + "killer", + "Members", + "shooter", + "Investigators", + "Joshua", + "participated", + "innocent", + "Richmond", + "itor", + "Dal", + "Operator", + "makeup", + "conf", + "NEWS", + "Def", + "chase", + "Cost", + "mont", + "\":", + "arrangements", + "stein", + "retire", + "Luis", + "renewed", + "Township", + "checked", + "arts", + "Cash", + "centres", + "chers", + "Solutions", + "legend", + "ige", + "most", + "osed", + "Por", + "premiere", + "FS", + "missiles", + "Lang", + "sing", + "best", + "tail", + "riders", + "Picture", + "zen", + "Kent", + "transform", + "wildlife", + "smoking", + "preseason", + "Lucas", + "Anne", + "owski", + "tape", + "displayed", + "forum", + "anonymity", + "Indianapolis", + "hips", + "acc", + "Moreover", + "lers", + "area", + "Indeed", + "conducting", + "infection", + "dealt", + "OB", + "asing", + "Gaza", + "itter", + "Ka", + "hopeful", + "Snow", + "entitled", + "affecting", + "eager", + "circle", + "laugh", + "Prosecutors", + "Dur", + "barriers", + "Poll", + "oun", + "Palm", + "chi", + "samples", + "compromise", + "atter", + "enormous", + "é", + "coming", + "Pharmaceutical", + "rank", + "Let", + "transgender", + "Cloud", + "FO", + "Bor", + "bonus", + "ordinary", + "Pres", + "HIV", + "ires", + "OSE", + "dancing", + "HD", + "versions", + "88", + "rate", + "tackles", + "knock", + "Emma", + "motivated", + "Bennett", + "Burn", + "grid", + "embrace", + "Spurs", + "flows", + "Ger", + "sponsored", + "survival", + "ching", + "1995", + "reward", + "depends", + "postseason", + "loaded", + "neutral", + "Pop", + "BL", + "revolution", + "Freedom", + "recovering", + "requiring", + "ALL", + "ARE", + "mini", + "lt", + "FDA", + "carpet", + "Prior", + "admission", + "Ever", + "Tribune", + "Ronaldo", + "thick", + "lanes", + "84", + "Memphis", + "opt", + "BO", + "faculty", + "Chad", + "SUV", + "Hen", + "este", + "Hu", + "Agriculture", + "store", + "Drug", + "inter", + "1996", + "ident", + "backup", + "Honda", + "Hope", + "oes", + "ums", + "amer", + "breath", + "110", + "joke", + "Ald", + "wondering", + "Assad", + "Rem", + "fundraising", + "pot", + "è", + "questioning", + "pent", + "Money", + "Medicine", + "wick", + "Knights", + "batting", + "Mos", + "designated", + "isse", + "spotlight", + "lake", + "caution", + "inmates", + "lap", + "CE", + "Javascript", + "Deutsche", + "Fargo", + "guaranteed", + "borough", + "functions", + "Elementary", + "Chuck", + "pitched", + "Krist", + "steal", + "chips", + "alarm", + "beloved", + "scale", + "assaulted", + "Pentagon", + "temporarily", + "93", + ">", + "Portugal", + "ti", + "HL", + "decreased", + "existence", + "isolated", + "deposit", + "studied", + "\")", + "trophy", + "Brooks", + "battling", + "weaker", + "Private", + "Access", + "virtually", + "shortage", + "gaining", + "bathroom", + "TON", + "concerning", + "engineer", + "bread", + "demonstrate", + "Dh", + "horses", + "intersection", + "colors", + "delegation", + "notable", + "withdrawal", + "Dennis", + "locally", + "coastal", + "comply", + "Moh", + "Albert", + "closest", + "CITY", + "83", + "cancelled", + "ðŁ", + "sharply", + "RS", + "productivity", + "basket", + "SS", + "admit", + "ool", + "ination", + "BB", + "sur", + "Steel", + "Ted", + "Pac", + "patterns", + "listing", + "replacing", + "Pradesh", + "roots", + "broker", + "Writing", + "sued", + "organised", + "Thanksgiving", + "NOT", + "journalism", + "uel", + "kilometers", + "hunt", + "berry", + "Mother", + "legitimate", + "input", + "Rel", + "Guardian", + "Ar", + "transported", + "bedroom", + "ashing", + "bats", + "cleaning", + "wrapped", + "Pacific", + "fence", + "testified", + "1994", + "interference", + "matching", + "expression", + "eta", + "Spencer", + "strategist", + "who", + "victories", + "2022", + "stakes", + "buses", + "Housing", + "editorial", + "86", + "Bishop", + "frustrated", + "appearing", + "http", + "IGHT", + "memo", + "insiders", + "Even", + "classroom", + "chef", + "aining", + "].", + "McD", + "87", + "Punjab", + "ancient", + "resolved", + "dying", + "destruction", + "governing", + "restructuring", + "Pick", + "municipal", + "engines", + "Hudson", + "Æ", + "repeal", + "standing", + "bound", + "OS", + "Commonwealth", + "description", + "households", + "mal", + "stopping", + "equ", + "regulator", + "containing", + "removing", + "withdraw", + "buried", + "lists", + "Gil", + "lowered", + "formally", + "Round", + "asi", + "¥", + "lett", + "progressive", + "Falcons", + "Raw", + "gun", + "contributing", + "hunting", + "valid", + "exception", + "Players", + "Tra", + "racism", + "hing", + "chen", + "differently", + "championships", + "Eng", + "NO", + "Auto", + "Erdogan", + "iding", + "warming", + "civilian", + "Dam", + "fantasy", + "Nav", + "itions", + "Drew", + "Nancy", + "trapped", + "Russians", + "IC", + "flexibility", + "ular", + "violated", + "ipped", + "garage", + "Deep", + "praise", + "Lab", + "Player", + "judicial", + "donate", + "separated", + "releases", + "nik", + "explanation", + "aph", + "loyal", + "strongest", + "Shar", + "rescued", + "ambitious", + "climb", + "scared", + "ignored", + "cut", + "stole", + "weakness", + "Ridge", + "oa", + "LA", + "dep", + "Powell", + "Do", + "protein", + "reiterated", + "Cox", + "aling", + "Unlike", + "Kane", + "McConnell", + "showcase", + "uniform", + "ower", + "discover", + "stop", + "ipper", + "treatments", + "grocery", + "subscribers", + "lock", + "ple", + "flew", + "ania", + "stepping", + "Soviet", + "consultant", + "ags", + "Lim", + "91", + "Code", + "ports", + "box", + "lakh", + "reminder", + "ym", + "Travis", + "pure", + "now", + "VR", + "achievement", + "Emirates", + "Thunder", + "merely", + "Ca", + "Average", + "Da", + "topped", + "Curry", + "chemicals", + "amendment", + "Border", + "Bat", + "130", + "programming", + "tele", + "Karl", + "averaged", + "Spe", + "world", + "PG", + "fights", + "Princess", + "CIA", + "Abe", + "acted", + "only", + "insight", + "athlete", + "Tar", + "commerce", + "averaging", + "cr", + "Palestinians", + "Well", + "bull", + "choosing", + "surely", + "Secret", + "teammate", + "Amendment", + "Birmingham", + "excitement", + "strong", + "Sin", + "damages", + "rated", + "rankings", + "conservation", + "home", + "erm", + "ield", + "disorder", + "acher", + "naturally", + "atur", + "packages", + "approaches", + "icks", + "ourn", + "odd", + "shore", + "Being", + "magic", + "tourist", + "largest", + "whenever", + "lenders", + "egg", + "Chair", + "lets", + "warnings", + "į", + "pol", + "drag", + "Amb", + "Cle", + "Louisville", + "Shaw", + "lands", + "anthem", + "Trail", + "accepting", + "anger", + "good", + "Broad", + "Lebanon", + "Million", + "Henderson", + "wh", + "dust", + "92", + "Mend", + "checking", + "Cow", + "sized", + "automatic", + "celebrates", + "arena", + "finger", + "Harvard", + "frustration", + "strict", + "preserve", + "sleeping", + "converted", + "insights", + "tra", + "jailed", + "chamber", + "toxic", + "ading", + "Triple", + "grade", + "Rest", + "Holy", + "oper", + "desk", + "matchup", + "steep", + "Got", + "lay", + "Cab", + "aked", + "Foster", + "runners", + "NA", + "destroy", + "supportive", + "Racing", + "trademark", + "jacket", + "horror", + "Ale", + "ass", + "sch", + "abb", + "planes", + "impression", + "Early", + "Pompe", + "king", + "silent", + "Cuba", + "medication", + "ences", + "list", + "ailing", + "WA", + "ella", + "prop", + "halt", + "slowing", + "Foods", + "anonymous", + "kh", + "traveled", + "communicate", + "ter", + "Hockey", + "Robin", + "swept", + "clinic", + "ration", + "len", + "au", + "careers", + "Sound", + "addresses", + "China", + "Sr", + "exhibit", + "Motors", + "Il", + "install", + "Okay", + ">>", + "hood", + "stand", + "audit", + "cake", + "flames", + "bel", + "Must", + "Manafort", + "commodity", + "night", + "Room", + "Lanka", + "commander", + "ln", + "database", + "Set", + "graduated", + "Target", + "outbreak", + "rou", + "Pope", + "Equ", + "polling", + "dig", + "brutal", + "Barn", + "definition", + "pit", + "pickup", + "Bitcoin", + "Reid", + "loving", + "Herald", + "Canadians", + "neighbor", + "dies", + "ione", + "Ref", + "big", + "guards", + "including", + "ente", + "partially", + "Image", + "bulk", + "slot", + "Northwest", + "Barclays", + "airlines", + "iver", + "isi", + "subsidiary", + "cont", + "Daniels", + "script", + "unfair", + "screens", + "prof", + "Irma", + "1992", + "mandatory", + "Sant", + "suspicion", + "NES", + "Lauren", + "igen", + "prevention", + "tension", + "ema", + "tasks", + "shake", + "explosive", + "affects", + "mum", + "Dog", + "rer", + "opted", + "trio", + "lesson", + "automotive", + "where", + "Montgomery", + "couples", + "89", + "AF", + "info", + "Form", + "spectrum", + "bands", + "okay", + "stroke", + "Netanyahu", + "wealthy", + "Around", + "Glenn", + "sec", + "there", + "ickets", + "Budget", + "BMW", + "flagship", + "rier", + "podcast", + "pursuing", + "pos", + "Islands", + "Urban", + "page", + "emotions", + "ided", + "dividends", + "boom", + "accusing", + "ird", + "Nam", + "ava", + "wishes", + "Ny", + "Stanford", + "criteria", + "Jews", + "engineers", + "accuracy", + "displays", + "deserves", + "ridge", + "omm", + "aur", + "dramatically", + "unity", + "speed", + "declining", + "permits", + "Kn", + "consulting", + "aux", + "ATE", + "Wat", + "Editor", + "sy", + "urn", + "Using", + "asc", + "ital", + "cre", + "quality", + "ce", + "enemy", + "offence", + "icket", + "Dick", + "TH", + "Championships", + "overwhelming", + "rib", + "ku", + "rap", + "homer", + "acion", + "member", + "erv", + "aney", + "MB", + "eded", + "punishment", + "negotiate", + "File", + "stream", + "Hur", + "nose", + "Fab", + "iter", + "painful", + "ITY", + "eren", + "collecting", + "Additional", + "entrepreneurs", + "bal", + "exploring", + "guitar", + "partnerships", + "furniture", + "authorized", + "easing", + "shirt", + "Gross", + "politician", + "Simpson", + "drone", + "Katie", + "profitability", + "NHS", + "Sierra", + "Norway", + "ASHINGTON", + "ific", + "condemned", + "team", + "Nebraska", + "thrilled", + "iller", + "patrol", + "WR", + "orm", + "spectacular", + "Knight", + "Travel", + "nam", + "muscle", + "Rain", + "Colombia", + "nursing", + "migration", + "Mitch", + "releasing", + "Besides", + "Mul", + "headline", + "contemporary", + "dev", + "Chan", + "indicates", + "Ap", + "Lt", + "Marvel", + "remembered", + "®", + "Forces", + "Colin", + "Gabriel", + "objects", + "RHP", + "kar", + "Ko", + "signals", + "inner", + "real", + "RO", + "romantic", + "cat", + "Kel", + "gut", + "Boys", + "youngest", + "Celtics", + "slated", + "remind", + "productive", + "set", + "Co", + "Bailey", + "renewable", + "Carson", + "Dj", + "Kos", + "urge", + "fin", + "pursuit", + "CON", + "Chapter", + "pal", + "gate", + "Packers", + "Reports", + "Rugby", + "Masters", + "MO", + "98", + "catches", + "Agreement", + "Tillerson", + "Ice", + "rumors", + "Leonard", + "Dolphins", + "LP", + "top", + "Crist", + "Hon", + "blaze", + "rhetoric", + "ands", + "ady", + "David", + "igh", + "buzz", + "Strong", + "shocking", + "Rh", + "negotiating", + "tender", + "Johnny", + "Mario", + "97", + "Heritage", + "exists", + "prayers", + "lengthy", + "safer", + "Halloween", + "Jared", + "Connect", + "bump", + "strain", + "filling", + "trauma", + "completing", + "cht", + "killings", + "anne", + "GE", + "Rescue", + "dealers", + "locals", + "Victor", + "tragic", + "delivers", + "orts", + "rugby", + "installation", + "asa", + "Bart", + "journal", + "school", + "Come", + "Veterans", + "Sun", + "crowds", + "transparent", + "implications", + "Huawei", + "sex", + "rallied", + "responses", + "debris", + "convention", + "mothers", + "BE", + "Route", + "rebel", + "Emmanuel", + "aster", + "understands", + "pound", + "Castle", + "2021", + "rik", + "GR", + "convince", + "ault", + "passionate", + "Sciences", + "arrives", + "idad", + "celebrities", + "ends", + "Fans", + "dish", + "Corps", + "hat", + "employer", + "Hy", + "powered", + "grandmother", + "FL", + "oured", + "VE", + "Inst", + "Perez", + "tune", + "citizenship", + "ignore", + "doubles", + "IB", + "programmes", + "inda", + "entities", + "Interior", + "prompting", + "wire", + "theatre", + "%)", + "heels", + "Ju", + "deposits", + "trash", + "mond", + "she", + "iana", + "islands", + "Tommy", + "pub", + "discipline", + "SW", + "musicians", + "embassy", + "QB", + "hander", + "UES", + "Ferguson", + "blocking", + "ahn", + "fines", + "tactics", + "bullet", + "equipped", + "escaped", + "Sil", + "Pack", + "Athletic", + "Mic", + "Does", + "Carr", + "Chargers", + "Kyl", + "zones", + "µ", + "iki", + "greatly", + "MD", + "immigrant", + "Construction", + "Born", + "iment", + "Wade", + "visa", + "genuine", + "electronics", + "Sat", + "sponsors", + "Montana", + "spell", + "Sachs", + "Et", + "foster", + "locker", + "explaining", + "Age", + "gunman", + "sauce", + "cry", + "stimulus", + "array", + "compare", + "boats", + "ext", + "iders", + "Ast", + "Parks", + "ester", + "94", + "relating", + "vegetables", + "accountable", + "hyper", + "Wim", + "newest", + "Rome", + "Chancellor", + "CBS", + "businessman", + "Delaware", + "lands", + "court", + "aria", + "approaching", + "cker", + "Salt", + "Mak", + "treating", + "subsequently", + "Ell", + "xton", + "180", + "determination", + "Salman", + "Joel", + "classified", + "span", + "earthquake", + "ranked", + "96", + "Tiger", + "advocacy", + "mit", + "colleges", + "Yeah", + "Captain", + "orange", + "projections", + "electrical", + "MA", + "olog", + "Newcastle", + "oppers", + "representation", + "lawsuits", + "just", + "aced", + "Race", + "Aqu", + "Bills", + "exclusively", + "Profile", + "hometown", + "Stan", + "starring", + "deciding", + "Rating", + "Medicare", + "Transport", + "mystery", + "Ta", + "Pad", + "Swedish", + "Carroll", + "about", + "torn", + "nurse", + "NE", + "waited", + "Jeffrey", + "Until", + "bone", + "Bobby", + "pronounced", + "pharmaceutical", + "Gallery", + "Match", + "economists", + "Marketing", + "face", + "Petroleum", + "ories", + "Mets", + "Core", + "billion", + "examination", + "Porter", + "2016", + "golden", + "sem", + "Duterte", + "Jefferson", + "Tehran", + "Leicester", + "DA", + "adapt", + "Dame", + "Ric", + "unchanged", + "ect", + "sections", + "kg", + "igned", + "filings", + "react", + "urgent", + "vessels", + "spark", + "butter", + "Cons", + "stating", + "corporations", + "Hus", + "damaging", + "raw", + "equality", + "Two", + "Mills", + "iu", + "obligation", + "Brook", + "arian", + "Re", + "photographs", + "epic", + "Student", + "Therefore", + "god", + "FILE", + "iqu", + "describing", + "proceed", + "cas", + "Kat", + "Bra", + "adequate", + "passage", + "thanked", + "USA", + "Neither", + "Legislature", + "finances", + "inst", + "ĵ", + "Angels", + "vet", + "Dead", + "Ex", + "kicks", + "force", + "soy", + "Windsor", + "enhanced", + "1993", + "Czech", + "gradually", + "Magic", + "shadow", + "neighborhoods", + "Rivers", + "rapper", + "Girl", + "Rot", + "crackdown", + "fish", + "preventing", + "produces", + "Mi", + "notified", + "underground", + "WE", + "admits", + "boxing", + "refer", + "commitments", + "Woman", + "denies", + "col", + "Side", + "ambulance", + "Rodgers", + "aftermath", + "deck", + "irmed", + "errors", + "Convention", + "curb", + "Shop", + "Thai", + "ma", + "respected", + "MVP", + "borrowing", + "cruise", + "Sure", + "sentencing", + "Obamacare", + "Ir", + "Sale", + "Pete", + "openly", + "startup", + "rock", + "cargo", + "telecom", + "Download", + "extending", + "Current", + "competitions", + "Kids", + "shy", + "Kerry", + "Never", + "Devils", + "prim", + "Con", + "curve", + "assumed", + "adjust", + "immune", + "UE", + "Ur", + "conventional", + "grandchildren", + "Bol", + "Ad", + "Maduro", + "fi", + "UAE", + "Organ", + "indicating", + "iem", + "Against", + "Ambassador", + "Seoul", + "criminals", + "how", + "put", + "reminded", + "parked", + "lich", + "continent", + "matched", + "Nicole", + "genetic", + "humanity", + "Tem", + "indicator", + "vessel", + "defendant", + "Griffin", + "jan", + "vend", + "boro", + "brokerage", + "Fall", + "mere", + "VILLE", + "lasted", + "Mind", + "patch", + "Insider", + "Comm", + "technique", + "IM", + "Cavaliers", + "shame", + "mil", + "oot", + "irt", + "cop", + "Leon", + "frozen", + "slip", + "pton", + "panels", + "pitching", + "leather", + "Logan", + "Nearly", + "urch", + "instructions", + "Row", + "Kurdish", + "this", + "legendary", + "su", + "stabbed", + "sters", + "teenage", + "def", + "oversight", + "volatile", + "transmission", + "Sgt", + "Indigenous", + "Oxford", + "Casey", + "cor", + "salaries", + "sponsor", + "prescription", + "mat", + "Leeds", + "Pakistani", + "evil", + "tables", + "Abdul", + "expectation", + "legislature", + "Lin", + "¹", + "contractor", + "shifting", + "generous", + "Eddie", + "puck", + "utt", + "dubbed", + "nowhere", + "betting", + "disclose", + "Ĥ", + "Fashion", + "Harper", + "handed", + "isha", + "Reds", + "achievements", + "ume", + "shootings", + "advisers", + "Easter", + "internationally", + "Wi", + "Gandhi", + "Christians", + "recruiting", + "experiment", + "sol", + "difficulties", + "influential", + "hybrid", + "formation", + "Boulevard", + "flags", + "formula", + "front", + "inclusion", + "None", + "ICE", + "filming", + "Lou", + "Reynolds", + "pump", + "exceptional", + "ANG", + "Corporate", + "SAN", + "Healthcare", + "Ukrainian", + "aron", + "pants", + "drops", + "ete", + "Studies", + "wounds", + "END", + "shower", + "reviewing", + "Greater", + "»", + "itors", + "alled", + "squ", + "Ronald", + "Inv", + "tougher", + "balanced", + "lined", + "principle", + "1950", + "leak", + "Be", + "circuit", + "unfortunate", + "Gran", + "Fish", + "friendship", + "asp", + "OO", + "obligations", + "coup", + "OK", + "breakdown", + "hook", + "researcher", + "inated", + "Marie", + "Gab", + "WA", + "quez", + "General", + "Swift", + "gust", + "Carol", + "Century", + "OPEC", + "Rd", + "Cop", + "subjects", + "Comments", + "ases", + "relation", + "Environment", + "ı", + "gasoline", + "Log", + "icon", + "profitable", + "Retail", + "ANC", + "appealing", + "villages", + "pizza", + "mall", + "tower", + "Linda", + "accomplished", + "pod", + "leaked", + "Wed", + "mer", + "opposing", + "!'", + "stomach", + "revealing", + "ho", + "DF", + "Sterling", + "solely", + "pres", + "Cy", + "Latest", + "Pitt", + "Think", + "capability", + "aled", + "executed", + "alling", + "Silva", + "restricted", + "declaration", + "kilometres", + "rol", + "identifying", + "donors", + "vent", + "costly", + "ense", + "Seeking", + "OURCE", + "iving", + "placing", + "tech", + "bottles", + "writer", + "Seahawks", + "oming", + "Arthur", + "ously", + "bin", + "Va", + "bias", + "liability", + "ift", + "rak", + "aves", + "cautious", + "Prize", + "iley", + "Sharma", + "global", + "wars", + "sm", + "Remember", + "wind", + "Richardson", + "Sum", + "Vincent", + "Rice", + "inf", + "consultation", + "range", + "bacteria", + "architecture", + "pole", + "Mach", + "cattle", + "abused", + "being", + "HERE", + "fame", + "hearings", + "Brit", + "joins", + "McGregor", + "oppose", + "cheer", + "itting", + "imes", + "usage", + "stint", + "outlet", + "shoppers", + "Baptist", + "inappropriate", + "ALSO", + "stealing", + "pledge", + "Ran", + "photographer", + "prevented", + "01", + "Engineering", + "Products", + "universe", + "McCarthy", + "¿", + "graded", + "inspection", + "ind", + "Fi", + "aren", + "protections", + "sorts", + "Works", + "billionaire", + "Gay", + "iPad", + "IX", + "defendants", + "band", + "farms", + "hom", + "gal", + "iant", + "northeast", + "Joint", + "canceled", + "toys", + "rein", + "Tumblr", + "pees", + "Aut", + "Police", + "aide", + "achieving", + "mund", + "Commercial", + "first", + "anticipate", + "iac", + "probation", + "hem", + "ports", + "Ker", + "supplier", + "Father", + "Anti", + "ashed", + "Table", + "bledon", + "unf", + "Rash", + "LeBron", + "Car", + "bu", + "Derek", + "accounted", + "Pri", + "nings", + "receives", + "lev", + "bilateral", + "List", + "LG", + "Jazz", + "restored", + "battles", + "ials", + "occupied", + "repairs", + "radar", + "MLB", + "NC", + "flexible", + "Command", + "coat", + "Vir", + "Colts", + "BC", + "twin", + "prisoners", + "slowed", + "hop", + "Inn", + "conflicts", + "measured", + "autonomous", + "Bow", + "disc", + "inson", + "Sche", + "aire", + "SU", + "Peterson", + "drafted", + "Pelosi", + "Soon", + "mechanism", + "accountability", + "Northeast", + "fo", + "analytics", + "Everything", + "perceived", + "bers", + "celebrations", + "instruments", + "strip", + "Juventus", + "unfortunately", + "GA", + "wrestling", + "statue", + "vis", + "five", + "marine", + "Samuel", + "responsibilities", + "hill", + "recruit", + "referee", + "Rail", + "Eagle", + "Congressional", + "breathing", + "bass", + "hit", + "spreading", + "evacuated", + "intellectual", + "sovereign", + "ocked", + "slammed", + "formerly", + "arch", + "difficulty", + "AFC", + "Fresh", + "invite", + "oner", + "Mich", + "pitches", + "stock", + "initiated", + "Ku", + "Florence", + "yd", + "Fast", + "musician", + "Chile", + "anga", + "dairy", + "contractors", + "ador", + "Planning", + "ultra", + "prayer", + "suggestions", + "Ek", + "random", + "Sullivan", + "sensor", + "homicide", + "Income", + "settings", + "acknowledge", + "Stay", + "terminal", + "1991", + "West", + "hard", + "arc", + "combine", + "privately", + "barrier", + "median", + "whereas", + "Titans", + "incentives", + "historically", + "indictment", + "hiding", + "PDT", + "rebuild", + "hol", + "pour", + "airports", + "Edinburgh", + "appoint", + "Jul", + "confusion", + "dam", + "ork", + "calculated", + "hood", + "Temple", + "Yorkshire", + "EP", + "ented", + "apology", + "awi", + "facilitate", + "Sheffield", + "rides", + "compelling", + "Gonzalez", + "roll", + "ONG", + "UP", + "Aj", + "pen", + "Var", + "IPO", + "Animal", + "shifted", + "140", + "tobacco", + "El", + "ild", + "uncertain", + "Un", + "caps", + "recreational", + "Tu", + "enc", + "More", + "iko", + "Everton", + "Walk", + "murdered", + "pur", + "divisions", + "ivo", + "farming", + "courage", + "ped", + "crying", + "attributed", + "ée", + "implementing", + "Wang", + "speeds", + "alk", + "aming", + "eries", + "avoided", + "Messi", + "considerable", + "rt", + "inauguration", + "PH", + "soldier", + "ore", + "ollywood", + "otive", + "Auburn", + "Sav", + "Put", + "emphasis", + "af", + "owed", + "diagnosis", + "cart", + "assisted", + "Order", + "Estate", + "intends", + "Common", + "adventure", + "beliefs", + "lasting", + "cel", + "deployment", + "tra", + "Stories", + "quote", + "feared", + "convenience", + "optimism", + "scientist", + "Enterprise", + "Rex", + "Fel", + "poses", + "root", + "evacuation", + "presidents", + "Rather", + "grave", + "Heights", + "jumping", + "driven", + "aluminum", + "holders", + "boot", + "iber", + "precious", + "uation", + "FP", + "uses", + "commentary", + "advances", + "Nissan", + "bronze", + "inspire", + "starters", + "Evan", + "rah", + "body", + "crops", + "seeds", + "harsh", + "Homeland", + "enabled", + "ological", + "workshop", + "chains", + "amps", + "amongst", + "Bear", + "certified", + "Julie", + "mountains", + "VA", + "fed", + "buyer", + "ahl", + "Bos", + "Crystal", + "quest", + "Stein", + "acceptable", + "unbeaten", + "iring", + "ural", + "uncomfortable", + "partial", + "sacrifice", + "Grande", + "arrangement", + "packaging", + "screen", + "mirror", + "sweep", + "connecting", + "panic", + "Jacksonville", + "Kremlin", + "origin", + "Brien", + "northwest", + "carriers", + "Riley", + "aud", + "appreciation", + "eliminated", + "Analyst", + "CR", + "firearm", + "accommodate", + "structural", + "appealed", + "charter", + "ressing", + "alike", + "white", + "slowdown", + "weigh", + "Palmer", + "ound", + "Conn", + "branches", + "ace", + "insists", + "yo", + "Lynn", + "CC", + "Within", + "coll", + "sustain", + "emerge", + "Battle", + "VER", + "aviation", + "enables", + "Production", + "Grove", + "nationally", + "Baldwin", + "rent", + "firearms", + "irm", + "considers", + "Cosby", + "McK", + "Ent", + "incumbent", + "iance", + "giants", + "kan", + "minimal", + "ivity", + "Say", + "Nass", + "lovely", + "Furthermore", + "displaced", + "contacts", + "NY", + "technological", + "ancy", + "ant", + "ope", + "FY", + "favorable", + "Virgin", + "casual", + "Lat", + "populations", + "romance", + "forgotten", + "fleeing", + "specialty", + "drill", + "applying", + "cocaine", + "rea", + "heroin", + "sweeping", + "Maj", + "troubled", + "colleague", + "edged", + "omes", + "Happy", + "´", + "militant", + "boy", + "aver", + "Yes", + "llo", + "supporter", + "Subscribe", + "Bird", + "Gibson", + "hill", + "newspapers", + "PHOTO", + "outing", + "define", + "ann", + "robot", + "regret", + "Could", + "raz", + "ceiling", + "organizers", + "Tw", + "criticised", + "Joh", + "Je", + "Bulls", + "teeth", + "Ranch", + "Andrea", + "conservatives", + "mag", + "vey", + "predecessor", + "JPMorgan", + "draws", + "umber", + "vaccine", + "Das", + "disappeared", + "Iron", + "litigation", + "vert", + "belong", + "Ret", + "owers", + "rain", + "controlled", + "Kil", + "rehab", + "Austria", + "privilege", + "bounce", + "bout", + "Islamist", + "taxi", + "ody", + ".'\"", + "dos", + "shire", + "accidents", + "demonstration", + "His", + "BO", + "ICE", + "van", + "File", + "Manning", + "ounded", + "directions", + "lled", + "offences", + "laptop", + "Universal", + "milestone", + "Narendra", + "notion", + "uns", + "Lower", + "midfield", + "outper", + "trans", + "Ja", + "three", + "Adds", + "pressures", + "prohibited", + "utilities", + "bes", + "Reporter", + "commodities", + "leton", + "slower", + "EE", + "auer", + "tablet", + "sl", + "iously", + "aiming", + "eland", + "NEXT", + "tered", + "IVE", + "onic", + "May", + "Military", + "Mark", + "lender", + "mate", + "aboard", + "they", + "respondents", + "conversion", + "securing", + "entity", + "Harbor", + "Cu", + "cats", + "ACC", + "Ibrahim", + "GL", + "invitation", + "cond", + "Records", + "Adrian", + "brave", + "mineral", + "sooner", + "satisfied", + "pets", + "notably", + "ı", + "marking", + "RO", + "Haw", + "Vis", + "marketplace", + "Nat", + "Forward", + "Left", + "aggravated", + "Close", + "acey", + "landmark", + "disruption", + "Challenge", + "Days", + "Coun", + "ahan", + "aides", + "South", + "Dylan", + "Ravens", + "Nature", + "lli", + "diplomats", + "350", + "Drake", + "tag", + "licensed", + "Denmark", + "cancel", + "instant", + "DI", + "punch", + "Jenkins", + "strengthening", + "des", + "-$", + "allegation", + "sizes", + "iza", + "mentally", + "Residents", + "acked", + "sensors", + ",'\"", + "illion", + "Champion", + "excessive", + "hum", + "Comp", + "rend", + "Lakes", + "burst", + "trainer", + "clearing", + "Silicon", + "350", + "DE", + "Gates", + "Horn", + "ests", + "Courtesy", + "bipartisan", + "habits", + "Alexa", + "walk", + "snapped", + "Eight", + "itis", + "zel", + "customs", + "southwest", + "vary", + "Because", + "payout", + "accelerate", + "Barr", + "tu", + "fined", + "cost", + "Theater", + "Corbyn", + "stem", + "undermine", + ".;", + "stays", + "breakthrough", + "turnover", + "hot", + "triumph", + "painted", + "Winnipeg", + "Kas", + "Stuart", + "irk", + "Am", + "trusted", + "aze", + "Late", + "accessories", + "memorable", + "Fool", + "rotation", + "Bulldogs", + "Chen", + "poised", + "Monte", + "Clarke", + "leading", + "venues", + "beneficial", + "Liam", + "Brothers", + "Need", + "conc", + "olly", + "Julian", + "ogue", + "founding", + "sidelines", + "declare", + "Member", + "examine", + "abs", + "boundaries", + "Brisbane", + "launches", + "lor", + "Ga", + "thr", + "expected", + "wal", + "Barnes", + "clashes", + "content", + "Clemson", + "iger", + "Mar", + "accord", + "southeast", + "ģ", + "Starbucks", + "osing", + "seasonal", + "icking", + "loyalty", + "tent", + "Dy", + "evident", + "lobby", + "tours", + "bombing", + "uations", + "rises", + "demonstrations", + "WATCH", + "pin", + "deb", + "Draft", + "rog", + "seal", + "Performance", + "LGBT", + "sed", + "gig", + "nan", + "rainfall", + "fabric", + "manages", + "lifting", + "Magazine", + "Criminal", + "hikes", + "catching", + "1989", + "OG", + "disappointment", + "ir", + "EV", + "stown", + "pass", + "120", + "medals", + "Simmons", + "inaugural", + "Corn", + "motorcycle", + "lets", + "Skype", + "ét", + "scary", + "opp", + "thirds", + "Mohamed", + "teenagers", + "ANK", + "server", + "outs", + "dishes", + "four", + "dr", + "Ot", + "Sandy", + "Shane", + "orters", + "SH", + "touching", + "Nike", + "HBO", + "driving", + "plug", + "Baseball", + "eling", + "hn", + "ulate", + "eed", + "Christine", + "Globe", + "ethics", + "Trevor", + "iya", + "360", + "awaiting", + "counterpart", + "subsidies", + "pointers", + "spy", + "ILL", + "takeover", + "Beyond", + "surprisingly", + "TION", + "Song", + "ni", + "commonly", + "jack", + "substitute", + "ews", + "recalls", + "Commons", + "sin", + "del", + "Mod", + "pressing", + "Television", + "Inside", + "ª", + "backlash", + "credible", + "Jenner", + "Pu", + "Stevens", + "WE", + "Last", + "insurers", + "Join", + "bled", + "digit", + "flooded", + "Shore", + "Trophy", + "zing", + "Immigration", + "superior", + "IAN", + "casino", + "enabling", + "meantime", + "performers", + "proportion", + "lawmaker", + "Conf", + "convert", + "farmer", + "bu", + "GE", + "Representative", + "Bannon", + "Help", + "PT", + "formed", + "Superintendent", + "frustrating", + "Register", + "Political", + "boots", + "Ru", + "Sha", + "instrument", + "tor", + "Belt", + "Walsh", + "recipe", + "ilt", + "Clean", + "iors", + "twenty", + "iler", + "nder", + "winger", + "wheat", + "Aviation", + "corrupt", + "connectivity", + "Ven", + "order", + "esc", + "break", + "metals", + "traditionally", + "bell", + "violating", + "rough", + "introducing", + "guided", + "Mol", + "desert", + "Bree", + "Le", + "Zone", + "Glass", + "EUR", + "Yahoo", + "laps", + "differ", + "Hold", + "timely", + "successor", + "comic", + "bears", + "licence", + "reject", + "sophisticated", + "Too", + "objectives", + "Id", + "urers", + "raid", + "COM", + "elect", + "Hampshire", + "lens", + "designers", + "presently", + "RCMP", + "Egyptian", + "Walter", + "Wallace", + "2025", + "utics", + "ried", + "refuse", + "siblings", + "Nothing", + "dressing", + "nerve", + "AST", + "uncertainties", + "tale", + "Talk", + "issuing", + "shot", + "Tak", + "acid", + "Nintendo", + "wash", + "pd", + "Claire", + "Scot", + "suits", + "Bayern", + "gest", + "applicable", + "interaction", + "Enforcement", + "Rohingya", + "jan", + "united", + "Coalition", + "legislators", + "detectives", + "Sing", + "Between", + "Poly", + "pool", + "mal", + "reply", + "schemes", + "Holmes", + "Senators", + "Verizon", + "welcoming", + "Cricket", + "Marco", + "Years", + "Living", + "counterparts", + "Paradise", + "Trad", + "#", + "iw", + "Soccer", + "umbled", + "deceased", + "heim", + "evaluation", + "wrap", + "mild", + "aji", + "UCLA", + "Native", + "president", + "Xbox", + "enterprises", + "Slam", + "oga", + "Rock", + "piece", + "Coleman", + "comparable", + "uba", + "provinces", + "Formula", + "ipt", + "ô", + "tick", + "IMF", + "anch", + "atta", + "rew", + "However", + "LS", + "etta", + "Customs", + "SU", + "publishing", + "inch", + "kills", + "¤", + "Sus", + "Beth", + "steam", + "jpg", + "pointer", + "turnovers", + "powder", + "USB", + "Wildlife", + "Direct", + "atively", + "Ferrari", + "pleasure", + "Matthews", + "ski", + "ography", + "Vermont", + "Margaret", + "Munich", + "layer", + "Property", + "economics", + "Crew", + "UK", + "unnecessary", + "Glasgow", + "sealed", + "clarity", + "surplus", + "Canyon", + "Apart", + "acceptance", + "Ellis", + "uster", + "rid", + "Hawks", + "statewide", + "threaten", + "Jail", + "inclusive", + "mud", + "pat", + "bitter", + "alternatives", + "affiliate", + "evaluate", + "Baby", + "perception", + "tim", + "refusing", + "grey", + "arguably", + "firmly", + "Dark", + "excuse", + "Raymond", + "ballots", + "inton", + "125", + "Catherine", + "sacks", + "Deb", + "workout", + "web", + "batteries", + "breaking", + "ML", + "unacceptable", + "Valentine", + "YOU", + "RT", + "jurisdiction", + "examined", + "strom", + "Pocket", + "cement", + "universal", + "Oz", + "kit", + "churches", + "suburban", + "Kushner", + "Davidson", + "Sports", + "email", + "realistic", + "intend", + "Grey", + ",''", + "scholarship", + "philosophy", + "wheels", + "motivation", + "eway", + "match", + "Date", + "John", + "controlling", + "750", + "aven", + "filmed", + "160", + "Brock", + "Details", + "logistics", + "assumptions", + "Step", + "fails", + "Notre", + "juice", + "counting", + "photograph", + "fortunate", + "establishing", + "NJ", + "Workers", + "Quinn", + "Heather", + "timeline", + "imported", + "NASCAR", + "exercises", + "searched", + "Ralph", + "alf", + "gene", + "dependent", + "én", + "iate", + "Bristol", + "hung", + "tropical", + "intensity", + "Idaho", + "Mull", + "suite", + "blockchain", + "cz", + "ovich", + "worn", + "LE", + "AV", + "emi", + "identification", + "tunnel", + "ARE", + "Arm", + "outrage", + "twist", + "uka", + "Gra", + "jets", + "Thus", + "compound", + "financially", + "2019", + "asse", + "spare", + "Noah", + "Made", + "Mom", + "phenomenon", + "nurses", + "outlined", + "polit", + "Carm", + "leagues", + "math", + "modified", + "willingness", + "Amanda", + "grandfather", + "Of", + "DR", + "dip", + "RAM", + "Christie", + "argues", + "EX", + "Nine", + "Scroll", + "THIS", + "Pro", + "keys", + "processor", + "scam", + "Training", + "honey", + "Ĵ", + "facebook", + "Legal", + "aging", + "spiritual", + "Host", + "lung", + "USC", + "dirt", + "fe", + "after", + "Diana", + "ounce", + "date", + "Finals", + "Ķ", + "thorough", + "viable", + "anytime", + "fost", + "orter", + "ware", + "Holland", + "Mand", + "Send", + "2013", + "Volkswagen", + "suitable", + "ifies", + "comedian", + "neighbours", + "Know", + "curious", + "Twenty", + "Prevention", + "Stephanie", + "pilots", + "stored", + "dire", + "fits", + "ision", + "Shell", + "shifts", + "pepper", + "attendees", + "Name", + "hers", + "rip", + "watchdog", + "andy", + "bio", + "publisher", + "powered", + "CM", + "rian", + "Rand", + "wise", + "Jesse", + "ladies", + "Metropolitan", + "Micro", + "kicking", + "meg", + "clouds", + "trim", + "wear", + "ML", + "consists", + "rig", + "honestly", + "GS", + "Nicholas", + "cope", + "publish", + "working", + "bur", + "Nar", + "olds", + "aja", + "Sad", + "clicking", + "bids", + "Zuckerberg", + "900", + "exam", + "ivers", + "pray", + "reader", + "Seth", + "inem", + "confront", + "stra", + "AW", + "Gian", + "accordance", + "interact", + "Sharks", + "fireworks", + "gment", + "illy", + "const", + "ARY", + "prizes", + "shoulders", + "accessed", + "ecosystem", + "licensing", + "La", + "dedication", + "dé", + "youths", + "lem", + "toy", + "Prom", + "ounding", + "rod", + "1000", + "ishes", + "Over", + "gaps", + "missions", + "railway", + "Day", + "orp", + "Schumer", + "eclipse", + "shell", + "BY", + "Many", + "Record", + "drunk", + "ayan", + "suggestion", + "defenders", + "Newton", + "disputes", + "evolution", + "credibility", + "Tenn", + "plain", + "size", + "cont", + "lone", + "fingers", + "BUR", + "Investigation", + "Qualcomm", + "var", + "countless", + "Rebecca", + "½", + "abi", + "reflecting", + "Turn", + "interactive", + "incentive", + "second", + "offs", + "Berkeley", + "Texans", + "heated", + "scorer", + "Sharif", + "migrant", + "west", + "Holiday", + "wrist", + "chairs", + "recommends", + "Wildcats", + "Ped", + "Quarter", + "IV", + "Arch", + "standings", + "bombs", + "capped", + "Can", + "caring", + "Lah", + "lim", + "dragged", + "Beat", + "DB", + "aired", + "jeans", + "action", + "generating", + "Gir", + "risk", + "lon", + "stage", + "âĤ¬", + "earing", + "Together", + "reun", + "Corey", + "Bak", + "prestigious", + "applicants", + "here", + "Mattis", + "ridiculous", + "Less", + "rains", + "presenting", + "anti", + "disabilities", + "apartments", + "storm", + "Hem", + "habit", + "Ruth", + "NPR", + "nut", + "appreciated", + "separation", + "uda", + "minus", + "Photos", + "blew", + "Voice", + "rallies", + "fond", + "Taking", + "yt", + "FE", + "Tory", + "ressed", + "Ly", + "rocks", + "Rah", + "elementary", + "nis", + "Presidential", + "nutrition", + "baseman", + "superstar", + "Wa", + "lar", + "staged", + "Learn", + "broadcaster", + "boasts", + "doubts", + "rum", + "bare", + "cap", + "climbing", + "Select", + "Cant", + "Nord", + "Beck", + "Kad", + "ello", + "enforce", + "Ze", + "ked", + "elly", + "LED", + "Operations", + "Luk", + "certificate", + "deter", + "spill", + "grain", + "league", + "Up", + "Kid", + "using", + "Jays", + "occasionally", + "MI", + "yes", + "detect", + "propaganda", + "neighboring", + "sub", + "avan", + "Astros", + "oti", + "threatening", + "shorter", + "INGS", + "feeding", + "elevated", + "Wenger", + "undergo", + "psychological", + "autom", + "NP", + "anks", + "Nokia", + "drones", + "recognised", + "heroes", + "agen", + "parole", + "Bah", + "homeowners", + "Sweet", + "instances", + "Parish", + "SL", + "unw", + "delicious", + "¯", + "Investments", + "Philippine", + "inos", + "mes", + "bite", + "cornerback", + "Hat", + "deserved", + "ologists", + "[", + "wrongdoing", + "Trent", + "Ve", + "Deal", + "Mr", + "overs", + "honors", + "ITV", + "payroll", + "confused", + "elaborate", + "ange", + "World", + "Resort", + "ilia", + "Kr", + "conclude", + "First", + "DR", + "peer", + "runway", + "Potter", + "cons", + "bad", + "si", + "Climate", + "Holl", + "weighing", + "epidemic", + "Bible", + "hon", + "renew", + "gambling", + "Nationals", + "itable", + "Outlook", + "reactions", + "Cos", + "Dana", + "India", + "Airbus", + "power", + "watch", + "styles", + "ordinance", + "cam", + "invent", + "Durant", + "exchanged", + "yoga", + "Michel", + "Wyoming", + "Phase", + "Hannah", + "tem", + "fare", + "omer", + "trails", + "quietly", + "Fourth", + "wise", + "appetite", + "pedestrian", + "fierce", + "hin", + "ako", + "vacant", + "dynamics", + "bust", + "GT", + "century", + "permitted", + "fog", + "recruitment", + "Due", + "bro", + "sil", + "Opp", + "phrase", + "Chip", + "Base", + "jazz", + "enemies", + "remainder", + "bles", + "105", + "Gur", + "retiring", + "Cour", + "Si", + "inevitable", + "Advisory", + "Campaign", + "Peninsula", + "base", + "justify", + "inen", + "North", + "freezing", + "photography", + "appointments", + "Tree", + "Os", + "divide", + "MMA", + "declines", + "Abbott", + "ACH", + "Jah", + "spr", + "skilled", + "Try", + "ANT", + "ael", + "McN", + "tariff", + "generation", + "Mans", + "Or", + "raped", + "disability", + "nominations", + "happiness", + "LSU", + "Interstate", + "Dance", + "Making", + "bailout", + "oro", + "Obviously", + "inbox", + "football", + "hy", + "Case", + "entertaining", + "hardest", + "Opposition", + "flip", + "Pirates", + "anu", + "Klopp", + "ballistic", + "printed", + "NFC", + "UST", + "glasses", + "rum", + "Duncan", + "hal", + "preview", + "BER", + "dec", + "sustainability", + "aff", + "hungry", + "service", + "avi", + "sometime", + "mod", + "Lib", + "oko", + "fundraiser", + "crowded", + "mates", + "creativity", + "Hell", + "treaty", + "Software", + "Randy", + "Polish", + "sa", + "ardi", + "cab", + "Camera", + "licenses", + "1988", + "continuous", + "paired", + "tally", + "grip", + "cho", + "surged", + "podium", + "contrary", + "SL", + "Researchers", + "cing", + "mi", + "disputed", + "grades", + "severely", + "McL", + "ondo", + "shelters", + "domain", + "Switch", + "testify", + "case", + "omet", + "atch", + "Aff", + "casting", + "berger", + "intimate", + "erc", + "plan", + "Past", + "Ut", + "apologized", + "Det", + "alle", + "whilst", + "pel", + "execute", + "harmful", + "RB", + "onda", + "Ful", + "II", + "Those", + "cryptocurrency", + "realise", + "Athens", + "Application", + "ORD", + "midst", + "Sem", + "messaging", + "cousin", + "Marsh", + "Almost", + "uto", + "wire", + "Managing", + "sends", + "Derby", + "pad", + "devoted", + "Working", + "Westminster", + "dirty", + "ements", + "Lew", + "door", + "advisor", + "ival", + "subscribe", + "credited", + "pressed", + "brick", + "rehabilitation", + "\"[", + "erry", + "transformed", + "arp", + "receivers", + "Fan", + "Kris", + "Charlottesville", + "ste", + "constructed", + "broadly", + "Better", + "Janet", + "enthusiasm", + "Irving", + "Const", + "Everyone", + "agn", + "Crawford", + "regards", + "Burns", + "jokes", + "erg", + "ARD", + "apped", + "travelled", + "Poor", + "Holly", + "container", + "infected", + "lean", + "Would", + "magnitude", + "Dou", + "minded", + "pastor", + "wherever", + "ulation", + "1986", + "Megan", + "graphic", + "talents", + "kn", + "EC", + "McM", + "Kon", + "eni", + "Esc", + "inas", + "Nom", + "chasing", + "arl", + "Hungary", + "mainland", + "Dist", + "utes", + "rubber", + "iat", + "Morrison", + "ushing", + "iny", + "copies", + "Fat", + "agged", + "floating", + "Curtis", + "fatally", + "Manuel", + "graduates", + "nar", + "Kenny", + "retreat", + "retro", + "Pierre", + "listed", + "Dale", + "ding", + "intentions", + "sentences", + "Sere", + "invasion", + "premiums", + "Gardner", + "shipments", + "col", + "bell", + "ilo", + "worthy", + "interceptions", + "complain", + "icle", + "Tah", + "Mt", + "Syracuse", + "Since", + "aches", + "Cand", + "interactions", + "Shawn", + "nc", + "theaters", + "ART", + "Th", + "alter", + "aley", + "imo", + "responders", + "kan", + "Darren", + "deliveries", + "PI", + "125", + "laughing", + "Patterson", + "infections", + "tur", + "130", + "hackers", + "warn", + "freeze", + "screaming", + "Echo", + "Dom", + "MAN", + "Joy", + "beneath", + "Half", + "patent", + "ugly", + "lip", + "nominees", + "Grade", + "influenced", + "abilities", + "limiting", + "smell", + "esc", + "Bernard", + "cs", + "Myers", + "oted", + "Black", + "lim", + "sworn", + "Blair", + "anes", + "Event", + "mature", + "positioned", + "erupted", + "grand", + "Tell", + "backdrop", + "yeah", + "Clear", + "significance", + "patience", + "Wing", + "horrible", + "deploy", + "ipe", + "bitcoin", + "committing", + "dismiss", + "Blood", + "Meyer", + "selling", + "regarded", + "lottery", + "Luther", + "pipe", + "cro", + "ANC", + "Solar", + "similarly", + "ham", + "Honor", + "tar", + "gin", + "Armstrong", + "browser", + "agon", + "via", + "entries", + "infl", + "graduation", + "alleges", + "Loading", + "superb", + "ially", + "administrator", + "uls", + "artistic", + "ANGEL", + "Bang", + "fossil", + "¨", + "poly", + "Guardiola", + "Perth", + "educate", + "Cl", + "committees", + "forthcoming", + "adjustments", + "count", + "incoming", + "brook", + "Minneapolis", + "gown", + "Croatia", + "host", + "competitor", + "lyrics", + "belonging", + "Frances", + "Haley", + "Bruins", + "mask", + "Pv", + "dollar", + "bowling", + "jewelry", + "Julia", + "broadband", + "Bhar", + "Armed", + "vy", + "government", + "kov", + "premises", + "jersey", + "applies", + "Freeman", + "grows", + "Equity", + "materially", + "figured", + "ience", + "majors", + "Ye", + "Hey", + "oned", + "aping", + "toilet", + "Connor", + "avoiding", + "pos", + "Once", + "Rockets", + "Snapchat", + "Go", + "solidarity", + "Affordable", + "dial", + "Omar", + "xt", + "Vatican", + "anta", + "Superior", + "beaches", + "Ki", + "Ã¥", + "KY", + "gro", + "Empire", + "occurs", + "joked", + "quotes", + "Saskatchewan", + "pert", + "maintains", + "olt", + "upgrades", + "Cho", + "Alexis", + "Hundreds", + "Bud", + "centuries", + "Investor", + "Gomez", + "conceded", + "expressing", + "IBM", + "advancing", + "Dollar", + "jer", + "exceed", + "author", + "rist", + "seat", + "Primary", + "Forbes", + "Alzheimer", + "devastated", + "awful", + "Studio", + "bullpen", + "mobility", + "analyze", + "lie", + "AFP", + "iche", + "Royals", + "coupled", + "dug", + "Ring", + "environments", + "national", + "Congo", + "alleging", + "wn", + "ulating", + "ur", + "reaches", + "Pine", + "threshold", + "tournaments", + "heating", + "Gard", + "Hamas", + "«", + "Holding", + "possibilities", + "Hassan", + "Mohammad", + "offenders", + "automated", + "realised", + "ouse", + "building", + "Dub", + "Geneva", + "facial", + "Restaurant", + "Ng", + "tot", + "grace", + "CP", + "poster", + "hart", + "Ni", + "reaff", + "prov", + "111", + "Aid", + "scrap", + "izers", + "ogen", + "tissue", + "vibrant", + "rider", + "CD", + "Kitchen", + "genre", + "¬", + "depth", + "kind", + "endorsed", + "simultaneously", + "intern", + "Drag", + "embraced", + "counted", + "uj", + "Og", + "physician", + "IR", + "IST", + "Kir", + "hacking", + "Sources", + "astic", + "growing", + "Wake", + "hint", + "compiled", + "reign", + "cinema", + "boosting", + "accommodation", + "Europa", + "subsidiaries", + "closures", + "Bil", + "Bou", + "wh", + "Aw", + "FT", + "hole", + "Nova", + "NSW", + "rap", + "encourages", + "GR", + "ds", + "Muk", + "Survey", + "Reagan", + "oning", + "neighbouring", + "McCl", + "acht", + "finishes", + "Esp", + "pat", + "destinations", + "Wagner", + "confronted", + "square", + "pie", + "brand", + "hl", + "absent", + "surf", + "rifle", + "SS", + "Death", + "wich", + "beds", + "Lock", + "Agu", + "atives", + "jee", + "oral", + "budgets", + "inspiring", + "IONS", + "works", + "spirits", + "cabin", + "satisfaction", + "voluntary", + "Municipal", + "deportation", + "Writer", + "VI", + "VERTISEMENT", + "/.", + "Southampton", + "aces", + "Helen", + "Hum", + "110", + "garbage", + "through", + "kingdom", + "MT", + "augh", + "bizarre", + "Starting", + "wooden", + "Progress", + "iron", + "sten", + "Sergio", + "HR", + "turnout", + "Americas", + "Sara", + "agrees", + "apper", + "bra", + "recycling", + "oom", + "flee", + "distinct", + "IAL", + "aha", + "fever", + "Partnership", + "Yu", + "Pixel", + "Block", + "Melissa", + "igg", + "decides", + "Norman", + "mas", + "held", + "PD", + "sheer", + "Dim", + "Cass", + "columnist", + "Bros", + "turnaround", + "Value", + "Bachelor", + "awn", + "assignment", + "ested", + "Judiciary", + "diamond", + "mus", + "indigenous", + "lines", + "1984", + "igroup", + "ict", + "Jaguars", + "lun", + "profiles", + "computing", + "Belgian", + "Lloyd", + "Going", + "disp", + "1987", + "eder", + "Vin", + "govern", + "blend", + "Sebastian", + "Midwest", + "iga", + "spl", + "topping", + "networking", + "Emer", + "oxygen", + "Interest", + "Moy", + "trader", + "bay", + "sticking", + "Movement", + "bidding", + "tax", + "academy", + "MO", + "Spirit", + "healing", + "wen", + "Prix", + "cal", + "Operating", + "instantly", + "Tonight", + "sacked", + "automation", + "umps", + "Ney", + "March", + "Buck", + "concentration", + "Here", + "travelers", + "protective", + "Moody", + "entrepreneur", + "fac", + "kowski", + "preparations", + "dominate", + "spray", + "disturbing", + "Fraser", + "Cody", + "ashi", + "Pel", + "risky", + "awkward", + "VA", + "ails", + "angle", + "undergoing", + "albums", + "afterwards", + "Naw", + "uge", + "enter", + "Sussex", + "Recently", + "likelihood", + "large", + "snaps", + "ibr", + "Malcolm", + "cru", + "altogether", + "setup", + "torture", + "fiber", + "quarterbacks", + "Getting", + "ipping", + "Norwegian", + "Miles", + "Arnold", + "Disease", + "tends", + "ife", + "Caroline", + "navigate", + "brush", + "Associates", + "bath", + "Centers", + "MC", + "taxpayer", + "comp", + "accomplish", + "Traffic", + "Bru", + "greenhouse", + "Malaysian", + "Pur", + "ased", + "Knicks", + "aters", + "alt", + "ICK", + "calculations", + "mindset", + "unch", + "gu", + "steadily", + "fiction", + "Pap", + "forming", + "Actor", + "Berry", + "imp", + "Upper", + "assessed", + "lawn", + "Roh", + "clearance", + "funded", + "pret", + "Hom", + "VS", + "Tourism", + "Ry", + "Gonz", + "Studios", + "anchor", + "recognise", + "cooperate", + "enny", + "aza", + "Meet", + "eventual", + "SW", + "Counsel", + "Save", + "lucrative", + "slim", + "Greens", + "chemistry", + "Sheikh", + "bridges", + "business", + "Saf", + "Gy", + "protocol", + "nephew", + "Brands", + "Culture", + "orship", + "(£", + "Dell", + "astics", + "proving", + "Mann", + "aca", + "indoor", + "Uganda", + "Romney", + "Stage", + "ward", + "Amber", + "haw", + "tw", + "bullying", + "CAR", + "associates", + "Hopkins", + "suburb", + "aggressively", + "postponed", + "bas", + "burglary", + "Found", + "floors", + "Any", + "jam", + "visibility", + "benefited", + "Aud", + "aying", + "iku", + "Pas", + "GPS", + "Owens", + "reluctant", + "Olivia", + "ols", + "emotion", + "Heavy", + "hostile", + "favorites", + "feat", + "Cord", + "GO", + "indicted", + "idal", + "IL", + "Ħ", + "acer", + "ICH", + "oda", + "recipients", + "tribal", + "resist", + "Critics", + "sang", + "Math", + "Brighton", + "Kw", + "limitations", + "interception", + "onde", + "Robertson", + "enjoys", + "site", + "wings", + "Celtic", + "relaxed", + "Share", + "warrants", + "oco", + "critically", + "GC", + "cute", + "laying", + "itude", + "Mediterranean", + "watches", + "disagree", + "Return", + "ARC", + "people", + "twelve", + "overdose", + "Lot", + "FROM", + "Peters", + "administrators", + "slam", + "jar", + "OH", + "Initiative", + "teamed", + "Majority", + "June", + "Plaza", + "lake", + "glimpse", + "rings", + "os", + "mentor", + "have", + "languages", + "uncle", + "agu", + "Wine", + "Category", + "Ing", + "contests", + "Rosen", + "Whatever", + "denying", + "ean", + "spec", + "grad", + "tenants", + "show", + "Gregory", + "contention", + "unanimously", + "Pin", + "fa", + "Pink", + "switched", + "acre", + "Trading", + "VP", + "Maple", + "Neill", + "discounts", + "alls", + "sounded", + "rumours", + "Cre", + "hall", + "Tele", + "thankful", + "surveyed", + "UB", + "dignity", + "nod", + "misleading", + "TX", + "Burke", + "mounting", + "skies", + "besides", + "Garrett", + "tha", + "intelligent", + "tanks", + "apping", + "Rat", + "aint", + "entertain", + "Abdullah", + "sink", + "Lan", + "Manufacturing", + "NFL", + "themes", + "Haven", + "Davies", + "Kerr", + "Len", + "courtroom", + "failures", + "lately", + "Electronics", + "gorgeous", + "notification", + "2030", + "aved", + "deer", + "economic", + "Statistics", + "confrontation", + "governors", + "Haram", + "LGBTQ", + "processed", + "Duchess", + "downs", + "pork", + "humor", + "ocese", + "needing", + "midterm", + "Oval", + "corners", + "tablets", + "eds", + "vere", + "attacker", + "Paul", + "pee", + "Alice", + "renowned", + "09", + "ocking", + "creditors", + "Pedro", + "Phone", + "surveys", + "Welsh", + "cow", + "builds", + "000", + "Azerbaijan", + "Yad", + "infant", + "motorists", + "poorly", + "medications", + "stupid", + "Castro", + "user", + "antly", + "alty", + "Cond", + "issa", + "Ivan", + "costume", + "08", + "hence", + "dangers", + "bullish", + "Life", + "flavor", + "Charleston", + "bikes", + "workshops", + "arranged", + "contender", + "sequel", + "Plant", + "donor", + "factories", + "rict", + "ellen", + "robots", + "Wor", + "Directors", + "Peru", + "queen", + "Timothy", + "Too", + "observers", + "ears", + "bel", + "link", + "uns", + "homers", + "adjacent", + "confidential", + "stunned", + "iden", + "illed", + "ESS", + "convenient", + "Lindsey", + "por", + "upp", + "borrow", + "Ahmad", + "ORT", + "relate", + "Self", + "Vanguard", + "utter", + "Branch", + "Bolton", + "bat", + "outright", + "fighters", + "Bed", + "pes", + "inski", + "gunshot", + "printing", + "Sent", + "vern", + "harvest", + "bubble", + "refund", + "fuels", + "dive", + "diplomat", + "pile", + "Very", + "rot", + "Search", + "Joyce", + "Pruitt", + "Level", + "BP", + "Lac", + "had", + "expenditure", + "Madd", + "pockets", + "Clippers", + "Dear", + "Give", + "hal", + "vertical", + "wholesale", + "what", + "Springfield", + "ayed", + "Som", + "secrets", + "charts", + "iar", + "ibility", + "LAND", + "bearing", + "prom", + "tab", + "sheets", + "GL", + "endless", + "opening", + "Owen", + "underneath", + "Erik", + "DACA", + "steering", + "footprint", + "Roma", + "Ducks", + "Ellen", + "Professional", + "Gardens", + "goalie", + "shine", + "turmoil", + "hunger", + "âĢĭ", + "active", + "hey", + "blessed", + "ason", + "oping", + "Thousands", + "dose", + "Lor", + "evolved", + "charities", + "PE", + "Rub", + "ws", + "mist", + "Shen", + "biological", + "Tweet", + "collections", + "substantially", + "inner", + "battled", + "Cong", + "Hold", + "wp", + "wells", + "sake", + "unrest", + "Kurt", + "ripped", + "itation", + "neighbourhood", + "inv", + "cad", + "Cuban", + "Wealth", + "tuition", + "declaring", + "sch", + "orne", + "wondered", + "Chaff", + "dealer", + "Number", + "Mobile", + "scratch", + "prepares", + "Sens", + "Istanbul", + "Panama", + "Cay", + "allocation", + "itutional", + "har", + "Nazi", + "Sund", + "warehouse", + "backyard", + "Ill", + "unlawful", + "Reform", + "basement", + "Hi", + "Pictures", + "transfers", + "Sell", + "fluid", + "ambitions", + "wife", + "intensive", + "steals", + "festive", + "Hayes", + "restoration", + "branded", + "Journal", + "macro", + "console", + "Melania", + "Rahul", + "disposal", + "cult", + "petrol", + "tires", + "kidnapping", + "115", + "swap", + "Sud", + "blown", + "Hindu", + "Beckham", + "Gul", + "fixture", + "wisdom", + "mines", + "fort", + "rivers", + "Cyber", + "touches", + "race", + "relax", + "crashes", + "constituency", + "1979", + "bureau", + "interface", + "detected", + "Bio", + "highlighting", + "ames", + "corresponding", + "great", + "gray", + "advantages", + "ME", + "Abbas", + "naked", + "rington", + ".),", + "Face", + "third", + "transcript", + "ples", + "Good", + "Arctic", + "tolerance", + "reat", + "green", + "Mik", + "outreach", + "rolls", + "gen", + "supplied", + "guarantees", + "aug", + "semif", + "ounds", + "running", + "fitting", + "Risk", + "iveness", + "family", + "ti", + "Isaac", + "dump", + "Patricia", + "passport", + "Rhode", + "Who", + "log", + "stat", + "rat", + "ango", + "SB", + "Maur", + "smiling", + "strikeouts", + "pupils", + "complications", + "Advanced", + "Monetary", + "Tall", + "ALL", + "contributor", + "Advertising", + "horrific", + "competed", + "Kenneth", + "hailed", + "bones", + "bolster", + "Boss", + "hospitalized", + "Telegraph", + "Independence", + "dr", + "Hang", + "documented", + "subtle", + "invest", + "bounced", + "MAN", + "profession", + "Ń", + "excellence", + "Inspector", + "BL", + "disrupt", + "Winston", + "Communist", + "Sharon", + "mechanical", + "treats", + "desperately", + "Indy", + "Gi", + "Composite", + "Heath", + "aser", + "Cardiff", + "ilit", + "eased", + "prospective", + "commissioned", + "tire", + "align", + "gesture", + "weakened", + "URE", + "SN", + "nationals", + "relies", + "IRS", + "Count", + "medicines", + "congress", + "stranger", + "Qu", + "lessly", + "Queens", + "Alleg", + "uing", + "Wy", + "Miguel", + "idi", + "civic", + "Petro", + "endo", + "Obviously", + "reflection", + "Stop", + "Fitzgerald", + "placed", + "shore", + "correctly", + "NE", + "amy", + "CT", + "some", + "Mb", + "oi", + "Hogan", + "Innovation", + "Villa", + "CAN", + "Cemetery", + "into", + "questionable", + "creator", + "rug", + "semifinals", + "mission", + "cle", + "Waters", + "Nixon", + "BT", + "assuming", + "Jer", + "Clay", + "pack", + "Cool", + "may", + "decor", + "spike", + "Somalia", + "Karn", + "Damascus", + "Shares", + "sus", + "Moss", + "1985", + "superintendent", + "Results", + "spends", + "prom", + "shipped", + "laundering", + "Leslie", + "meteor", + "abandon", + "deliberately", + "Sentinel", + "fascinating", + "enrollment", + "Experts", + "Similarly", + "Cuomo", + "bor", + "une", + "neutral", + "hamstring", + "negotiated", + "zes", + "Leo", + "Doctor", + "curriculum", + "Focus", + "travels", + "beverage", + "Including", + "tz", + "type", + "Range", + "floods", + "coached", + "dominance", + "letico", + "Rafael", + "predictions", + "prosperity", + "Cav", + "clinics", + "Banking", + "Coming", + "ears", + "Kaepernick", + "Blvd", + "retained", + "isions", + "ko", + "ensemble", + "precise", + "compact", + "MD", + "Jet", + "ached", + "Tru", + "Bass", + "Icon", + "excluding", + "sur", + "construct", + "voiced", + "pan", + "inability", + "exc", + "mate", + "trailing", + "successive", + "bets", + "gauge", + "minorities", + "IND", + "Vel", + "GP", + "oid", + "bon", + "pred", + "dash", + "performer", + "occasional", + "aken", + "mes", + "America", + "liver", + "Sp", + "Big", + "wildfires", + "Jackie", + "Led", + "Finland", + "jurors", + "olic", + "urance", + "Edge", + "open", + "scenarios", + "glory", + "entry", + "Coffee", + "rep", + "Chand", + "Vas", + "Islamabad", + "bur", + "Fle", + "Edition", + "shoe", + "ï¸ı", + "**", + "tle", + "Eb", + "keeping", + "Basketball", + "Von", + "CF", + "MENT", + "amm", + "Fernando", + "compares", + "Double", + "convictions", + "atop", + "cops", + "remembers", + "lacking", + "dom", + "itate", + "Beauty", + "develops", + "Gor", + "functional", + "COUNTY", + "Upon", + "sprint", + "injection", + "minors", + "Tamil", + "Gat", + "101", + "ety", + "drum", + "tasked", + "pact", + "170", + "MR", + "Ramos", + "candy", + "Sc", + "iced", + "supermarket", + "worrying", + "sellers", + "Tag", + ".:", + "mixture", + "oting", + "Bl", + "Ll", + "Jal", + "ican", + "Bid", + "country", + "Strategy", + "adverse", + "plunged", + "Mit", + "stark", + "aton", + "booking", + "Tr", + "containers", + "vintage", + "Pit", + "surfaced", + "independently", + "detection", + "Beyon", + "casualties", + "stabbing", + "oved", + "barred", + "thereby", + "partnered", + "posing", + "Shannon", + "Chapel", + "technically", + "uous", + "»", + "ometer", + "wildfire", + "share", + "heart", + "ammunition", + "thrive", + "Stre", + "GP", + "cé", + "Monaco", + "goal", + "Um", + "HSBC", + "Hilton", + "Viv", + "Kell", + "decisive", + "motive", + "amo", + "feld", + "WH", + "iry", + "ulu", + "Schneider", + "campaigning", + "separately", + "igo", + "ED", + "Ramirez", + "metro", + "Patel", + "Chi", + "Audi", + "characteristics", + "restart", + "keyboard", + "SD", + "his", + "biz", + "Soft", + "Grammy", + "contested", + "weekends", + "112", + "cycling", + "healthier", + "ija", + "header", + "employ", + "İ", + "shortages", + "Ask", + "Ivanka", + "partisan", + "flowing", + "cave", + "ENS", + "ups", + "read", + "ouch", + "102", + "forming", + "bot", + "bie", + "enrolled", + "concussion", + "affidavit", + "mysterious", + "uries", + "Mang", + "authentic", + "metrics", + "Twins", + "prep", + "IJ", + "desired", + "Div", + "wall", + "Tab", + "compet", + "relied", + "inequality", + "manual", + "Bucks", + "agging", + "corporation", + "banner", + "graphics", + "accurately", + "Meeting", + "consult", + "ser", + "protesting", + "hurting", + "omed", + "tes", + "rode", + "startups", + "handing", + "Nest", + "consistency", + "anned", + "dem", + "Lyon", + "Competition", + "tricky", + "cos", + "Bengals", + "arry", + "underwent", + "Kit", + "à", + "uploads", + "skate", + "''", + "jun", + "Content", + "focused", + "lat", + "Exp", + "ought", + "nightmare", + "Expect", + "precisely", + "Monica", + "lobbying", + "Chester", + "Invest", + "Former", + "imminent", + "NL", + "comparing", + "Ches", + "ede", + "Nobel", + "mers", + "Kin", + "Boko", + "ount", + "thoroughly", + "scattered", + "sharing", + "markets", + "Mis", + "ambition", + "preference", + "effectiveness", + "rio", + "heavyweight", + "overt", + "anya", + "Kanye", + "ishi", + "rewards", + "uled", + "bach", + "emphasized", + "apologize", + "Recent", + "!!", + "animated", + "Exxon", + "fruits", + "stripped", + "fold", + "Indonesian", + "ller", + "dementia", + "kidney", + "halted", + "years", + "concerts", + "refers", + "Fri", + "Your", + "irl", + "leap", + "jud", + "Hugh", + "FO", + "sore", + "kil", + "Mate", + "cci", + "setback", + "tightening", + "keeper", + "Albany", + "policymakers", + "disorders", + "CBC", + "Diaz", + "maps", + "routinely", + "verify", + "bash", + "Jinping", + "disasters", + "Monroe", + "Louise", + "JP", + "Nevertheless", + "concessions", + "Pog", + "going", + "Fifth", + "Jill", + "ICT", + "FM", + "Sugar", + "Barb", + "midway", + "tin", + "Pic", + "PL", + "leaks", + "grief", + "tattoo", + "`", + "ment", + "Nu", + "marry", + "diving", + "1982", + "coin", + "Poc", + "starred", + "Riverside", + "sidelined", + "miners", + "STON", + "belongs", + "Santos", + "Technical", + "aco", + "advise", + "streams", + "cooler", + "HE", + "ordering", + "Task", + "ACT", + "Anton", + "certification", + "Leafs", + "TS", + "Serbia", + "azi", + "inks", + "EST", + "relay", + "°", + "disappearance", + "Romania", + "oven", + "owed", + "Strip", + "ulated", + "UC", + "ITE", + "bling", + "Then", + "ppy", + "unlimited", + "calories", + "merchandise", + "blonde", + "Spicer", + "performing", + "impl", + "plates", + "mosque", + "demon", + "ought", + "dumped", + "tracked", + "even", + "stabil", + "imet", + "Liga", + "ugh", + "ther", + "agar", + "architect", + "allocated", + "Joey", + "marathon", + "master", + "Bert", + "ast", + "Ebola", + "Conservation", + "nic", + "parallel", + "inmate", + "locate", + "distribute", + "guard", + "tackling", + "ential", + "vi", + "cups", + "rhythm", + "endured", + "Hub", + "ois", + "Liberals", + "Redskins", + "EP", + "Knox", + "fr", + "massacre", + "oka", + "compl", + "raft", + "Published", + "attraction", + "Stephens", + "ility", + "Pul", + "Capt", + "exploded", + "exceeded", + "lying", + "cal", + "Mart", + "paintings", + "inate", + "Brendan", + "fortune", + "onductor", + "physicians", + "Study", + "Bul", + "Modern", + "HD", + "Bour", + "tying", + "1967", + "lighter", + "toss", + "inspired", + "greeted", + "cycl", + "verified", + "merit", + "sign", + "lder", + "debts", + "Snyder", + "amendments", + "indicators", + "Dortmund", + "then", + "Listen", + "FB", + "ref", + "IoT", + "Brewers", + "Leadership", + "Nicolas", + "Body", + "sam", + "Advisor", + "cord", + "abuses", + "Portuguese", + "flown", + "VR", + "consumed", + "reass", + "alien", + "rivalry", + "REPORT", + "Rush", + "directing", + "searches", + "HP", + "Roll", + "Fay", + "Clare", + "haul", + "riot", + "settlements", + "norm", + "accelerated", + "Lok", + "clever", + "hyd", + "stats", + "Hull", + "kers", + "buys", + "uter", + "fue", + "https", + "UD", + "isolation", + "suspend", + "Rules", + "Circle", + "Hopefully", + "played", + "âĢ³", + "PRE", + "sim", + "edd", + "Properties", + "beans", + "revive", + "Bir", + "oug", + "mob", + "showdown", + "iman", + "pap", + "vol", + "wu", + "diver", + "pill", + "Marlins", + "Lamar", + "persistent", + "condolences", + "Thor", + "Ab", + "impress", + "Raptors", + "references", + "stiff", + "Bash", + "eding", + "murders", + "Gene", + "Manila", + "brokers", + "Ms", + "start", + "Dhabi", + "etz", + "submission", + "Schmidt", + "Personal", + "Beverly", + "Movie", + "Lamb", + "placement", + "folk", + "frequency", + "planted", + "twins", + "prov", + "rec", + "permanently", + "coordination", + "Cart", + "obstacles", + "literature", + "tu", + "chill", + "Reserved", + "lovers", + "Outside", + "slideshow", + "Gru", + "ty", + "salad", + "laboratory", + "Holt", + "103", + "urb", + "Organisation", + "Andrews", + "recipient", + "arch", + "bleeding", + "Pand", + "overturned", + "listened", + "clause", + "nationalist", + "resumed", + "Cout", + "Pride", + "layers", + "Bella", + "reversed", + "priest", + "FX", + "albeit", + "halfway", + "cotton", + "Carey", + "TE", + "OCK", + "buck", + "ributes", + "ea", + "fancy", + "Buc", + "bans", + "uters", + "liabilities", + "Sou", + "Bernie", + "intervene", + "food", + "NDP", + "insist", + "contracted", + "hawk", + "),\"", + "Dawn", + "mol", + "commissioners", + "stranded", + "overwhelmed", + "recipes", + "va", + "rad", + "scare", + "rez", + "eliminating", + "resc", + "Break", + "chn", + "delight", + "iot", + "freely", + "TI", + "Bluetooth", + "Month", + "Flor", + "Freddie", + "trailed", + "investigative", + "imposing", + "attracting", + "awk", + "Sherman", + "succeeded", + "vent", + "reconciliation", + "Cel", + "Throughout", + "Downtown", + "Brother", + "traditions", + "mir", + "stamp", + "tery", + "etti", + "isch", + "tic", + "banning", + "loss", + "Speedway", + "stalled", + "EN", + "ASH", + "thing", + "Appeals", + "rac", + "distress", + "Conservatives", + "Premium", + "usa", + "slump", + "imm", + "Supp", + "Wong", + "distant", + "104", + "tide", + "Norfolk", + "Yang", + "smashed", + "Barrett", + "inho", + "robbed", + "Farmers", + "filled", + "BT", + "autumn", + "temple", + "Jacobs", + "precipitation", + "Hours", + "Flight", + "beside", + "Ore", + "!)", + "Turnbull", + "pig", + "cooling", + "servers", + "oriented", + "locks", + "Sears", + "aving", + "Quick", + "Glob", + "Mining", + "horizon", + "arians", + "Om", + "writing", + "believing", + "bon", + "mounted", + "punt", + "ucci", + "uzz", + "cul", + "kiss", + "Ont", + "Cyprus", + "relying", + "piano", + "cure", + "continuously", + "Nobody", + "Bund", + "osis", + "Aurora", + "Bach", + "Kendall", + "echoed", + "iable", + "conscious", + "monster", + "omo", + "proof", + "Nate", + "filmmaker", + "Naj", + "vendor", + "Foot", + "Chang", + "Fest", + "selfie", + "enters", + "Conor", + "Mosul", + "WHAT", + "wa", + "Gamb", + "osta", + "cautioned", + "Tucker", + "Airways", + "visitor", + "·", + "Revolution", + "aching", + "earliest", + "Quality", + "shorts", + "ube", + "Operation", + "Sabha", + "strengths", + "ikes", + "sexy", + "rot", + "ibles", + "colours", + "THE", + "ailed", + "woke", + "Embassy", + "infamous", + "rov", + "State", + "âĢ¦.", + "pond", + "capt", + "fore", + "De", + "edited", + "self", + "Hey", + "portrait", + "Manufact", + "Stand", + "contenders", + "':", + "acker", + "withdrawn", + "Braves", + "Hosp", + "changing", + "Bag", + "adjustment", + "Cousins", + "AAP", + "fi", + "outdoors", + "lacked", + "BM", + "WHO", + "PST", + "Luck", + "assisting", + "Ground", + "Teen", + "Ole", + "embarrassing", + "Walt", + "Vision", + "Fal", + "Zoo", + "Worth", + "Floyd", + "Gujarat", + "tipped", + "fam", + "Dad", + "worship", + "tyre", + "rebuilding", + "qualities", + "Lives", + "beats", + "450", + "existed", + "Georg", + "poured", + "rows", + "Ox", + "Sid", + "mac", + "teaches", + "Eli", + "alla", + "downside", + "Bend", + "non", + "Armenia", + "cultures", + "Mae", + "duration", + "Athletics", + "juvenile", + "lid", + "bankers", + "overview", + "wy", + "orbit", + "Vs", + "because", + "Ps", + "Fran", + "touring", + "wary", + "106", + "laser", + "Vij", + "âĦ¢", + "surrender", + "press", + "rees", + "NO", + "Shortly", + "Kor", + "edu", + "hatred", + "tee", + "famously", + "keeper", + "ND", + "reduces", + "HC", + "hay", + "unnamed", + "Tes", + "attackers", + "Few", + "Richards", + "1968", + "speeches", + "cybersecurity", + "Infrastructure", + "07", + "ENCE", + "uties", + "anxious", + "Gang", + "announcements", + "lette", + "oret", + "Rockies", + "Employees", + "Thrones", + "hugely", + "clin", + "Hob", + "fraction", + "Official", + "Mariners", + "Else", + "sanctuary", + "Photograph", + "reopen", + "lf", + "hm", + "vest", + "speeding", + "tooth", + "Shi", + "Title", + "Mes", + "Jobs", + "fair", + "Danish", + "Malik", + "laughed", + "navy", + "Actress", + "Williamson", + "overs", + "reckless", + "jo", + "otic", + "assaulting", + "pri", + "Pi", + "lesser", + "tit", + "dat", + "nail", + "Marathon", + "Gren", + "Dol", + "jointly", + "amended", + "mine", + "Bashar", + "Hyundai", + "uncovered", + "educated", + "atti", + "pres", + "BRE", + "ya", + "Bank", + "odd", + "lit", + "Links", + "switching", + "itte", + "Sind", + "erved", + "**", + "positively", + "frankly", + "revenge", + "Trinity", + "CDC", + "threatens", + "hammer", + "NET", + "Mut", + "sy", + "unidentified", + "icken", + "drills", + "tense", + "foreigners", + "OST", + "ethical", + "Durham", + "Qual", + "territories", + "id", + "hor", + "enders", + "Mc", + "OV", + "percent", + "dom", + "upward", + "amb", + "visas", + "zan", + "Ãĥ", + "undocumented", + "suburbs", + "hydro", + "Job", + "Adelaide", + "oya", + "SR", + "Mick", + "consolidation", + "emotionally", + "Hop", + "Her", + "loses", + "Moto", + "eled", + "regulated", + "ental", + "encountered", + "hop", + "Trafford", + "sticks", + "veto", + "expose", + "stretched", + "fin", + "inance", + "chair", + "Gareth", + "Pil", + "Hammond", + "serial", + "omy", + "cellphone", + "Clara", + "reacted", + "Nic", + "Homes", + "Broadcasting", + "Fut", + "Supply", + "assing", + "Newman", + "charitable", + "Clayton", + "sovereignty", + "convincing", + "Principal", + "Higher", + "Cut", + "Carrie", + "Spot", + "Sometimes", + "Jar", + "Consider", + "ieu", + "refinery", + "bloody", + "wheel", + "cryptocurrencies", + "Fund", + "Sunderland", + "Events", + "âĢĭ", + "accidentally", + "deep", + "franc", + "bec", + "Hartford", + "stellar", + "wright", + "kick", + "UG", + "Beast", + "refusal", + "Roberto", + "Dixon", + "Diane", + "name", + "asts", + "Charter", + "fueled", + "contents", + "accessing", + "troubles", + "tops", + "debuted", + "icating", + "investigator", + "subscribing", + "coordinated", + "Fil", + "six", + "teen", + "withdrew", + "Gilbert", + "1983", + "arsity", + "imagination", + "handgun", + "Alibaba", + "bug", + "107", + "COMP", + "Something", + "reliability", + "FCC", + "Fowler", + "singled", + "nom", + "knocking", + "meddling", + "determining", + "reports", + "shade", + "SN", + "anto", + "complaining", + "Nan", + "WS", + "youngsters", + "Il", + "Kaw", + "Prop", + "Cell", + "Hurricanes", + "publicity", + "Xin", + "rial", + "ICO", + "supervision", + "Spotify", + "Newport", + "prince", + "anche", + "subscriber", + "Vic", + "ACT", + "Raf", + "Acting", + "collusion", + "pet", + "isl", + "commerce", + "Health", + "Abraham", + "pri", + "lightweight", + "insurer", + "Like", + "helmet", + "evac", + "look", + "Naval", + "160", + "Fleet", + "vol", + "expired", + "Klein", + "Emmy", + "ABLE", + "Morocco", + "Trip", + "uted", + "nos", + "Vista", + "mas", + "Rocky", + "Flint", + "enberg", + "Brow", + "signatures", + "polar", + "ajo", + "endorsement", + "reservations", + "LIN", + "anny", + "elli", + "last", + "oversee", + "cm", + "Oilers", + "Are", + "judiciary", + "onte", + "Track", + "supervisor", + "erk", + "isher", + "intact", + "slid", + "icals", + "paid", + "MAR", + "lement", + "Liu", + "Large", + "Wings", + "pect", + "Rum", + "analyzed", + "employs", + "arte", + "ims", + "Eventually", + "affiliated", + "hospitality", + "Sprint", + "resolutions", + "liquor", + "NAFTA", + "ANY", + "radiation", + "Prov", + "pause", + "TMZ", + "elbow", + "resilience", + "Parents", + "mus", + "Safe", + "interpretation", + "raced", + "IND", + "KR", + "hinted", + "Erin", + "Bahrain", + "credentials", + "eless", + "procurement", + "Webb", + "Lowe", + "Nak", + "Learning", + "zh", + "dipped", + "Suite", + "misdemeanor", + "ALE", + "strengthened", + "Sophie", + "confirms", + "rac", + "gey", + "shootout", + "ble", + "circles", + "Chef", + "comprised", + "Santiago", + "feud", + "beat", + "staffers", + "acute", + "ski", + "polled", + "Kur", + "Jen", + "Ultimately", + "anded", + "Honey", + "announces", + "amateur", + "around", + "functioning", + "group", + "Squ", + "Where", + "void", + "Sandra", + "isers", + "helicopters", + "Gym", + "Wol", + "mouth", + "subjected", + "ici", + "ually", + "Wash", + "Lindsay", + "Vers", + "jumps", + "neglect", + "Kuwait", + "fund", + "ĭ", + "ather", + "lly", + "ei", + "Although", + ".''", + "unhappy", + "pills", + "magical", + "dro", + "inviting", + "Johnston", + "oving", + "450", + "Merc", + "admitting", + "insisting", + "Cru", + "Resource", + "oir", + "complexity", + "Roth", + "Cher", + "July", + "raf", + "aggregate", + "helm", + "uclear", + "olan", + "offenses", + "Wolves", + "Fu", + "Pierce", + "emailed", + "Stra", + "pedestrians", + "ER", + "Conway", + "blowing", + "CLOSE", + "hab", + "Greene", + "confessed", + "Torres", + "Holocaust", + "repay", + "demonstrates", + "Pool", + "gent", + "deleted", + "$$", + "SO", + "dri", + "Neg", + "VP", + "PF", + "Prep", + "organizing", + "icker", + "manufactured", + "enson", + "adas", + "wines", + "machinery", + "specialists", + "Detective", + "DL", + "Op", + "quicker", + "Penguins", + "Engine", + "zone", + "sequence", + "Lost", + "warmer", + "Ethiopia", + "affirmed", + "fest", + "resses", + "soap", + "booth", + "notorious", + "amin", + "pursued", + "Cer", + "SB", + "livestock", + "trace", + "respects", + "arden", + "April", + "128", + "Said", + "ennial", + "namely", + "Bot", + "108", + "Lem", + "nell", + "confirming", + "logged", + "profound", + "elo", + "Chambers", + "RT", + "newer", + "sideline", + "Cardinal", + "este", + "narrowly", + "compromised", + "policing", + "porn", + "arc", + "learnt", + "INE", + "step", + "Domin", + "waist", + "boycott", + "mitted", + "iffs", + "ground", + "Materials", + "ceasefire", + "Right", + "Zen", + "estyle", + "Thank", + "OnePlus", + "MLS", + "constituents", + "oster", + "Prosecutor", + "priorit", + "Debbie", + "Expand", + "uv", + "integrate", + "immun", + "disciplinary", + "Imm", + "ja", + "gardens", + "Him", + "obe", + "hitter", + "bullets", + "evolving", + "Scientists", + "Michael", + "DO", + "unbelievable", + "looming", + "downturn", + "mentality", + "reopened", + "ash", + "Chapman", + "loop", + "UT", + "Tier", + "unaware", + "gratitude", + "performs", + "olk", + "\"(", + "lacks", + "instructed", + "Recreation", + "sample", + "requesting", + "Canada", + "supposedly", + "Hardy", + "holder", + "change", + "Dominic", + "Xavier", + "lig", + "candid", + "Rab", + "conferences", + "Burton", + "Dr", + "municipalities", + "crushed", + "seekers", + "Citizens", + "heightened", + "Casino", + "desktop", + "whoever", + "Impact", + "cocktail", + "philanthrop", + "SAN", + "Preston", + "obesity", + "restrict", + "Kab", + "Providence", + "scar", + "Chart", + "bosses", + "Rate", + "sav", + "pay", + "transplant", + "Noble", + "child", + "conclusions", + "FI", + "sack", + "experimental", + "holder", + "oca", + "herty", + "MT", + "catcher", + "LY", + "grams", + "reet", + "adaptation", + "humble", + "bot", + "identical", + "ication", + "ifer", + "Crow", + "regain", + "Lightning", + "kg", + "composed", + "correspondent", + "reunion", + "observe", + "comprising", + "impeachment", + "resh", + "lemon", + "Snap", + "proprietary", + "een", + "ourt", + "detective", + "labels", + "corridor", + "Clinic", + "arra", + "Pearl", + "informal", + "Und", + "Venezuelan", + "peninsula", + "defeating", + "syndrome", + "iere", + "spite", + "bag", + "aran", + "specialized", + "AA", + "Lyn", + "instrumental", + "Smith", + "pivotal", + "nightclub", + "Cob", + "colorful", + "artwork", + "1981", + "dawn", + "erville", + "uated", + "ief", + "linking", + "Ow", + "appreci", + "reductions", + "elling", + "salmon", + "bb", + "Phillip", + "yle", + "assure", + "discretion", + "efficiently", + "Mau", + "abil", + "intentionally", + "activated", + "immense", + "Strategic", + "cheating", + "Trend", + "Samantha", + "comple", + "hack", + "Serie", + "Text", + "stylish", + "Faith", + "GST", + "exterior", + "blessing", + "blanket", + "cooked", + "retaliation", + "tro", + "shelves", + "rose", + "Gram", + "sho", + "Argentine", + "clerk", + "specific", + "agreeing", + "standout", + "black", + "trending", + "violate", + "Get", + "ño", + "Opt", + "Frankfurt", + "Franco", + "eness", + "lining", + "zoo", + "oil", + "lia", + "rab", + "organize", + "woods", + "scan", + "urgency", + "occurring", + "reliance", + "concepts", + "eligibility", + "0000", + "Brief", + "abusive", + "Bench", + "rub", + "Dil", + "mount", + "maturity", + "Nut", + "nee", + "enc", + "gunfire", + "Kill", + "gates", + "flower", + "iol", + "shaped", + "undoubtedly", + "backgrounds", + "Complex", + "\":{\"", + "naming", + "monument", + "oh", + "embedded", + "bang", + "Kro", + "aggression", + "Mits", + "During", + "Ep", + "iners", + "Anaheim", + "rom", + "outgoing", + "fulfill", + "reminds", + "ren", + "à¤", + "Sue", + "refresh", + "lif", + "fil", + "Lead", + "regulate", + "Teachers", + "clarify", + "obs", + "blasted", + "Ax", + "flavors", + "mega", + "hurdles", + "inspector", + "Salvador", + "prescribed", + "renovation", + "OUR", + "util", + "Bradford", + "wasted", + "lineman", + "palm", + "icate", + "overseeing", + "otted", + "Rapids", + "justified", + "aby", + "extends", + "oath", + "bow", + "Rivera", + "Jan", + "Imran", + "forests", + "Shel", + "Brun", + "aerial", + "NOW", + "PAR", + "beverages", + "ettel", + "fragile", + "codes", + "Į", + "abel", + "Watch", + "road", + "dismissal", + "Rosa", + "crunch", + "²", + "innovations", + "habitat", + "forefront", + "Koch", + "Chevrolet", + "wheelchair", + "considerably", + "expenditures", + "texts", + "prompt", + "skating", + "petroleum", + "ICC", + "vit", + "fit", + "prolonged", + "Lucy", + "cho", + "rocked", + "Brom", + "freed", + "yours", + "Eden", + "monitored", + "asted", + "oversees", + "ieri", + "ideology", + "Fine", + "tering", + "Top", + "damp", + "uta", + "lethal", + "purple", + "udge", + "Chemical", + "Petersburg", + "warns", + "collectively", + "â", + "plaintiffs", + "Boris", + "sheep", + "oves", + "Author", + "campuses", + "destroying", + "gloves", + "cease", + "delegates", + "preceded", + "realDonaldTrump", + "forwards", + "erton", + "BuzzFeed", + "occupation", + "Legion", + "stir", + "shale", + "terrific", + "newborn", + "standoff", + "OWN", + "muscles", + "Herman", + "Liz", + "Experience", + "Success", + "Hispanic", + "CCTV", + "complement", + "Bing", + "prem", + "Johannes", + "dent", + "itar", + "Hein", + "Nicola", + "concludes", + "Khal", + "parish", + "shaking", + "Schw", + "mod", + "Lil", + "ña", + "Bog", + "Fight", + "gre", + "fel", + "heal", + "err", + "TM", + "airo", + "health", + "swings", + "tier", + "anka", + "ribune", + "emouth", + "Bloom", + "owing", + "Tech", + "dough", + "batch", + "Lion", + "Zamb", + "crashing", + "XL", + "ppers", + "Doctors", + "Sor", + "video", + "cigarettes", + "Boxing", + "constitute", + "concentrate", + "Armenian", + "semester", + "position", + "emic", + "NYC", + "Campus", + "alternate", + "exped", + "publishers", + "2015", + "unanimous", + "Previous", + "wellness", + "Creative", + "edy", + "AGE", + "Cavs", + "1978", + "fu", + "Tata", + "Choice", + "woes", + "Cable", + "~", + "Gem", + "consolidated", + "Manitoba", + "Cloud", + "rounded", + "Ventura", + "shark", + "dresses", + "traction", + "eda", + "div", + "dental", + "Wh", + "Gig", + "Boyd", + "Transit", + "televised", + "SON", + "Vince", + "closes", + "apt", + "Wheeler", + "Tyson", + "forensic", + "punished", + "seas", + "navigation", + "precedent", + "extremist", + "composite", + "PO", + "survivor", + "Vale", + "gars", + "HT", + "Riyadh", + "revival", + "Payne", + "collaborative", + "Customers", + "Pf", + "proves", + "erve", + "elev", + "Paper", + "chore", + "thriller", + "straw", + "cock", + "Gu", + "aligned", + "Chronicle", + "shouting", + "1976", + "lightning", + "worlds", + "Opening", + "enton", + "Ana", + "Gol", + "Techn", + "lis", + "orientation", + "Arri", + "PG", + "ross", + "sank", + "LOS", + "Allison", + "smiles", + "USD", + "kits", + "Bar", + "Bri", + "ounces", + "Nielsen", + "eno", + "109", + "norms", + "skip", + "180", + "monitors", + "2012", + "incorporate", + "mechanisms", + "Hack", + "Bomb", + "Gavin", + "Natalie", + "discusses", + "assembled", + "cognitive", + "owner", + "genuinely", + "disappear", + "AK", + "stal", + "soup", + "Finn", + "cares", + "finest", + "tuned", + "ende", + "Stefan", + "accompanying", + "î", + "Maybe", + "offender", + "TT", + "212", + "volleyball", + "needed", + "quo", + "dim", + "Historical", + "Lance", + "gmail", + "Gate", + "demonstrators", + "dy", + "cia", + "Steele", + "Joan", + "Kerala", + "KA", + "Electoral", + "paths", + "ø", + "Ne", + "accepts", + "lowering", + "portions", + "Valencia", + "festivals", + "generic", + "usk", + "Vernon", + "Orioles", + "renewal", + "belonged", + "breathe", + "220", + "recruited", + "logic", + "recreation", + "verbal", + "Haz", + "double", + "favourites", + "fundamentals", + "Soc", + "360", + "SO", + "alerted", + "briefed", + "Bruno", + "seating", + "freight", + "Amer", + "wished", + "table", + "growth", + "Went", + "hilarious", + "throat", + "bet", + "gon", + "ample", + "hee", + "Hood", + "Iceland", + "Ankara", + "iang", + "practicing", + "azer", + "leaf", + "hottest", + "marginal", + "revelations", + "Prices", + "Lar", + "times", + "handles", + "Naz", + "institute", + "translate", + "JP", + "soared", + "consume", + "Tap", + "Celebrity", + "Mayweather", + "Oracle", + "mor", + "ANA", + "paperwork", + "aste", + "dil", + "decorated", + "promotional", + "Merrill", + "appliances", + "COP", + "lips", + "Brennan", + "Mile", + "Networks", + "Comment", + "Ib", + "Agg", + "IDE", + "initiate", + "knockout", + "bargain", + "accordingly", + "bee", + "Gerald", + "problematic", + "trap", + "finalists", + "addy", + "would", + "strictly", + "Ramsey", + "downward", + "extract", + "famed", + "OUT", + "induct", + "Auckland", + "poetry", + "mos", + "Guinea", + "management", + "ohan", + "Guide", + "aily", + "umping", + "enacted", + "Eye", + "vision", + "umi", + "aped", + "bicycle", + "Houth", + "NAS", + "tapped", + "wer", + "otti", + "EA", + "surprises", + "Update", + "Pun", + "Miz", + "Oro", + "costumes", + "title", + "surviving", + "According", + "themed", + "Peoples", + "Se", + "associations", + "hett", + "Time", + "essay", + "mu", + "Score", + "Spani", + "SEE", + "males", + "rage", + "EU", + "Yellow", + "rupt", + "apparel", + "sweat", + "nearest", + "zman", + "anticipation", + "injuring", + "ousted", + "chan", + "Alert", + "ber", + "atal", + "Com", + "04", + "afterward", + "edge", + "Booker", + "lex", + "Whole", + "toughest", + "Maharashtra", + "lier", + "Tennis", + "handy", + "Metal", + "iTunes", + "Discovery", + "compassion", + "LIVE", + "economically", + "endangered", + "GO", + "mound", + "word", + "Touch", + "ogo", + "incomes", + "when", + "Aside", + "scandals", + "functionality", + "Aer", + "councils", + "denial", + "140", + "implied", + "outfits", + "suited", + "1973", + "Pizza", + "debates", + "record", + "hype", + "Rus", + "Robbie", + "touted", + "Sharp", + "beings", + "slavery", + "encies", + "Rooney", + "nan", + "raids", + "instructor", + "Market", + "shook", + "deliberate", + "Northwestern", + "Ess", + "whatsoever", + "Confederate", + "YS", + "Cameroon", + "Flip", + "Yeah", + "washing", + "mand", + "Lex", + "issuance", + "niche", + "fold", + "Wendy", + "hy", + "bucket", + "VW", + "Cairo", + "SK", + "Kang", + "intake", + "hills", + "anz", + "©", + "ugu", + "Fortunately", + "Marqu", + "imprisonment", + "oking", + "distributors", + "zie", + "stip", + "Wire", + "councillors", + "sue", + "Regardless", + "Enc", + "baking", + "Venture", + "intriguing", + "upheld", + "Active", + "genes", + "Dawson", + "Previously", + "Rac", + "metric", + "Files", + "iPhones", + "Welcome", + "burns", + "Screen", + "ashes", + "Apr", + "theories", + "san", + "Renault", + "Singer", + "founders", + "Russian", + "Belfast", + "imagined", + "Planet", + "Catalan", + "Rochester", + "evolve", + "OT", + "password", + "homelessness", + "backlog", + "presenter", + "fal", + "ISH", + "EM", + "icked", + "unlock", + "city", + "negotiation", + "dancers", + "dan", + "COL", + "VC", + "boat", + "overly", + "deal", + "lander", + "diss", + "ICS", + "fifty", + "owe", + "prisons", + "ifications", + "wo", + "Au", + "apiece", + "Courtney", + "1975", + "surpass", + "identities", + "integral", + "documentation", + "elegant", + "Ig", + "dear", + "113", + "Gupta", + "contentious", + "rish", + "clues", + "additions", + "ep", + "rus", + "centered", + "Phillies", + "father", + "borough", + "buttons", + "deported", + "REC", + "Already", + "eh", + "hur", + "upbeat", + "omen", + "detailing", + "wr", + "varied", + "Economics", + "ensures", + "Civic", + "unpaid", + "sold", + "Hil", + "Mult", + "Rising", + "Mini", + "neuro", + "penal", + "neighbour", + "Chavez", + "jew", + "VIP", + "Connor", + "Talking", + "correction", + "standpoint", + "roads", + "Wool", + "verification", + "mic", + "olf", + "exemption", + "filter", + "balloon", + "leases", + "ician", + "Spr", + "toe", + "unconstitutional", + "manslaughter", + "tossed", + "Meg", + "ATIONS", + "ACK", + "Rouge", + "Hansen", + "Hook", + "Out", + "Horse", + "Bath", + "Always", + "incorporated", + "conjunction", + "Fit", + "examining", + "wallet", + "ensured", + "acclaimed", + "ippers", + "beneficiaries", + "unexpectedly", + "exploit", + "Willie", + "comb", + "Walton", + "rica", + "icky", + "ate", + "Padres", + "rib", + "snacks", + "Fernandez", + "Machine", + "ction", + "illnesses", + "Hoffman", + "SpaceX", + "ju", + "swift", + "embark", + "Railway", + "measuring", + "agers", + "arsh", + "essence", + "angle", + "olive", + "Commander", + "iggs", + "rewarded", + "dispatched", + "playground", + "½", + "Programme", + "studios", + "skeptical", + "Olymp", + "Keys", + "Sunshine", + "amba", + "Donna", + "lightly", + "obtaining", + "poisoning", + "az", + "1972", + "unconscious", + "ECT", + "lied", + "Kaz", + "06", + "Moving", + "num", + "oral", + "assessments", + "scholarships", + "evacuate", + "Sunni", + "quake", + "fort", + "ques", + "Alonso", + "thread", + "squeeze", + "arat", + "oly", + "Alphabet", + "uting", + "icio", + "Retirement", + "ither", + "asleep", + "pairs", + "manufacture", + "Hazard", + "sidewalk", + "wears", + "Craft", + "emen", + "ieth", + "bypass", + "Lancaster", + "flour", + "charge", + "CLICK", + "potatoes", + "Karachi", + "valley", + "sights", + "fallout", + "ords", + "BN", + "sunshine", + "undertaken", + "contestants", + "accomplishments", + "conditioning", + "cel", + "Halifax", + "accent", + "***", + "pitchers", + "adopting", + "justices", + "rip", + "ince", + "elimination", + "aerospace", + "Beer", + "Basin", + "unwanted", + "goers", + "isco", + "Twin", + "Desert", + "rix", + "darkness", + "Dunn", + "City", + "pop", + "1969", + "ataka", + "tal", + "autism", + "McLaren", + "UEFA", + "classrooms", + "Leave", + "Americans", + "las", + "qui", + "undefeated", + "otto", + "NRA", + "Porsche", + "nuts", + "oys", + "Methodist", + "att", + "tweeting", + "children", + "eller", + "inquiries", + "millennials", + "Wembley", + "INS", + "autopsy", + "Elon", + "Hicks", + "ugg", + "wreck", + "Comcast", + "stones", + "public", + "Kem", + "bedroom", + "ļ", + "itated", + "semic", + "uman", + "Cal", + "ANN", + "Gaz", + "undisclosed", + "Planned", + "Yale", + "IST", + "lies", + "Standing", + "relieved", + "EO", + "graduating", + "park", + "âĢķ", + "pensions", + "rave", + "Wonder", + "AZ", + "costing", + "editors", + "totaled", + "spacecraft", + "meter", + "02", + "Nikki", + "sworth", + "Crit", + "asha", + "knees", + "hats", + "uity", + "Panther", + "tan", + "Buzz", + "Glad", + "Pleasant", + "SM", + "tricks", + "plac", + "Danielle", + "ours", + "washed", + "haven", + "drain", + "Uttar", + "apple", + "junk", + "turkey", + "Dug", + "diplomacy", + "empire", + "pinch", + "ferry", + "Dustin", + "03", + "elder", + "Everything", + "Progressive", + "ution", + "VI", + "dam", + "lever", + "Australians", + "consequence", + "itan", + "condemn", + "neg", + "Overview", + "successes", + "probable", + "Mirror", + "mor", + "verse", + "evaluating", + "Bes", + "imm", + "harness", + "resilient", + "Build", + "straightforward", + "ADE", + "grandparents", + "marched", + "Kiev", + "chiefs", + "oha", + "vest", + "kn", + "enda", + "Sev", + "batters", + "Jos", + "Que", + "Course", + "Corner", + "Mess", + "mourn", + "keepers", + "Regina", + "Everybody", + "trajectory", + "defenseman", + "Articles", + "spur", + "PhD", + "pipes", + "duck", + "combining", + "Hit", + "Georgetown", + "Bee", + "Cor", + "composition", + "connects", + "MARK", + "taker", + "certainty", + "hefty", + "Hezbollah", + "Ship", + "malicious", + "AI", + "bits", + "styl", + "impaired", + "CBI", + "Despite", + "othe", + "Ryder", + "Alf", + "ifa", + "Ind", + "blaming", + "Toledo", + "EW", + "Essex", + "iated", + "Aberdeen", + "ANCE", + "possess", + "superhero", + "overhead", + "quet", + "Ricky", + "dock", + "Telecom", + "shelf", + "³", + "maritime", + "portrayed", + "Yesterday", + "collided", + "cookies", + "Cul", + "indexes", + "naval", + "oval", + "105", + "Weber", + "chief", + "arma", + "Rey", + "auditor", + "Marion", + "Martha", + "Sally", + "sedan", + "Alison", + "nce", + "Es", + "Parade", + "pharmacy", + "Kre", + "loe", + "cks", + "mitigate", + "designing", + "2024", + "portable", + "improves", + "AMD", + "excluded", + "CON", + "Oscars", + "fixtures", + "comb", + "Berg", + "bother", + "boring", + "observation", + "Cad", + "recordings", + "Cultural", + "weaken", + "accuse", + "Abd", + "abor", + "115", + "uffle", + "highways", + "atham", + "empt", + "Deer", + "EDT", + "Wait", + "athan", + "accumulated", + "guilt", + "exempt", + "diluted", + "Jamal", + "shit", + "cross", + "eve", + "shirts", + "satisfy", + "Paulo", + "AH", + "sic", + "Chloe", + "Cities", + "Swansea", + "precision", + "Tracy", + "ping", + "continually", + "demographic", + "cliff", + "jaw", + "isted", + "Develop", + "AJ", + "aisle", + "Lionel", + "predominantly", + "mel", + "lifelong", + "hs", + "shouted", + "lad", + "dest", + "packs", + "Kath", + "Cruise", + "fired", + "oder", + "hua", + "goodbye", + "interfere", + "eca", + "ré", + "atum", + "itas", + "Lodge", + "Wald", + "midday", + "umble", + "asting", + "©", + "Leg", + "Nepal", + "chased", + "idge", + "conv", + "fraudulent", + "opera", + "shr", + "Universe", + "Jerome", + "1977", + "Dancing", + "RS", + "±", + "eks", + "chic", + "punish", + "propose", + "arin", + "Chop", + "Ahead", + "Gallagher", + "Bangkok", + "Shelby", + "NS", + "cheek", + "onia", + "relegation", + "Hind", + "Cory", + "fingerprint", + "strive", + "mm", + "igs", + "holy", + "favored", + "Someone", + "Latino", + "Patt", + "challenger", + "Cotton", + "Sw", + "itten", + "XI", + "Stat", + "DIS", + "automakers", + "evaluated", + "Arc", + "persuade", + "Af", + "reunited", + "abs", + "bride", + "purely", + "uce", + "uded", + "settling", + "lodged", + "fixing", + "succession", + "Alfred", + "Alvarez", + "mac", + "Font", + "contra", + "affle", + "copied", + "masses", + "Elections", + "Than", + "soaring", + "jay", + "suing", + "concentrated", + "convey", + "240", + "gs", + "Neal", + "nasty", + "LB", + "odi", + "Sergei", + "thumb", + "servants", + "revelation", + "discharge", + "Bright", + "Bent", + "Chrysler", + "mill", + "Imagine", + "receptions", + "personalities", + "silly", + "Loc", + "Zero", + "HI", + "rice", + "gar", + "far", + "enh", + "Biden", + "Entreprene", + "assumption", + "nicely", + "Either", + "|", + "NW", + "Kens", + "Nolan", + "owning", + "atures", + "Pastor", + "Registration", + "experiments", + "assurance", + "hashtag", + "oint", + "Bin", + "qualification", + "center", + "austerity", + "Pers", + "scoop", + "pros", + "Fields", + "fur", + "Jas", + "planting", + "security", + "Train", + "Kathy", + "demand", + "Lev", + "tut", + "tier", + "QU", + "exploitation", + "ignoring", + "Sex", + "adapted", + "disastrous", + "empower", + "creators", + "Lay", + "Dragon", + "Wyn", + "1974", + "acious", + "performance", + "Tiffany", + "isting", + "individually", + "Leading", + "Sask", + "catastrophic", + "punched", + "Vienna", + "surgical", + "Gr", + "odo", + "gem", + "Minority", + "mice", + "Historic", + "Kot", + "caster", + "suff", + "journal", + "presumably", + "Bit", + "inary", + "bre", + "enhancing", + "gru", + "Running", + "hardt", + "troubling", + "pumps", + "Prospect", + "etic", + "martial", + "councillor", + "atra", + "ths", + "Sark", + "Champ", + "scoring", + "Wel", + "rup", + "terrifying", + "Catch", + "inspections", + "pornography", + "bra", + "Keeping", + "banker", + "angers", + "Crimea", + "Disclosure", + "iba", + "turf", + "schedules", + "Jorge", + "Across", + "solving", + "sensation", + "WW", + "cial", + "atz", + "lion", + "certificates", + "itive", + "Wes", + "Prison", + "PlayStation", + "duty", + "variable", + "strangers", + "istrates", + "vs", + "reigning", + "sliding", + "Shin", + "telecommunications", + "installing", + "recogn", + "subway", + "too", + "McKin", + "Stoke", + "sensitivity", + "bas", + "san", + "(-", + "Suarez", + "averages", + "ammu", + "Fen", + "refined", + "outh", + "cob", + "Laz", + "essa", + "positioning", + "Three", + "oils", + "assaults", + "companion", + "Flash", + "Mam", + "Till", + "blues", + "Jae", + "Pier", + "bedrooms", + "Hawkins", + "Cornell", + "answering", + "sec", + "recognizes", + "Red", + "Jamaica", + "insurgents", + "brace", + "ra", + "Tai", + "ocation", + "ignment", + "reasonably", + "inating", + "bonuses", + "sandwich", + "inadequate", + "delicate", + "adorable", + "palace", + "smallest", + "practically", + "Crosby", + "levy", + "lend", + "boards", + "shaped", + "vulnerability", + "Kelley", + "sponsorship", + "ract", + "slew", + "federation", + "Lal", + "acies", + "Families", + "proposing", + "hyp", + "elected", + "inkle", + "Says", + "Apollo", + "Wis", + "imer", + "combines", + "tim", + "Question", + "borrowers", + "swiftly", + "Magn", + "headphones", + "Russia", + "tongue", + "bye", + "nn", + "seller", + "Word", + "Tom", + "Devin", + "Surrey", + "quad", + "courthouse", + "gi", + "Grill", + ">", + "rational", + "Flames", + "Cham", + "vacuum", + "Rays", + "escalating", + "outer", + "stretches", + "Speed", + "negatively", + "absorb", + "Austrian", + "slice", + "Diet", + "bun", + "tactical", + "CBD", + "edges", + "nest", + "strained", + "ulates", + "Tina", + "Net", + "ķ", + "Gos", + "God", + "White", + "proudly", + "usion", + "Arlington", + "Near", + "Maxwell", + "bomber", + "cared", + "approvals", + "exams", + "Economy", + "posters", + "Hampton", + "Pere", + "Contract", + "housed", + "instruction", + "Jess", + "acre", + "congestion", + "Gener", + "dioxide", + "var", + "Alexandria", + "Spider", + "coins", + "225", + "territorial", + "SPD", + "float", + "null", + "calculate", + "Din", + "eto", + "cows", + "punct", + "expire", + "kidnapped", + "cou", + "attitudes", + "Leh", + "Hero", + "Kabul", + "cubic", + "digits", + "RES", + "pipelines", + "icide", + "Single", + "hurts", + "Maz", + "Pak", + "slate", + "multimedia", + "ADA", + "Mexico", + "Release", + "chard", + "garlic", + "Fletcher", + "aforementioned", + "05", + "Parkway", + "firefighter", + "counseling", + "utions", + "Cap", + "consultants", + "Meh", + "ouring", + "DI", + "mic", + "phones", + "encounters", + "Happ", + "cartoon", + "flight", + "undertake", + "Hans", + "plunge", + "Parenthood", + "kickoff", + "Celsius", + "Ras", + "Dund", + "ounce", + "purse", + "mortality", + "brains", + "conglomerate", + "Observer", + "Sector", + "Apparently", + "blank", + "iston", + "weighs", + "gro", + "Paw", + "COM", + "Purdue", + "netted", + "Linux", + "Mike", + "faithful", + "magazines", + "headquartered", + "Ips", + "indications", + "Look", + "Elite", + "supreme", + "chunk", + "Sz", + "Vine", + "rise", + "Yas", + "general", + "Opera", + "priests", + "Assad", + "aunt", + "whopping", + "enzie", + "vegan", + "influx", + "Consult", + "waiver", + "Having", + "inning", + "proximity", + "classical", + "Islanders", + "advertisers", + "Ce", + "Sochi", + "memoir", + "Playing", + "yers", + "stud", + "observations", + "admire", + "hiking", + "batter", + "confusing", + "precaution", + "kil", + "clusive", + "opoulos", + "Westbrook", + "Tanzania", + "Cedar", + "usted", + "destructive", + "Indies", + "osi", + "Amid", + "intercepted", + "partnering", + "substances", + "Suns", + "promotes", + "bird", + "Gen", + "aper", + "Ey", + "terrain", + "1930", + "zon", + "breed", + "broken", + "uchin", + "Prim", + "Roland", + "fitted", + "protects", + "114", + "RP", + "disrupted", + "Baylor", + "oren", + "Keen", + "mansion", + "grassroots", + "Victory", + "barn", + "depreciation", + "oped", + "immer", + "garnered", + "Lip", + "Tob", + "creatures", + "ooter", + "consortium", + "obi", + "Monster", + "arks", + "turn", + "sketch", + "predicting", + "minimize", + "Ethan", + "anson", + "Adjusted", + "Hornets", + "NZ", + "Kathleen", + "Kier", + "Mercury", + "ghost", + "haw", + "Demand", + "Collection", + "Fortune", + "cruel", + "furious", + "Kun", + "Salem", + "unsuccessful", + "Lomb", + "Fury", + "ahi", + "enthusiastic", + "surgeries", + "ACE", + "roller", + "Stamford", + "Being", + "Dec", + "check", + "affection", + "gifted", + "energ", + "varying", + "Charl", + "solved", + "NV", + "laptops", + "kindness", + "mart", + "Penny", + "116", + "Feder", + "Cisco", + "educators", + "minim", + "gangs", + "festivities", + "Original", + "yre", + "rying", + "tighter", + "Malta", + "shield", + "interest", + "buoy", + "supplement", + "Sof", + "ok", + "prosecuted", + "interventions", + "seize", + "caravan", + "Carlson", + "Enterprises", + "Christina", + "Wellington", + "altered", + "TP", + "expresses", + "comfortably", + "staffing", + "afa", + "itu", + "saving", + "inflammation", + "hatt", + "Miranda", + "icious", + "grabbing", + "ANY", + "objections", + "dot", + "cle", + "relates", + "tribe", + "boarding", + "Episode", + "Enjoy", + "arding", + "athletics", + "flies", + "mortgages", + "ruct", + "ink", + "KC", + "Secondary", + "fer", + "Qaeda", + "OA", + "Frank", + "track", + "Chandler", + "env", + "Leaders", + "Kemp", + "unsafe", + "sponsored", + "San", + "Users", + "PE", + "Account", + "otta", + "Mix", + "Cindy", + "En", + "175", + "overlooked", + "publications", + "rewarding", + "explicit", + "notch", + "specifics", + "designation", + "Appeal", + "contingent", + "cage", + "Kol", + "Johns", + "Reach", + "Tin", + "Africans", + "prec", + "Rural", + "Dw", + "uphold", + "suffers", + "weed", + "inst", + "cancellation", + "Shaun", + "leve", + "divisive", + "hel", + "fatigue", + "Schwartz", + "Kirst", + "arise", + "grandson", + "Lawson", + "collaborate", + "participant", + "Bryce", + "infield", + "mid", + "ut", + "notices", + "sneak", + "PAR", + "Chris", + "utilize", + "Byron", + "Zhang", + "PF", + "overwhelmingly", + "vegetable", + "absurd", + "Chem", + "etime", + "envoy", + "lover", + "length", + "revolutionary", + "Yam", + "shutting", + "mt", + "super", + "Toby", + "Coca", + "proposition", + "embracing", + "versatile", + "Walking", + "illicit", + "nude", + "unpredictable", + "take", + "gotta", + "Xiaomi", + "instit", + "Pep", + "Pearson", + "rejection", + "stead", + "mut", + "outspoken", + "Baghdad", + "Fly", + "wholly", + "RM", + "Fa", + "cleaner", + "frey", + "Hab", + "Liber", + "whereabouts", + "chefs", + "alumni", + "stopp", + "dd", + "forward", + "rast", + "Nash", + "Cort", + "potent", + "mold", + "distinctive", + "chip", + "Brunswick", + "populist", + "plagued", + "eka", + "IOC", + "ugs", + "Dob", + "magn", + "asser", + "hew", + "capturing", + "oos", + "crystal", + "alarming", + "135", + "iating", + "nap", + "umar", + "Expl", + "upgrading", + "decl", + "overturn", + "ARK", + "linked", + "Continued", + "slumped", + "Gaga", + "iful", + "Posted", + "Recommended", + "snake", + "explosives", + "hind", + "contempt", + "mock", + "NBA", + "stall", + "organisers", + "ingredient", + "blockbuster", + "Stream", + "Leah", + "Pic", + "ventures", + "oman", + "weakening", + "maximize", + "digging", + "uez", + "distinction", + "Mali", + "contaminated", + "hij", + "crafts", + "Fl", + "closet", + "Rapp", + "towers", + "amenities", + "opioids", + "contend", + "load", + "Jol", + "Books", + "sim", + "thrilling", + "meter", + "Multiple", + "arbitration", + "cracked", + "Pl", + "photographers", + "Te", + "Sidd", + "explored", + "170", + "pleasant", + "Capitals", + "Ri", + "Randall", + "overed", + "char", + "Everybody", + "Politics", + "moisture", + "thriving", + "Scotia", + "arded", + "imb", + "Fantasy", + "cemetery", + "Path", + "eur", + "Sec", + "Platform", + "departed", + "VIDEO", + "Pant", + "Syn", + "230", + "bleacher", + "live", + "prob", + "gymn", + "judged", + "orns", + "stemming", + "umbling", + "Hew", + "Cheryl", + "consciousness", + "cos", + "Tate", + "CNN", + "recognizing", + "meg", + "pant", + "ulk", + "MM", + "Prescott", + "Marcel", + "anas", + "happier", + "mag", + "Lov", + "spreads", + "Sample", + "popped", + "HR", + "Mitt", + "00", + "labeled", + "aspirations", + "?)", + "loads", + "Britt", + "hurst", + "Teams", + "extremists", + "Clement", + "lings", + "shirts", + "cheon", + "DEL", + "Location", + "presentations", + "Falcon", + "toddler", + "kl", + "prone", + "commemor", + "Stanton", + "201", + "ranges", + "fielder", + "attends", + "rade", + "proactive", + "hostage", + "Griffith", + "ockey", + "Adding", + "AFL", + "gas", + "istics", + "surgeon", + "tsunami", + "2014", + "constraints", + "cu", + "surrendered", + "azed", + "Airbnb", + "650", + "zed", + "injustice", + "dog", + "full", + "Hear", + "sprawling", + "homeland", + "SG", + "anced", + "pools", + "CE", + "beers", + "AE", + "Jac", + "recurring", + "Writing", + "genius", + "Frost", + "grounded", + "allege", + "lessness", + "jumper", + "vicious", + "secretly", + "hacked", + "Amsterdam", + "ibu", + "1971", + "Rosenstein", + "nick", + "arge", + "ladder", + "elled", + "satellites", + "assassination", + "Depot", + "built", + "unrelated", + "maid", + "Dod", + "Vanderbilt", + "boundary", + "Stafford", + "Bry", + "tribunal", + "outings", + "quantity", + "imming", + "Blacks", + "Br", + "eri", + "uffed", + "explicitly", + "Bieber", + "AKING", + "photographed", + "Polit", + "premature", + "hered", + "Vi", + "marsh", + "casters", + "Kra", + "dried", + "cafe", + "eting", + "shaping", + "aram", + "orf", + "richest", + "hurricanes", + "commands", + "Gl", + "anth", + "stunt", + "yearly", + "defeats", + "consultancy", + "call", + "lag", + "adh", + "Palestine", + "customized", + "Scar", + "Wesley", + "ready", + "persist", + "packing", + "ono", + "discharged", + "pouring", + "sburg", + "reconsider", + "Method", + "enez", + "cill", + "secular", + "pers", + "ple", + "ELS", + "Mine", + "pushes", + "Us", + "frames", + "Nets", + "Siem", + "Hitler", + "kill", + "rented", + "charm", + "pulls", + "Tide", + "insufficient", + "itted", + "Care", + "iera", + "couch", + "aders", + "ext", + "Citizen", + "logical", + "Meadows", + "Denis", + "Drivers", + "republic", + "advising", + "paramedics", + "insky", + "illard", + "encia", + "kh", + "rh", + "finalized", + "reins", + "Farrell", + "steer", + "proxy", + "unes", + "Soul", + "Copper", + "Kenyan", + "amped", + "conference", + "sted", + "Lon", + "replay", + "Ble", + "vibe", + "portfolios", + "sea", + "beautifully", + "airs", + "Rap", + "Katrina", + "berth", + "gold", + "Isaiah", + "iques", + "elson", + "relentless", + "Highland", + "Philippe", + "Fol", + "enduring", + "enz", + "aer", + "icing", + "HTC", + "doping", + "Alb", + "som", + "icia", + "coroner", + "damn", + "119", + "wiped", + "Auditor", + "hern", + "Jew", + "endra", + "osp", + "Rory", + "shapes", + "Pablo", + "foremost", + "Hos", + "Cunningham", + "145", + "Recovery", + "!!!", + "western", + "imaging", + "Rookie", + "MTV", + "unc", + "Sporting", + "patrons", + "Coverage", + "Observatory", + "fishermen", + "Province", + "Aston", + "Osh", + "Weekend", + "recruits", + "density", + "FM", + "Gorsuch", + "Erie", + "lining", + "showcased", + "Rubio", + "chaotic", + "attractions", + "hug", + "Herbert", + "Respond", + "happily", + "tor", + "OTHER", + "runner", + "Shakespeare", + "stretching", + "Judy", + "wyn", + "Cafe", + "greens", + "Hend", + "glam", + "iation", + "Kingston", + "incremental", + "Live", + "Braun", + "USS", + "reb", + "imperative", + "sympathy", + "refuge", + "administered", + "rance", + "Liberia", + "mobil", + "heads", + "inevitably", + "Eugene", + "Berkshire", + "Harbour", + "Trends", + "TB", + "deficits", + "listings", + "readings", + "tumor", + "offic", + "opy", + "distracted", + "appropriately", + "Willis", + "skirt", + "Tea", + "shades", + "bargaining", + "retention", + "Concert", + "Meteor", + "Custom", + "inputs", + "Sah", + "enta", + "Love", + "Burg", + "Cynthia", + "Moses", + "ubb", + "peoples", + "dh", + "Fro", + "bean", + "cigarette", + "tta", + "umm", + "phenomenal", + "yelling", + "inaug", + "conven", + "Gore", + "request", + "colonial", + "Aleppo", + "demolition", + "amounted", + "staggering", + "clips", + "inconsistent", + "Milton", + "Wireless", + "Reno", + "Perkins", + "unusually", + "memor", + "hectares", + "lat", + "central", + "Dig", + "Marina", + "Partner", + "daily", + "your", + "Reilly", + "pope", + "phy", + "assessing", + "Rodrigo", + "wi", + "compatible", + "imate", + "gentle", + "Rhodes", + "Brexit", + "ieve", + "breaches", + "chopped", + "cancers", + "VEL", + "sluggish", + "Ultra", + "Ul", + "crises", + "ONE", + "Equipment", + "cater", + "adjourn", + "readily", + "Rolling", + "Bott", + "inel", + "Rule", + "grind", + "Hussain", + "ussie", + "depressed", + "Imperial", + "ongo", + "uniforms", + "117", + "chambers", + "Dum", + "ifi", + "Betty", + "TA", + "promotions", + "itary", + "cried", + "branding", + "Bahamas", + "Dat", + "antibiotics", + "Aus", + "umbrella", + "gradual", + "altercation", + "lure", + "Jakarta", + "unified", + "chin", + "ettes", + "Rwanda", + "ulations", + "brink", + "broadcasting", + "Artist", + "recon", + "aqu", + "Serv", + "999", + "Participants", + "Ventures", + "fight", + "activism", + "structured", + "portal", + "tendency", + "Associate", + "calf", + "Ord", + "Ti", + "Francois", + "uary", + "Vik", + "urchase", + "fried", + "booming", + "particles", + "amas", + "INA", + "Super", + "supp", + "urring", + "Watts", + "affer", + "DEC", + "roadway", + "border", + "sequ", + "entially", + "ieg", + "camping", + "750", + "cycles", + "Reese", + "Fellow", + "isters", + "Vehicle", + "kies", + "Jonas", + "foundations", + "Nigel", + "stab", + "congressman", + "Wichita", + "antes", + "progression", + "ditch", + "lik", + "sid", + "ele", + "Mund", + "stairs", + "lete", + "lingering", + "sadly", + "ay", + "Em", + "deadliest", + "soon", + "tangible", + "abusing", + "comprises", + "vil", + "Bun", + "doubling", + "commun", + "slogan", + "loading", + "shallow", + "attributes", + "Che", + "cheering", + "refuses", + "cam", + "bes", + "hon", + "Spartans", + "cept", + "Computer", + "Canberra", + "WARNING", + "stuffed", + "block", + "Jennings", + "AU", + "atin", + "om", + "bachelor", + "prediction", + "Winner", + "agne", + "rob", + "Katherine", + "li", + "Humph", + "PEOPLE", + "IRO", + "Cola", + "guitarist", + "isen", + "Highlights", + "welcomes", + "prisoner", + "psychology", + "extradition", + "rou", + "Lund", + "thoughtful", + "RY", + "orman", + "Alex", + "laughter", + "fumble", + "synthetic", + "digit", + "Roc", + "Factory", + "ellery", + "ishment", + "ilar", + "Earl", + "Sutton", + "Jur", + "Allan", + "Koreans", + "uki", + "culinary", + "PU", + "Stock", + "stars", + "Dayton", + "beck", + "instability", + "Bring", + "breeding", + "miracle", + "bons", + "donating", + "Kick", + "Sag", + "afi", + "harassed", + "asm", + "Their", + "inity", + "academics", + "statute", + "Amit", + "pressured", + "east", + "\"),", + "iso", + "220", + "airplane", + "McCabe", + "ctions", + "Mesa", + "sensational", + "FE", + "Neigh", + "bribery", + "flaws", + "females", + "misses", + "Color", + "Vietnamese", + "Mental", + "Unfortunately", + "Pont", + "1940", + "dry", + "Gazette", + "Ans", + "whistle", + "symbolic", + "possessions", + "Driver", + "bracket", + "Reign", + "oji", + "oct", + "tube", + "Felix", + "translated", + "promptly", + "Ernest", + "arth", + "dumb", + "influences", + "taking", + "privat", + "erers", + "malware", + "predictable", + "tighten", + "heights", + "fairness", + "facing", + "rematch", + "poet", + "fundamentally", + "coveted", + "livelihood", + "ABOUT", + "sourced", + "deferred", + "slashed", + "Schultz", + "triggering", + "Shiv", + "lithium", + "ahead", + "leisure", + "backpack", + "ilateral", + "Nuclear", + "Leone", + "Nice", + "enthusiasts", + "September", + "enroll", + "Wear", + "erey", + "angs", + "such", + "unpopular", + "disciplined", + "shrinking", + "Brewing", + "Really", + "directive", + "175", + "notifications", + "fortunes", + "Hour", + "Gan", + "Churchill", + "Dodge", + "Jeep", + "sour", + "derived", + "ft", + "riv", + "laundry", + "fentanyl", + "Sioux", + "achi", + "workers", + "workload", + "rooms", + "QU", + "Truth", + "defenses", + "dunk", + "IJ", + "derby", + "Motion", + "Mayo", + "Ike", + "preferences", + "ped", + "elman", + "moon", + "shoots", + "Noel", + "milit", + "Cambodia", + "MLA", + "honoured", + "fast", + "algorithms", + "stormed", + "NT", + "Benz", + "vaccines", + "marching", + "118", + "Wilmington", + "GM", + "coin", + "underwater", + "Clearly", + "organs", + "mir", + "denounced", + "pless", + "imal", + "Kom", + "fatalities", + "youngster", + "thirty", + "internally", + "222", + "demonstrating", + "busiest", + "perpetrators", + "stun", + "Both", + "McCoy", + "gn", + "Dalton", + "DAY", + "sacred", + "consuming", + "(+", + "Pioneer", + "Applications", + "Bolt", + "Barkley", + "Expo", + "Lore", + "Privacy", + "Harley", + "tractor", + "tenth", + "Haiti", + "ÃŃn", + "TVs", + "Cathedral", + "unite", + "binding", + "oks", + "Jenny", + "caller", + "Ingram", + "Prairie", + "runoff", + "asserted", + "icit", + "Sie", + "102", + "MB", + "obstruction", + "groom", + "tolerate", + "cans", + "forth", + "villain", + "defining", + "Frenchman", + "otte", + "contr", + "clock", + "onder", + "prolific", + "Electronic", + "Sak", + "annie", + "ASS", + "multinational", + "Associated", + "IZ", + "Belle", + "mand", + "asis", + "Mac", + "pretend", + "Communication", + "heartbreaking", + "Shepherd", + "BIG", + "mph", + "Shield", + "Liv", + "Status", + "bikini", + "ranch", + "peacefully", + "ITCH", + "bourne", + "Variety", + "stationed", + "hed", + "exhausted", + "surpassed", + "catalyst", + "smuggling", + "uating", + "123", + "dup", + "Sul", + "conf", + "jit", + "maiden", + "asta", + "Calvin", + "borne", + "grim", + "tort", + "cott", + "olas", + "NR", + "breakout", + "Hun", + "Guatemala", + "historian", + "Lawyers", + "Display", + "obstruct", + "Osborne", + "therapies", + "Aub", + "injunction", + "stroke", + "seafood", + "hazardous", + "Wolver", + "Violence", + "Billion", + "Letter", + "Worldwide", + "Real", + "expires", + "flawed", + "European", + "rigorous", + "Similar", + "Surface", + "EF", + "mys", + "Funds", + "ographer", + "tribes", + "spouse", + "unsure", + "aways", + "trainers", + "arie", + "Zar", + "Comedy", + "Lit", + "Noon", + "gallon", + "consulate", + "Bras", + "iology", + "onies", + "Belichick", + "Root", + "Lux", + "Sed", + "Tos", + "inherited", + "tw", + "deaf", + "driveway", + "jah", + "Scientific", + "Nottingham", + "both", + "awan", + "nut", + "Lebanese", + "AAA", + "Suzuki", + "BU", + "ells", + "specify", + "Notes", + "voluntarily", + "Molly", + "outskirts", + "behaviors", + "militia", + "splash", + "personalized", + "Fiat", + "Kind", + "Truck", + "py", + "WIN", + "dist", + "itational", + "APP", + "Pelicans", + "Gam", + "mel", + "mandated", + "balances", + "Wizards", + "iary", + "Available", + "kay", + "jin", + "eyed", + "sterling", + "concealed", + "FedEx", + "PO", + "Jacqu", + "anted", + "eme", + "Defensive", + "manship", + "reliever", + "shortstop", + "phot", + "Gain", + "Concern", + "due", + "algorithm", + "fell", + "Mountains", + "icians", + "honoring", + "uploaded", + "tore", + "GH", + "orde", + "Coin", + "Aven", + "literary", + "Before", + "tactic", + "socially", + "Sik", + "thermal", + "hor", + "price", + "rooted", + "arrow", + "circulating", + "laughs", + "Lines", + "lig", + "judgement", + "....", + "sewer", + "dancer", + "Pens", + "sig", + "ische", + "wives", + "gran", + "Bron", + "Hyde", + "yards", + "candidacy", + "hey", + "contributors", + "Updated", + "190", + "halls", + "emphas", + "Cherry", + "rim", + "billed", + "baked", + "Popular", + "lb", + "gravity", + "Under", + "reservation", + "organ", + "Pict", + "Whitney", + "onboard", + "NEY", + "Breaking", + "flagged", + "rar", + "Basic", + "Domestic", + "Pent", + "vigilant", + "zoning", + "Fire", + "corrected", + "isbury", + "Laure", + "Devon", + "print", + "Topics", + "Fuel", + "circulation", + "Pratt", + "skiing", + "tornado", + "dep", + "Unless", + "ifting", + "fool", + "should", + "inspectors", + "protested", + "ba", + "ussia", + "spun", + "grass", + "phone", + "potato", + "Behind", + "cil", + "concession", + "applause", + "Chin", + "ceremonies", + "pit", + "traumatic", + "basics", + "parameters", + "Moz", + "AIDS", + "Ph", + "judging", + "lecture", + "municipality", + "cardiac", + "ogan", + "pir", + "could", + "Channel", + "shattered", + "AV", + "continental", + "chie", + "ibi", + "Oy", + "Mon", + "CN", + "WC", + "distributor", + "Savannah", + "cleaned", + "Flores", + "embarrassed", + "clay", + "volcano", + "stressful", + "summoned", + "Seg", + "statistical", + "Shak", + "adequately", + "worthy", + "fighting", + "alan", + "necessity", + "residency", + "sober", + "arius", + "Taj", + "mount", + "wards", + "aesthetic", + "Coin", + "Dew", + "were", + "SK", + "powerhouse", + "cleanup", + "WITH", + "Hers", + "Rao", + "Flyers", + "dominating", + "issued", + "McGr", + "insurgency", + "burial", + "Plains", + "ensive", + "Present", + "Mo", + "nerves", + "smoothly", + "staff", + "restoring", + "Generation", + "commuters", + "Legend", + "Gad", + "lied", + "issuer", + "Dozens", + "phases", + "Wu", + "Tunisia", + "Pacers", + "dur", + "IG", + "annon", + "sided", + "vo", + "NI", + "vitamin", + "soc", + "immunity", + "generates", + "McGu", + "explores", + "assistants", + "stems", + "ushed", + "Zak", + "Owners", + "variant", + "ardy", + "Newark", + "Catalonia", + "autonomy", + "greet", + "await", + "Luckily", + "Ticket", + "STOR", + "asy", + "incorrect", + "consisting", + "perspectives", + "Quint", + "totaling", + "northeastern", + "characterized", + "surfaces", + "nation", + "prevents", + "Sho", + "electorate", + "shortfall", + "chy", + "aws", + "Address", + "defensively", + "quel", + "chester", + "terr", + "ahu", + "lined", + "Nev", + "unn", + "Def", + "pc", + "Sig", + "nonetheless", + "Sundays", + "BAS", + "policemen", + "Goal", + "apa", + "rope", + "outage", + "Paso", + "sadness", + "Growing", + "Kyr", + "ale", + "Breitbart", + "Via", + "Brig", + "idence", + "145", + "quire", + "distraction", + "Odd", + "Simply", + "Nin", + "competent", + "ded", + "iper", + "Katy", + "Solomon", + "feeds", + "Mort", + "Rica", + "affe", + "cooperating", + "arrivals", + "delete", + "Ath", + "trustees", + "tub", + "saga", + "otes", + "CJ", + "exited", + "stakes", + "influ", + "2000", + "Donovan", + "Nur", + "outline", + "audition", + "oked", + "Jag", + "money", + "cardiovascular", + "song", + "Often", + "Goff", + "Oaks", + "Will", + "acon", + "?", + "Har", + "Lambert", + "atoon", + "AF", + "Mavericks", + "nia", + "Chennai", + "\"},\"", + "pairing", + "mad", + "ause", + "Ride", + "111", + "Fallon", + "Hyder", + "Piper", + "filmmakers", + "icon", + "Beau", + "butt", + "lot", + "rifles", + "sunglasses", + "TRA", + "magnetic", + "arty", + "Yo", + "Weight", + "?!", + "ether", + "aspir", + "hunters", + "contamination", + "Ben", + "political", + "],\"", + "Bever", + "monuments", + "won", + "auc", + "expressions", + "lakes", + "iao", + "abin", + "pleading", + "discounted", + "disappoint", + "TW", + "craft", + "societies", + "Augusta", + "bott", + "marker", + "Wrestling", + "CBC", + "athy", + "AZ", + "fabulous", + "valued", + "optical", + "shaken", + "OSS", + "Imp", + "AUD", + "inals", + "revital", + "controller", + "grasp", + "uling", + "Frederick", + "ague", + "bull", + "Ladies", + "disruptive", + "benefiting", + "verge", + "Dak", + "grabs", + "PAC", + "GN", + "McMahon", + "rob", + "Especially", + "Chrome", + "Bundesliga", + "104", + "liberty", + "SF", + "varieties", + "East", + "growers", + "socialist", + "unemployed", + "AMI", + "totals", + "Gib", + "defect", + "Ortiz", + "Perfect", + "praying", + "ISS", + "ul", + "thrust", + "osc", + "Otherwise", + "obsessed", + "650", + "Website", + "spectators", + "Scout", + "Boone", + "Dillon", + "abortions", + "lect", + "utz", + "villagers", + "accelerating", + "slap", + "vague", + "jurisdictions", + "League", + "Uruguay", + "obstacle", + "manufactures", + "campaigned", + "Advance", + "Nort", + "emer", + "1964", + "irre", + "prog", + "Featured", + "commute", + "handset", + "akis", + "Ars", + "tail", + "iker", + "crafted", + "upl", + "Marcos", + "Looking", + "seated", + "Boat", + "readiness", + "LLP", + "otechnology", + "facebook", + "Scouts", + "Ear", + "Adv", + "Democracy", + "NI", + "oci", + "Snapdragon", + "Saturday", + "Pra", + "Coastal", + "Voters", + "Leigh", + "ohn", + "orry", + "technicians", + "armed", + "shrink", + "spinning", + "agram", + "320", + "liner", + "Contest", + "Countries", + "farewell", + "CW", + "aris", + "storytelling", + "passer", + "sailing", + "control", + "dissent", + "Rih", + "edit", + "spoilers", + "itched", + "Bentley", + "cant", + "mn", + "Macy", + "indefinitely", + "vill", + "meth", + "EL", + "optional", + "remark", + "Vanessa", + "ã", + "masks", + "Provincial", + "culprit", + "Tol", + "snack", + "Infinity", + "Pub", + "brakes", + "clar", + "inception", + "love", + "wonders", + "forged", + "CEOs", + "specifications", + "irst", + "ension", + "Marin", + "det", + "ordeal", + "Feed", + "December", + "strokes", + "fect", + "orial", + "showcasing", + "stack", + "UAL", + "Alexandra", + "poison", + "Fry", + "Cars", + "prototype", + "USDA", + "IF", + "flows", + "tailored", + "Gear", + "myth", + "platinum", + "seven", + "founded", + "encing", + "Tip", + "Mald", + "geopolitical", + "112", + "enqu", + "NR", + "Nadu", + "leen", + "Tat", + "colon", + "Size", + "vis", + "bere", + "Annie", + "Watkins", + "pumping", + "cur", + "Bates", + "slug", + "miss", + "forecasting", + "source", + "acknowledges", + "prosecute", + "testament", + "cum", + "ems", + "socks", + "Same", + "competitiveness", + "definitive", + "intensified", + "satisfying", + "physics", + "Harden", + "subsidy", + "Men", + "Paddock", + "workouts", + "Saw", + "crisp", + "Bezos", + "Vote", + "guiding", + "anged", + "staple", + "ŀ", + "ules", + "Avengers", + "optim", + "Buffett", + "timetable", + "oust", + "HE", + "Grab", + "Have", + "cca", + "waived", + "retaining", + "aber", + "offline", + "vigil", + "books", + "Rein", + "acknowledging", + "Doyle", + "proteins", + "mixing", + "Alcohol", + "JD", + "syn", + "thieves", + "homemade", + "feminist", + "Roosevelt", + "Coal", + "wishing", + "SIGN", + "Lad", + "empathy", + "Brooke", + "Mash", + "inations", + "''", + "ulators", + "drastically", + "floral", + "Guild", + "undercover", + "Laboratory", + "Rank", + "restraining", + "paragraph", + "persona", + "Employment", + "ogs", + "Gw", + "Medal", + "wildly", + "fare", + "CNBC", + "photo", + "transforming", + "termination", + "still", + "INT", + "bal", + "Econom", + "Larson", + "heck", + "quantitative", + "emergence", + "esta", + "knot", + "whale", + "ðŁĺ", + "perimeter", + "empowerment", + "mg", + "rents", + "refreshing", + "leasing", + "patents", + "andi", + "fathers", + "unse", + "processors", + "Down", + "reversal", + "veh", + "andal", + "Kov", + "Blue", + "specializes", + "Link", + "Considering", + "Edmund", + "neo", + "agger", + "rg", + "severity", + "cour", + "RL", + "Teresa", + "gallons", + "acquitted", + "accompl", + "cracks", + "sciences", + "Club", + "predicts", + "Vu", + "hints", + "Zack", + "refurb", + "destabil", + "Samar", + "Info", + "fs", + "ratios", + "inherent", + "Continental", + "treasure", + "caucus", + "enact", + "orporated", + "ineries", + "tastes", + "main", + "sq", + "ickson", + "corruption", + "ulture", + "Goodman", + "Ling", + "Sup", + "exposing", + "immers", + "responds", + "heimer", + "Air", + "Figures", + "longstanding", + "Analytics", + "enforced", + "nickname", + "clinch", + "Carpenter", + "Pharma", + "constructive", + "gel", + "Sham", + "TOP", + "Derrick", + "ör", + "birds", + "Tong", + "Batman", + "Rouhani", + "Olive", + "Riv", + "dessert", + "guides", + "sag", + "chemotherapy", + "slept", + "Franc", + "Dunk", + "writers", + "ÃĹ", + "401", + "outfielder", + "Hamburg", + "izu", + "scr", + "comparisons", + "whites", + "traits", + "collateral", + "LEY", + "ideshow", + "statutory", + "ruin", + "situated", + "tem", + "inject", + "rage", + "550", + "factions", + "Naomi", + "cutting", + "communicating", + "railroad", + "sparking", + "respiratory", + "Webster", + "Carbon", + "undertaking", + "composer", + "Figure", + "specified", + "Video", + "uber", + "sexuality", + "lected", + "Burger", + "Cards", + "SR", + "Lie", + "recount", + "exceeding", + "quoting", + "Jama", + "Victorian", + "sway", + "Ges", + "SI", + "Kazakhstan", + "accusation", + "etr", + "Ah", + "proc", + "lamb", + "Morales", + "Lily", + "derail", + "contributes", + "iddle", + "Concord", + "electr", + "equip", + "quantum", + "thereafter", + "arrange", + "raided", + "Move", + "Sang", + "Gaming", + "biology", + "Amnesty", + "demise", + "Barton", + "qualifier", + "ANI", + "undersc", + "royalty", + "INC", + "sne", + "ariat", + "Wan", + "cluster", + "quin", + "whales", + "Fear", + "Brew", + "deport", + "airs", + "census", + "OUS", + "respectful", + "bone", + "waivers", + "friend", + "systemic", + "Dion", + "James", + "Admission", + "stigma", + "TIME", + "underpin", + "Witnesses", + "digs", + "genocide", + "staging", + "rolled", + "specially", + "oop", + "baseline", + "RF", + "avis", + "vocals", + "COL", + "LD", + "impending", + "Caldwell", + "aluminium", + "stra", + "Tayyip", + "admissions", + "falls", + "realizing", + "oen", + "RV", + "Mog", + "advocating", + "Pepper", + "lived", + "Wick", + "Facebook", + "Spect", + "shout", + "fractured", + "vet", + "1966", + "compensate", + "Volume", + "categor", + "Huntington", + "Free", + "OUGH", + "local", + "Sch", + "uti", + "burger", + "bush", + "impacting", + "frost", + "tti", + "Fresno", + "onz", + "shaw", + "Libyan", + "assert", + "Legacy", + "IE", + "Kinder", + "Horizon", + "tum", + "signaled", + "Fors", + "speedy", + "rang", + "FT", + "selecting", + "pale", + "WD", + "probability", + "OUND", + "istrate", + "sens", + "ocating", + "interpret", + "puzzle", + "inland", + "manipulation", + "Sal", + "fulfilling", + "McMaster", + "Make", + "jun", + "giving", + "Niagara", + "scholars", + "ALT", + "Steam", + "omin", + "Sau", + "Downing", + "gy", + "Tit", + "Lav", + "Pepsi", + "dumping", + "Detect", + "TDs", + "Kob", + "SY", + "pioneer", + "_", + "clarified", + "Tests", + "opic", + "MN", + "Bowman", + "umin", + "widow", + "rallying", + "Pull", + "projection", + "escalation", + "libraries", + "Founder", + "Hugo", + "Style", + "freelance", + "listeners", + "discovering", + "Plans", + "franchises", + "Pam", + "farther", + "UI", + "opers", + "103", + "ublished", + "keys", + "aky", + "innov", + "¦", + "Drum", + "wraps", + "Congressman", + "Venus", + "fake", + "Bronx", + "Dinner", + "faced", + "backward", + "inge", + "arsenal", + "Ace", + "uden", + "fre", + "spa", + "Saunders", + "Matter", + "Spons", + "consultations", + "Russ", + "sculpture", + "uncommon", + "Nov", + "pg", + "otherapy", + "gol", + "Blazers", + "advises", + "Regulatory", + "Boyle", + "Äģ", + "cuisine", + "encouragement", + "yp", + "eny", + "Orchestra", + "Chicken", + "1965", + "Pret", + "Cooperation", + "Devices", + "Rodney", + "Honduras", + "Egg", + "churn", + "clutch", + "Bernstein", + "ain", + "formidable", + "Facility", + "pag", + "mons", + "bol", + "literacy", + "submissions", + "Hulu", + "Constitutional", + "Ish", + "Paula", + "olve", + "abundance", + "Ala", + "Ecuador", + "reconstruction", + "crush", + "reek", + "ÂŃ", + "ibo", + "practiced", + "pac", + "rett", + "pasta", + "resp", + "Flag", + "pal", + "commenting", + "recap", + "âĢĶâĢĶ", + "Toy", + "Meredith", + "receipt", + "separating", + "Map", + "mogul", + "Burlington", + "ger", + "coordinate", + "grad", + "escalated", + "proceeded", + "turned", + "upt", + "hum", + "Were", + "Whether", + "enjoyable", + "energy", + "prohibit", + "hurdle", + "divorced", + "commentator", + "GT", + "ATH", + "travellers", + "populated", + "Vo", + "Rebels", + "spurred", + "ideological", + "elephant", + "keyes", + "Pat", + "linger", + "reps", + "cocktails", + "Kristen", + "istically", + "gunmen", + "1920", + "quart", + "National", + "exceptions", + "kat", + "priced", + "Harold", + "Pistons", + "compounds", + "mouse", + "exhibits", + "Burk", + "classmates", + "circulated", + "attributable", + "Baton", + "organizer", + "durable", + "singers", + "Oman", + "hydrogen", + "slash", + "accidental", + "Abrams", + "KS", + "itty", + "rust", + "selections", + "porting", + "Emanuel", + "XX", + "Thornton", + "columns", + "sentiments", + "fun", + "plight", + "Sister", + "Maggie", + "hya", + "Daniel", + "plung", + "orio", + "Yorker", + "Saturdays", + "loc", + "aye", + "illon", + "Consulting", + "pled", + "Zin", + "Farms", + "Giuliani", + "MIN", + "Hanson", + "Complete", + "ourke", + "oche", + "Jord", + "professors", + "WILL", + "Cron", + "dorm", + "cracking", + "tur", + "ORS", + "Ant", + "deduction", + "SIM", + "igue", + "Valent", + "Ethereum", + "Sunny", + "Extra", + "ivan", + "Fo", + "leases", + "ibe", + "1800", + "slapped", + "emaker", + "fa", + "rien", + "Period", + "ES", + "Blu", + "preserving", + "smarter", + "mans", + "gest", + "zu", + "nu", + "divest", + "roc", + "Flood", + "Given", + "Norton", + "granting", + "dealings", + "geographic", + "esa", + "cub", + "criticizing", + "Cub", + "surroundings", + "Internal", + "sle", + "crushing", + "PP", + "izations", + "Abdel", + "Joe", + "Visitors", + "Carly", + "INGTON", + "GC", + "WB", + "gently", + "·", + "though", + "Alto", + "resting", + "Person", + "Ton", + "bore", + "Clar", + "mot", + "bathrooms", + "Typically", + "disconnect", + "tightly", + "Harvest", + "Hed", + "Germans", + "atar", + "keynote", + "improper", + "fil", + "intens", + "iev", + "medi", + "tenant", + "footsteps", + "uli", + "legalization", + "106", + "Lexington", + "folio", + "½", + "Rita", + "battered", + "inka", + "JavaScript", + "Musical", + "Talent", + "lounge", + "intimidation", + "ikh", + "Fam", + "therapeutic", + "balancing", + "rocky", + "liners", + "Predators", + "registering", + "diligence", + "Rover", + "Dot", + "terminated", + "Edu", + "charming", + "PLAY", + "Fact", + "Ci", + ").\"", + "Wrestle", + "hun", + "openings", + "fou", + "126", + "spe", + "AW", + "bud", + "Temper", + "Orthodox", + "progressed", + "tre", + "tasting", + "scrutin", + "Lima", + "layout", + "litter", + "ijk", + "Parkinson", + "Anfield", + "developmental", + "heaven", + "Woodward", + "index", + "pistol", + "reson", + "WS", + "emb", + "Lap", + "Ple", + "lington", + "Sit", + "abruptly", + "Senegal", + "Yates", + "aceutical", + "Jak", + "Hastings", + "iste", + "DB", + "Agent", + "preservation", + "Lank", + "Suffolk", + "boo", + "essed", + "empowering", + "enne", + "recycled", + "strateg", + "brake", + "135", + "Stef", + "Flake", + "Gregg", + "Rent", + "installment", + "FW", + "Cran", + "obo", + "ml", + "Jade", + "accuses", + "Nvidia", + "burg", + "High", + "bothered", + "Benn", + "interrupted", + "trek", + "serv", + "patron", + "dictator", + "owa", + "jad", + "Tulsa", + "boil", + "displaying", + "cinem", + "awaited", + "¸", + "reacts", + "Dee", + "Gron", + "igation", + "servic", + "capt", + "insane", + "Veteran", + "umen", + "End", + "Cream", + "extremism", + "Malone", + "Col", + "safeguard", + "tomatoes", + "die", + "champ", + "zero", + "PRES", + "choir", + "pediatric", + "privileged", + "downstream", + "Business", + "Fighting", + "atable", + "sums", + "insult", + "arten", + "WikiLeaks", + "pads", + "retali", + "Hunts", + "indie", + "Shields", + "Mortgage", + "oses", + "ampton", + "Videos", + "PER", + "itionally", + "Kimmel", + "sum", + "trade", + "acity", + "marked", + "Angus", + "temper", + "seizure", + "fictional", + "utton", + "eva", + "Rs", + "intra", + "Request", + "ppe", + "eBay", + "USS", + "1500", + "possessing", + "bacon", + "Sexual", + "Buff", + "slaughter", + "jur", + "zhou", + "suit", + "Cha", + "Buk", + "crime", + "Easy", + "Chain", + "aq", + "Pall", + "flation", + "225", + "oup", + "109", + "McKenzie", + "clearer", + "Dogs", + "oration", + "subs", + "Follow", + "Shirley", + "adjusting", + "EFF", + "flipped", + "conform", + "Laurent", + "circular", + "NOR", + "mort", + "texture", + "avour", + "flex", + "Hedge", + "ðŁĺ", + "trophies", + "INV", + "boast", + "Tyr", + "Nichols", + "Spa", + "cheered", + "prey", + "reach", + "breached", + "Regions", + "Lyft", + "Tul", + "Kore", + "endure", + "Cover", + "\").", + "Savage", + "ère", + "reens", + "nic", + "sector", + "weaknesses", + "reboot", + "210", + "imagery", + "Frem", + "clue", + "Lars", + "faction", + "hetic", + "allied", + "Marvin", + "methodology", + "TN", + "utter", + "270", + "Volvo", + "oline", + "ACLU", + "indirect", + "miner", + "Bale", + "Strange", + "Fuller", + "expelled", + "Tropical", + "remotely", + "TIM", + "innocence", + "confined", + "fares", + "prevalent", + "desp", + "House", + "azar", + "gestures", + "CES", + "DM", + "eal", + "Ð", + "burnt", + "framed", + "Dani", + "hol", + "Cannes", + "Hayden", + "wardrobe", + "Assange", + "Samp", + "bay", + "sky", + "Hence", + "Grizzlies", + "rates", + "laws", + "Mandela", + "Hoover", + "rics", + "charged", + "exclude", + "passive", + "continuation", + "blunt", + "vac", + "Emerging", + "rench", + "tv", + "Hollow", + "OC", + "advisors", + "rendered", + "Bernardino", + "Supporters", + "ronic", + "chancellor", + "1963", + "uranium", + "ak", + "Options", + "ermott", + "Berger", + "ibia", + "explosions", + "impairment", + "hail", + "alley", + "cruelty", + "Clarence", + "variations", + "realm", + "renovations", + "Norwich", + "belongings", + "merchants", + "Ministers", + "Dodd", + "viewer", + "neutrality", + "quer", + "Princeton", + "dead", + "arest", + "GET", + "Canadiens", + "Ign", + "clear", + "Mal", + "Bridges", + "Hayward", + "remarked", + "ingle", + "sob", + "depart", + "beans", + "preserved", + "Fairfax", + "forgot", + "Beh", + "Rob", + "cooperative", + "ullah", + "mates", + "rang", + "thigh", + "abducted", + "chaired", + "Hearts", + "identifies", + "Buckingham", + "ijn", + "Jab", + "clashed", + "feed", + "sites", + "Career", + "exp", + "Buccaneers", + "scape", + "updating", + "intentional", + "Guam", + "Breakfast", + "Hag", + "Media", + "tapping", + "pics", + "eaten", + "premise", + "Kim", + "Storage", + "extensively", + "outrageous", + "Sadly", + "Global", + "¢", + "leaning", + "CM", + "easiest", + "ument", + "122", + "daunting", + "ISE", + "sunset", + "reset", + "bent", + "Trust", + "Caleb", + "Rut", + "Bast", + "ETS", + "iencies", + "pu", + "ature", + "realities", + "omi", + "soda", + "unveil", + "Goldberg", + "opes", + "uprising", + "MR", + "endorse", + "sail", + "converting", + "glamorous", + "Hollande", + "108", + "isky", + "cushion", + "240", + "adventures", + "antitrust", + "Stockholm", + "pace", + "Vald", + "Transfer", + "ERT", + "McInt", + "surging", + "ogn", + "lauded", + "Zam", + "Rough", + "TOR", + "wed", + "origins", + "Eld", + "oso", + "supplying", + "Petty", + "Twe", + "Denise", + "Bec", + "behave", + "121", + "estone", + "Boulder", + "Blackhawks", + "Wyatt", + "figuring", + "Deborah", + "agi", + "significant", + "asthma", + "messy", + "mpire", + "ax", + "aspiring", + "NH", + "Gina", + "heavy", + "Vick", + "ÃŃs", + "something", + "bodily", + "unauthorized", + "Actually", + "OH", + "microphone", + "allah", + "rampant", + "relocated", + "widening", + "Cait", + "nel", + "BlackBerry", + "professionally", + "Interestingly", + "barbecue", + "resisting", + "Nunes", + "disc", + "groundbreaking", + "orable", + "Regulation", + "borrowed", + "leaking", + "lengths", + "unveiling", + "houses", + "155", + "Billboard", + "icion", + "Times", + "Zoe", + "Abby", + "bus", + "Minutes", + "ributed", + "parap", + "fertil", + "ABC", + "Isle", + "therapist", + "gubernatorial", + "Aust", + "Loan", + "Bo", + "NRL", + "rag", + "Clear", + "revision", + "flesh", + "BD", + "iji", + "productions", + "coconut", + "McCorm", + "Dash", + "geography", + "hearted", + "arson", + "goaltender", + "belly", + "qualifications", + "Activ", + "hooked", + "Hungarian", + "protocols", + "inking", + "fronts", + "Kuala", + "Toys", + "Fitness", + "warfare", + "outp", + "Questions", + "wel", + "Shan", + "Morton", + "Romero", + "glance", + "Tay", + "sneakers", + "Symphony", + "inspect", + "enna", + "Nobody", + "scrapped", + "DeVos", + "Dominican", + "planets", + "anova", + "notify", + "incurred", + "unders", + "detainees", + "Marriott", + "electric", + "Kes", + "union", + "Watt", + "ATING", + "slipping", + "raft", + "resisted", + "cred", + "tern", + "flurry", + "Line", + "consulted", + "analyzing", + "107", + "Wide", + "¶", + "human", + "FEMA", + "smash", + "corps", + "barric", + "collar", + "TB", + "without", + "Canucks", + "needle", + "Sidney", + "Lauderdale", + "glove", + "ilee", + "pic", + "benef", + "Hydro", + "Disc", + "Arg", + "termin", + "sympath", + "pest", + "Coff", + "advancement", + "social", + "pol", + "Emails", + "stacked", + "ibly", + "Albion", + "fist", + "hero", + "Marian", + "asia", + "township", + "slick", + "modeling", + "achers", + "Argent", + "SUN", + "arde", + "pinned", + "hitters", + "dare", + "ictions", + "arily", + "sting", + "primaries", + "appointed", + "formats", + "glitter", + "patches", + "strategically", + "aka", + "yielded", + "BY", + "jeopard", + "Vand", + "crowned", + "occupants", + "tanker", + "Visa", + "Great", + "seasoned", + "Aviv", + "fiery", + "derivatives", + "diverted", + "acqu", + "sandwiches", + "Lorenzo", + "pardon", + "Barber", + "Agricultural", + "Philly", + "regrets", + "Millions", + "Frazier", + "treasury", + "Kenn", + "destined", + "olved", + "Back", + "leader", + "lyss", + "Reyes", + "001", + "bags", + "Standards", + "Excellence", + "Maid", + "Anthem", + "FIELD", + "revived", + "Quad", + "distinguished", + "weighted", + "ritual", + "invites", + "wana", + "iture", + "CI", + "MAY", + "unfairly", + "KP", + "Midlands", + "mint", + "uers", + "catalog", + "arant", + "losers", + "scheduling", + "esar", + "transferring", + "bankrupt", + "methamphetamine", + "Esk", + "Treatment", + "Response", + "homework", + "Bald", + "embarrassment", + "poorest", + "Platinum", + "Fac", + "unleashed", + "brighter", + "002", + "disl", + "Lowry", + "ived", + "Demon", + "Nonetheless", + "arro", + "CONT", + "ifted", + "Freder", + "isson", + "rout", + "ARA", + "swinging", + "Oct", + "liable", + "leaning", + "lungs", + "380", + "Process", + "Cov", + "terrorism", + "resistant", + "pumped", + "tripled", + "Semitism", + "Mia", + "penetration", + "Lutheran", + "BU", + "odes", + "spanning", + "utch", + "Trans", + "Volunteers", + "pathway", + "infectious", + "drastic", + "Engineers", + "princess", + "acts", + "usting", + "utive", + "achel", + "DO", + "pave", + "Herrera", + "nearing", + "help", + "embarked", + "modes", + "Driving", + "opting", + "Best", + "behavioral", + "cables", + "App", + "otion", + "Ext", + "Sinclair", + "Insp", + "sinking", + "Next", + "Lumpur", + "Shadow", + "Donald", + "itals", + "mentions", + "floor", + "considerations", + "Squad", + "Plate", + "dos", + "Friday", + "Hopefully", + "arre", + "alum", + "\":\"/", + "fet", + "anza", + "dign", + "Nguyen", + "Rutgers", + "Sew", + "filters", + "ofi", + "unavailable", + "ranking", + "refining", + "UNC", + "max", + "yll", + "handsome", + "utterly", + "See", + "Stores", + "Ke", + "Advoc", + "ordon", + "umbles", + "bugs", + "olar", + "Cork", + "token", + "authorization", + "conscience", + "repl", + "edi", + "owitz", + "iven", + "lieu", + "lifts", + "Lean", + "magnificent", + "Films", + "onents", + "***", + "Green", + "Advocate", + "Arrow", + "blows", + "exploited", + "fly", + "Amar", + "NOTICE", + "sincere", + "found", + "Rud", + "cy", + "Heidi", + "empowered", + "weakest", + "Kru", + "Credit", + "aunted", + "exotic", + "aning", + "aw", + "Multi", + "animation", + "850", + "Counter", + "Nit", + "alli", + "capitalize", + "executing", + "descent", + "ovi", + "Kimberly", + "headed", + "mentioning", + ")-", + "Specifically", + "ayette", + "ihad", + "Iss", + "disagreed", + "Kum", + "urges", + "permitting", + "py", + "isp", + "hygiene", + "mourning", + "cyclists", + "cats", + "FER", + "cycl", + "newcomers", + "plead", + "mend", + "secret", + "fan", + "translates", + "unit", + "Tank", + "drive", + "Site", + "acceleration", + "Enrique", + "Elaine", + "staring", + "backwards", + "ot", + "vot", + "HK", + "fian", + "Lockheed", + "manifest", + "Zurich", + "pad", + "Rav", + "flow", + "moms", + "Solid", + "Ready", + "aughlin", + "reminding", + "COR", + "optimal", + "Crisis", + "cholesterol", + "Gerard", + "fest", + "sanction", + "dragging", + "inent", + "Bravo", + "amend", + "aval", + "poem", + "invasive", + "landsc", + "leigh", + "headache", + "Muse", + "Turning", + "girl", + "cess", + "falsely", + "plaintiff", + "heavier", + "rumored", + "eleven", + "Consumers", + "Originally", + "Statement", + "bors", + "revoked", + "Omaha", + "Fox", + "Kle", + "vault", + "outdated", + "umes", + "Ark", + "apologised", + "rockets", + "Marines", + "captures", + "MW", + "Walters", + "Factor", + "ensuing", + "Session", + "oons", + "132", + "gt", + "Points", + "exhaust", + "Osaka", + "heed", + "handic", + "amber", + "inging", + "ll", + "escorted", + "floated", + "merge", + "compliment", + "VC", + "insulin", + "Debt", + "ça", + "pens", + "assertion", + "redevelopment", + "moderate", + "leftist", + "BA", + "herd", + "insecurity", + "liter", + "commence", + "Caucus", + "novels", + "Chevron", + "erosion", + "Nicholson", + "Roof", + "Volunteer", + "compelled", + "congratulated", + "Panel", + "ov", + "idelity", + "spect", + "bee", + "Assistance", + "terrified", + "iew", + "weekday", + "Higgins", + "special", + "ubs", + "anton", + "bribes", + "neat", + "Cliff", + "disqualified", + "ND", + "vers", + "andra", + "graft", + "value", + "portray", + "daytime", + "ksh", + "consist", + "honesty", + "Timber", + "Nich", + "invented", + "Buch", + "skull", + "tags", + "124", + "ighth", + "relaxing", + "Online", + "sanctioned", + "Sport", + "Cove", + "comics", + "MW", + "AMA", + "mother", + "Home", + "Customer", + "strides", + "Wins", + "rollout", + "Weaver", + "shuttle", + "steak", + "glorious", + "Toll", + "trustee", + "installations", + "Opportunity", + "oper", + "horse", + "aided", + "irus", + "sleek", + "yelled", + "Socialist", + "applaud", + "Wah", + "devote", + "dh", + "architectural", + "MAC", + "centric", + "Sense", + "illas", + "Archbishop", + "glass", + "allowance", + "bundle", + "andon", + "eight", + "Kare", + "haus", + "Andreas", + "doll", + "RAM", + "volunteering", + "Raleigh", + "bees", + "nickel", + "generosity", + "homeowner", + "Lieutenant", + "landfall", + "Renew", + "Giving", + "Contribut", + "aret", + "ulf", + "reinforce", + "Salv", + "Venice", + "freedoms", + "Tools", + "1962", + "Warm", + "majority", + "pleas", + "oding", + "plant", + "tow", + "Blanc", + "Pipeline", + "Moor", + "refrain", + "Explore", + "language", + "cers", + "WT", + "sent", + "Nun", + "plastics", + "acas", + "disruptions", + "discomfort", + "enko", + "imprisoned", + "Copyright", + "myriad", + "parenting", + "spree", + "NBC", + "onion", + "Israelis", + "RA", + "relocate", + "113", + "Hir", + "Dre", + "Dry", + "ONE", + "Administrator", + "prints", + "Gret", + "undergraduate", + "Lif", + "avers", + "Carney", + "apex", + "lenses", + "liberals", + "gb", + "Whereas", + "countryside", + "amine", + "Terminal", + "intr", + "Trey", + "ALS", + "continental", + "selfies", + "FILE", + "Unity", + "authoritarian", + "originated", + "Except", + "yna", + "monet", + "undermining", + "GS", + "pi", + "iq", + "slides", + "Summary", + "pains", + "cluding", + "equation", + "locked", + "fraternity", + "withstand", + "devastation", + "demo", + "late", + "punches", + "geared", + "nen", + "Bowie", + "attle", + "politic", + "Gle", + "mented", + "Coordinator", + "upwards", + "Mega", + "angled", + "engineered", + "luggage", + "Wen", + "Sergeant", + "kindergarten", + "Portsmouth", + "uddin", + "ket", + "oba", + "oscill", + "esse", + "Olson", + "Borough", + "supplements", + "Evening", + "ANE", + "lava", + "gearing", + "setting", + "urgical", + "asty", + "Daytona", + "brewery", + "pledges", + "rounder", + "ulous", + "Hancock", + "rex", + "ram", + "proceeding", + "Murdoch", + "downgrade", + "statues", + "debated", + "Sleep", + "144", + "Ruby", + "Fi", + "123", + "Arabic", + "lasts", + "Ivy", + "Wid", + "rown", + "stick", + "?'\"", + "STEM", + "sensible", + "htar", + "harbor", + "cra", + "Album", + "Carnival", + "implies", + "agement", + "Initially", + "chooses", + "Jeff", + "Hig", + "tam", + "lump", + "ucks", + "repatri", + "Mercy", + "zza", + "365", + "Ricardo", + "ogram", + "undergone", + "system", + "tel", + "Kee", + "ully", + "istas", + "grains", + "Tomorrow", + "RC", + "Turk", + "freshmen", + "Away", + "Sach", + "Ultimate", + "offensively", + "ismo", + "teaser", + "Jud", + "legitimacy", + "opt", + "Cobb", + "rejecting", + "Solo", + "Archer", + "southeastern", + "Plain", + "Loss", + "minerals", + "Mari", + "scrambling", + "Peak", + "havoc", + "rings", + "unofficial", + "Haj", + "director", + "Canal", + "NSA", + "Eaton", + "PART", + "Commissioners", + "wellbeing", + "resa", + "understandable", + "dates", + "Sorry", + "astonishing", + "revise", + "Ec", + "Lack", + "endi", + "endale", + "also", + "colder", + "heel", + "cellular", + "Conn", + "Thur", + "massage", + "olla", + "clus", + "toilets", + "Celebr", + "tackled", + "chorus", + "ETA", + "anca", + "OLED", + "punk", + "Brain", + "Nuggets", + "seamless", + "make", + "atted", + "Rog", + "Patch", + "ruined", + "Ins", + "consolidate", + "gospel", + "Caption", + "overweight", + "screened", + "Kraft", + "Bain", + "breaker", + "Feinstein", + "Doc", + "deepest", + "OL", + "tunes", + "rightly", + "Lanc", + "Brotherhood", + "poultry", + "Pure", + "stimulate", + "discourse", + "Stark", + "museums", + "ention", + "taxation", + "Akron", + "ayer", + "Kirby", + "farm", + "oser", + "commend", + "unarmed", + "ensions", + "superst", + "oceans", + "misuse", + "LO", + "Byrne", + "Maritime", + "dense", + "excuses", + "suppose", + "Marks", + "rainy", + "replicate", + "boutique", + "Renaissance", + "jas", + "icted", + "referenced", + "Tir", + "Hatch", + "Cry", + "PayPal", + "fulfil", + "Hawaiian", + "come", + "Thirty", + "260", + "Yak", + "angles", + "landlord", + "lavish", + "Women", + "NT", + "reinforced", + "prevail", + "Communities", + "footwear", + "assurances", + "lb", + "airing", + "resorts", + "Fiji", + "Shay", + "prevailing", + "many", + "impe", + "Dul", + "symbols", + "zb", + "Cere", + "applauded", + "soundtrack", + "drunken", + "Europeans", + "herds", + "moving", + "WR", + "Hindi", + "waking", + "Jo", + "Andrew", + "rosse", + "Legislative", + "disgrace", + "Nothing", + "Bulgaria", + "humidity", + "translation", + "measurements", + "vying", + "Brid", + "Max", + "dir", + "unci", + "defines", + "perfection", + "ancers", + "Matt", + "Shinzo", + "Presidents", + "ginger", + "onna", + "existing", + "rika", + "enced", + "Bray", + "gall", + "disrespect", + "Cumber", + "contestant", + "ucky", + "anticipated", + "abled", + "LLOW", + "Bel", + "Kear", + "storyline", + "rigs", + "Scots", + "Chap", + "Thankfully", + "communist", + "Adviser", + "regist", + "annoying", + "DVD", + "ethic", + "Filipino", + "Adidas", + "billing", + "alleviate", + "smoked", + "hazard", + "EV", + "Ag", + "baum", + "doses", + "outcry", + "inclined", + "psychologist", + "itzer", + "January", + "mornings", + "aught", + "surreal", + "Cannon", + "avy", + "Cris", + "cf", + "interpreted", + "persecution", + "vation", + "upfront", + "Waste", + "mills", + "bombings", + "Heaven", + "Flat", + "boxer", + "avenues", + "Invest", + "Zika", + "backstage", + "idas", + "eston", + "ead", + "bishops", + "render", + "footballer", + "spilled", + "Only", + "saddened", + "Above", + "inator", + "tro", + "onen", + "AMC", + "stringent", + "footing", + "Ghost", + "texting", + "CPI", + "UW", + "accol", + "iries", + "Flex", + "Carolyn", + "Andre", + "siege", + "Muslim", + "automobile", + "reci", + "dean", + "atre", + "wax", + "wo", + "Duffy", + "fiance", + "fib", + "eagle", + "Catal", + "infants", + "submitting", + "downhill", + "staffer", + "Lights", + "eater", + "Californ", + "supervisors", + "Py", + "condemnation", + "sci", + "hated", + "til", + "Lavrov", + "sab", + "motors", + "logging", + "Own", + "pi", + "repeating", + "DOJ", + "enary", + "Chow", + "fat", + "balcony", + "orie", + "NING", + "Unified", + "Neil", + "Bill", + "Sims", + "uten", + "LV", + "EMS", + "sip", + "replaces", + "ichi", + "Fig", + "Charity", + "peek", + "rack", + "cousins", + "resolving", + "throne", + "Engine", + "Chak", + "lamented", + "wipe", + "nutrients", + "Chat", + "AMP", + "Oprah", + "uming", + "serving", + "fir", + "landlords", + "neck", + "upload", + "unspecified", + "icy", + "´", + "ze", + "prohibits", + "FI", + "Res", + "Eff", + "hell", + "umbo", + "receipts", + "operatives", + "stant", + "wives", + "Cinema", + "negligence", + "gases", + "Lau", + "brew", + "August", + "never", + "penned", + "incomplete", + "Zh", + "esi", + "ranged", + "apolis", + "withdrawing", + "Levi", + "Levy", + "Daly", + "delaying", + "MSNBC", + "Cyrus", + "Nutrition", + "NN", + "winding", + "glow", + "MY", + "goodwill", + "MON", + "slots", + "Nina", + "FIR", + "LTE", + "Innov", + "dev", + "ctic", + "analyses", + "Bangalore", + "tales", + "overcame", + "Thurs", + "cherry", + "Nou", + "Flowers", + "1000", + "updated", + "rieve", + "Beautiful", + "iak", + "playback", + "headset", + "ashamed", + "Min", + "adm", + "Lucky", + "Tucson", + "entirety", + "ranging", + "Vance", + "kered", + "image", + "Gord", + "War", + "similarities", + "dig", + "Jude", + "lonely", + "hra", + "Staples", + "ACA", + "measurement", + "cooper", + "ATER", + "Meng", + "barring", + "190", + "Batt", + "reproductive", + "Rowe", + "subsid", + "slogans", + "ugar", + "Keller", + "ingham", + "fuel", + "hid", + "afe", + "indul", + "cash", + "stressing", + "MIT", + "trump", + "ancer", + "Pes", + "Mint", + "crossover", + "Weiss", + "Elvis", + "Permanent", + "Khalid", + "unjust", + "exceptionally", + "fut", + "avid", + "Ethics", + "utilized", + "feasibility", + "catering", + "Press", + "wayne", + "October", + "favors", + "obsession", + "melt", + "mug", + "MK", + "apples", + "vine", + "cliffe", + "grat", + "spells", + "ounced", + "decree", + "issy", + "Team", + "deploying", + "Feb", + "miserable", + "wat", + "Bust", + "Norris", + "Timberwolves", + "angered", + "Arn", + "oft", + "rome", + "advertisements", + "onal", + "nun", + "torque", + "slave", + "nonsense", + "coy", + "cites", + "Game", + "architects", + "playing", + "gener", + "socio", + "meditation", + "forgive", + "smiled", + "%),", + "pers", + "Soph", + "occupy", + "atton", + "witnessing", + "apologise", + "predecessors", + "Cassidy", + "tallied", + "NER", + "tract", + "Holder", + "Pav", + "jackets", + "Mel", + "raud", + "exercising", + "Chung", + "Amin", + "athi", + "Mem", + "racked", + "carved", + "Mickey", + "Lafayette", + "grill", + "INFORMATION", + "usc", + "Promotion", + "yson", + "istry", + "fulfilled", + "restraint", + "popping", + "Slater", + "mercy", + "aden", + "submarine", + "Bowling", + "dogs", + "Swe", + "noticeable", + "bis", + "Premiership", + "spat", + "Tow", + "Wand", + "mechanics", + "while", + "Benson", + "molecules", + "crosses", + "recalling", + "Certainly", + "HAM", + "sever", + "Rudy", + "DUI", + "OLD", + "Tobacco", + "subdued", + "quota", + "TF", + "flats", + "emphasize", + "belts", + "Opinion", + "piled", + "Spark", + "Elias", + "classification", + "Hands", + "CV", + "toast", + "candle", + "atching", + "short", + "Dup", + "ult", + "bats", + "marketers", + "Avery", + "Colbert", + "Ik", + "Vac", + "Jackets", + "merits", + "eli", + "PORT", + "elevator", + "irming", + "effective", + "groceries", + "hi", + "INTER", + "SAP", + "NYPD", + "KY", + "angel", + "spectacle", + "ré", + "Roche", + "insects", + "commenced", + "Foley", + "darker", + "Ug", + "Mostly", + "termed", + "uci", + "Exec", + "Brittany", + "harmony", + "advocated", + "parcel", + "Hots", + "monarch", + "Siri", + "odge", + "Pag", + "progressing", + "grounds", + "onstage", + "warmth", + "Won", + "violates", + "Saudis", + "bumper", + "patrols", + "Barron", + "indoors", + "tar", + "Each", + "Val", + "applicant", + "Cater", + "classics", + "Threat", + "wrapping", + "Idlib", + "anking", + "Did", + "adia", + "Rig", + "Bram", + "Laurie", + "Hair", + "Cannabis", + "daylight", + "Norm", + "Rip", + "sin", + "unta", + "Pass", + "Acad", + "Cummings", + "theirs", + "Distribution", + "especially", + "grilled", + "affiliates", + "Vander", + "Cath", + "Productions", + "Trek", + "230", + "casinos", + "Cain", + "atu", + "idget", + "Winds", + "unanswered", + "intercept", + "Marty", + "refin", + "lieutenant", + "cas", + "Chief", + "average", + "ilot", + "scrimmage", + "Mud", + "speaking", + "Franken", + "Tories", + "abstract", + "awar", + "Terms", + "dal", + "Fur", + "humour", + "rh", + "situ", + "aed", + "FIN", + "transcripts", + "approved", + "Parsons", + "pigs", + "repayment", + "ARM", + "Elliot", + "Levine", + "tagged", + "pun", + "Dwight", + "configuration", + "sis", + "Adult", + "earthquakes", + "creature", + "MRI", + "mach", + "prescriptions", + "cover", + "ministries", + "inaccurate", + "Labs", + "MGM", + "tomato", + "eng", + "opposes", + "owan", + "mapping", + "consum", + "online", + "eters", + "code", + "Aug", + "Point", + "branded", + "pling", + "Calder", + "Oper", + "Middles", + "champagne", + "Tues", + "sampling", + "energetic", + "rano", + "Styles", + "neglected", + "Damon", + "endanger", + "southwestern", + "ATM", + "Duck", + "engers", + "dan", + "yth", + "bou", + "Decl", + "Gold", + "projecting", + "Google", + "Hussein", + "accomplishment", + "itarian", + "gossip", + "Rai", + "ril", + "Ske", + "psychiatric", + "MacBook", + "Adobe", + "Hodg", + "accompany", + "advertised", + "reminiscent", + "geographical", + "convertible", + "IK", + "CTV", + "communal", + "chim", + "selfish", + "drilled", + "tortured", + "blacks", + "noon", + "manifesto", + "Richie", + "acco", + "Im", + "debit", + "SNP", + "perfect", + "gard", + "Ratio", + "stubborn", + "accumulation", + "congregation", + "kissing", + "killers", + "Abbey", + "von", + "Fuj", + "Isabel", + "NB", + "Nish", + "Julius", + "Zimmer", + "uncover", + "dar", + "isle", + "Compar", + "counselor", + "Sok", + "Cumm", + "Hip", + "urgently", + "rentals", + "approving", + "irrigation", + "prostate", + "Judicial", + "Submit", + "Tanner", + "attack", + "emb", + "reclaim", + "ec", + "brutality", + "commanding", + "reasoning", + "Roy", + "Elect", + "Mobil", + "anding", + "mirrors", + "Israel", + "pavement", + "overdue", + "Md", + "street", + "thrill", + "pora", + "azon", + "brewing", + "enge", + "Disaster", + "builder", + "ods", + "utsch", + "terminals", + "Baird", + "enburg", + "hast", + "brass", + "parental", + "enture", + "Conduct", + "expands", + "luck", + "mur", + "Bj", + "administrations", + "Olivier", + "oux", + "narrowed", + "winner", + "makeshift", + "VAT", + "Javier", + "-,", + "systematic", + "enforcing", + "emin", + "Audio", + "United", + "gener", + "Kara", + "ivas", + "Pretty", + "Lob", + "petitions", + "Mercer", + "ampa", + "product", + "distributing", + "tunnels", + "condo", + "RSS", + "Carlo", + "pumpkin", + "sto", + "assumes", + "oway", + "hiba", + "lection", + "gam", + "Aires", + "transmitted", + "trousers", + "cheers", + "Jensen", + "emer", + "simpler", + "colored", + "Sustainable", + "instruct", + "poles", + "supervised", + "integ", + "Moreno", + "boarding", + "igrant", + "Yoga", + "environmentally", + "sacrifices", + "shores", + "127", + "estranged", + "intoxicated", + "emergencies", + "Kosovo", + "yang", + "fastball", + "packaged", + "LAN", + "hurry", + "Manny", + "porch", + "curiosity", + "Kend", + "thouse", + "Tou", + "mun", + "waving", + "passwords", + "Swan", + "prefers", + "Corrections", + "aic", + "ejected", + "dossier", + "Chal", + "facto", + "spine", + "leck", + "restriction", + "disagreement", + "grown", + "Edgar", + "quantities", + "Rapid", + "pals", + "spared", + "remarkably", + "ructure", + "backers", + "Goals", + "cles", + "rolling", + "Blasio", + "orchestra", + "ologies", + "Rise", + "Power", + "uptick", + "atha", + "Mob", + "shotgun", + "downs", + "Borg", + "morale", + "Call", + "wave", + "Duc", + "unwilling", + "oad", + "businessmen", + "refriger", + "gamers", + "cele", + "precip", + "renegoti", + "OY", + "Pharm", + "responsive", + "servant", + "eye", + "raping", + "vas", + "groin", + "Melvin", + "Kurds", + "stricter", + "Mum", + "ients", + "standalone", + "forums", + "commemorate", + "Far", + "Telegram", + "screenings", + "Leonardo", + "ighton", + "DOWN", + "module", + "remedy", + "280", + "Su", + "Becker", + "Gast", + "prem", + "Into", + "oyle", + "114", + "adhere", + "Report", + "Janeiro", + "Kry", + "Pakistan", + "robotic", + "ande", + "overlooking", + "Treaty", + "rect", + "yne", + "battlefield", + "Geoff", + "earns", + "Miner", + "teased", + "exemptions", + "vacancy", + "oku", + "vulnerabilities", + "Rou", + "observ", + "overlook", + "correspond", + "theatrical", + "robotics", + "Compl", + "Pasadena", + "laden", + "vastly", + "olit", + "justification", + "tampering", + "Sutherland", + "Mens", + "invisible", + "uren", + "Ashton", + "owl", + "disqual", + "Eva", + "friction", + "Irvine", + "aliens", + "Pension", + "Assets", + "Benedict", + "ittal", + "sword", + "underwear", + "Farmer", + "timber", + "dependence", + "Tang", + "165", + "Nazis", + "punching", + "Gloria", + "usat", + "luxurious", + "chuk", + "Cot", + "regained", + "reassure", + "hello", + "ante", + "negotiators", + "Add", + "paced", + "ér", + "demolished", + "Ann", + "joy", + "Jenna", + "Apple", + "disturbance", + "commissions", + "Politico", + "along", + "nem", + "auctions", + "ruck", + "OD", + "ofer", + "Play", + "carn", + "vez", + "tents", + "congratulate", + "Liquid", + "Coyotes", + "uku", + "Allah", + "bend", + "canvas", + "Clifford", + "volunteered", + "Luc", + "bp", + "Census", + "Shot", + "anonymously", + "Anglo", + "Bayer", + "Aber", + "Correctional", + "hardship", + "Buenos", + "Daw", + "baskets", + "upstairs", + "mindful", + "LCD", + "Blackburn", + "Hale", + "477", + "circus", + "Dragons", + "rubble", + "rb", + "headaches", + "aunt", + "itus", + "scaled", + "Comic", + "asio", + "Nordic", + "Per", + "bombers", + "ilitation", + "indirectly", + "Hod", + "andan", + "operation", + "puppy", + "Mats", + "stewards", + "roup", + "memorandum", + "patio", + "const", + "Bold", + "Kaiser", + "Following", + "compat", + "sidewalks", + "Fitzpatrick", + "sunlight", + "Lever", + "Becky", + "icles", + "Probably", + "garner", + "Tomas", + "blankets", + "uga", + "jiang", + "revel", + "Hutch", + "llers", + "trimmed", + "STR", + "KR", + "Pike", + "ASS", + "Bay", + "diagnostic", + "Steph", + "toured", + "Avoid", + "vic", + "Without", + "Clinical", + "blo", + "undo", + "Boise", + "speculated", + "Prot", + "vention", + "scholar", + "Sta", + "Featured", + "Prev", + "penny", + "Hath", + "rawn", + "renovated", + "Fried", + "itol", + "uddle", + "inquest", + "metropolitan", + "lights", + "tempo", + "onom", + "Import", + "Asia", + "owes", + "magistrate", + "Friedman", + "contacting", + "strains", + "homage", + "lent", + "ception", + "git", + "lively", + "scra", + "WW", + "ön", + "rill", + "Jack", + "Shank", + "iani", + "decreasing", + "MON", + "Supervisor", + "Cats", + "Fusion", + "racially", + "Tara", + "Purchase", + "Rally", + "Graph", + "Hello", + "hest", + "Varg", + "drowned", + "Thu", + "Wet", + "Eug", + "rainbow", + "telev", + "Amir", + "Based", + "cookie", + "uding", + "contracting", + "objected", + "fork", + "acent", + "Til", + "Lilly", + "Eur", + "hormone", + "nails", + "Fischer", + "pier", + "EMENT", + "eruption", + "visory", + "speculate", + "apan", + "Jub", + "Huckabee", + "string", + "stay", + "sustaining", + "VM", + "priv", + "clos", + "downloaded", + "Iv", + "financed", + "Sao", + "Everett", + "rene", + "Wo", + "Piet", + "engulfed", + "exiting", + "uni", + "horn", + "grav", + "ection", + "drainage", + "fuelled", + "organizational", + "bike", + "Areas", + "policeman", + "Firm", + "Slide", + "rand", + "Jedi", + "Ge", + "really", + "Manchester", + "Wise", + "parent", + "lad", + "urine", + "Colombian", + "geon", + "1961", + "Mania", + "graph", + "cod", + "fred", + "effic", + "Gateway", + "asket", + "diminished", + "Mass", + "205", + "Long", + "granddaughter", + "shining", + "Semitic", + "arising", + "330", + "DU", + "Zah", + "exclusion", + "Claus", + "ven", + "oine", + "API", + "reve", + "militias", + "fro", + "waved", + "Luxembourg", + "diamonds", + "stabilize", + "queue", + "Sponsor", + "eldest", + "Lud", + "wasting", + "dimension", + "motorcycles", + "ucker", + "Tav", + "supremacy", + "Take", + "CPU", + "cup", + "disregard", + "envelope", + "Cah", + "proposes", + "Maurice", + "hobby", + "harmon", + "ribbon", + "Origin", + "builders", + "conj", + "cert", + "eat", + "Stern", + "ulia", + "vals", + "cling", + "provocative", + "softer", + "1948", + "remod", + "Sob", + "maxim", + "blueprint", + "oit", + "Garner", + "fibre", + "search", + "Write", + "270", + "clergy", + "Palo", + "obile", + "Mad", + "clown", + "traced", + "280", + "Alberto", + "drums", + "Fridays", + "Strat", + "stated", + "Stevenson", + "Pr", + "boasted", + "Brees", + "Donn", + "Maya", + "relieve", + "1080", + "cheapest", + "uniquely", + "jungle", + "prevalence", + "outfield", + "Maps", + "accustomed", + "pac", + "combinations", + "Soros", + "stad", + "ket", + "disgusting", + "OFF", + "irs", + "biased", + "paved", + "iked", + "utterstock", + "ocal", + "surround", + "Guang", + "spear", + "Bellev", + "ortun", + "Rec", + "acho", + "frightening", + "tyres", + "normal", + "Yan", + "Warsaw", + "Bod", + "ourse", + "199", + "Ver", + "erent", + "sparkling", + "chanting", + "1945", + "turbo", + "hazards", + "IRE", + "Ronnie", + "splitting", + "Matte", + "roph", + "tended", + "vandalism", + "alis", + "SY", + "oversaw", + "Happy", + "TC", + "275", + "eco", + "Kers", + "extensions", + "Flan", + "Cena", + "Downs", + "drummer", + "awaited", + "ACL", + "legends", + "Rollins", + "hend", + "departing", + "tha", + "unre", + ".(", + "faded", + "retirees", + "vid", + "entrants", + "Stella", + "arer", + "teaspoon", + "Sheridan", + "irc", + "Relief", + "Butt", + "ris", + "undermined", + "sunk", + "Sam", + "kamp", + "riot", + "rating", + "clubhouse", + "peaked", + "Ski", + "airstrikes", + "conce", + "CPR", + "esp", + "Wave", + "Coliseum", + "outheastern", + "trou", + "feather", + "Soy", + "Bihar", + "intervened", + "mits", + "colored", + "330", + "procession", + "apeake", + "ité", + "riel", + "mart", + "afer", + "Guests", + "Pie", + "shiny", + "Sixers", + "Roads", + "kicker", + "Crimes", + "frontier", + "ansen", + "November", + "smith", + "Laun", + "fried", + "weet", + "Grass", + "sanitation", + "Eat", + "Parts", + "Tun", + "amar", + "Jupiter", + "FS", + "unsc", + "Done", + "leveraging", + "tucked", + "ineffective", + "riots", + "wei", + "Attend", + "pertaining", + "amen", + "monds", + "mism", + "serious", + "Viol", + "rous", + "129", + "uebl", + "umption", + "tri", + "Wedding", + "troopers", + "THR", + "olving", + "leys", + "Med", + "separatists", + "imper", + "Frontier", + "whit", + "Mutual", + "rested", + "unhealthy", + "gang", + "researching", + "Colonel", + "affordability", + "Regarding", + "Wend", + "Mellon", + "plots", + "canal", + "PER", + "Shopping", + "etry", + "occurrence", + "graves", + "BF", + "Kau", + "indust", + "beard", + "uate", + "Produ", + "Somali", + "ishers", + "Fell", + "Hutchinson", + "hust", + "illustration", + "//", + "sharks", + "coincidence", + "remake", + "mural", + "course", + "Sultan", + "arse", + "whip", + "Podcast", + "tightened", + "denim", + "landfill", + "future", + "superv", + "Hand", + "praising", + "Ely", + "Gust", + "Mayer", + "orphan", + "repaired", + "Pir", + "spiral", + "husband", + "ienne", + "iatric", + "marriages", + "horn", + "plain", + "Lum", + "ession", + "Features", + "breakup", + "entrepreneurship", + "rina", + "embargo", + "capitalism", + "Minor", + "promo", + "excel", + "Japan", + "worsening", + "stumbled", + "pins", + "swipe", + "exile", + "separatist", + "Bian", + "relocation", + "commanders", + "downed", + "blogger", + "packed", + "Schn", + "waterfront", + "Yus", + "negotiator", + "favourable", + "Iran", + "oulder", + "cance", + "vind", + "angel", + "authenticity", + "towel", + "bul", + "Neville", + "Buddhist", + "fields", + "uly", + "niece", + "corrections", + "assignments", + "Schl", + "harmed", + "375", + "wounding", + "Position", + "supermarkets", + "disclosures", + "185", + "esp", + "McCull", + "Male", + "sailors", + "mis", + "Sophia", + "unfolded", + "owell", + "Scarborough", + "entrepreneurial", + "118", + "ogy", + "Likewise", + "swung", + "drawings", + "drafting", + "Simple", + "Filip", + "arf", + "fade", + "merged", + "Leaf", + "sun", + "flame", + "indices", + "Create", + "ittle", + "Wer", + "Mond", + "oz", + "Smoke", + "replies", + "DH", + "jud", + "Falk", + "---", + "constitutes", + "theat", + "119", + "intermediate", + "vill", + "Gow", + "Hut", + "ł", + "155", + "Located", + "Door", + "sliced", + "aru", + "tearing", + "defense", + "oyer", + "produ", + "seminar", + "asso", + "peaks", + "conceal", + "crypto", + "setbacks", + "Alicia", + "FAA", + "continuity", + "catastrophe", + "beg", + "scales", + "apixel", + "salon", + "Ste", + "lesbian", + "anticip", + "utilization", + "chickens", + "spinal", + "Juliet", + "Fas", + "prising", + "Salvation", + "138", + "utilizing", + "âĢ¢", + "Messenger", + "rebellion", + "Alexand", + "insect", + "ribs", + "Bild", + "monopoly", + "Queen", + "Naples", + "133", + "hourly", + "ego", + "pencil", + "Pew", + "desirable", + "vant", + "LAT", + "perpet", + "lish", + "201", + "distances", + "distressed", + "Work", + "tattoos", + "stereotypes", + "istent", + "Coral", + "fo", + "payable", + "akin", + "Lis", + "Finding", + "susceptible", + "Kiw", + "forgiveness", + "Moment", + "Dmitry", + "renov", + "quint", + "Waterloo", + "Reality", + "stray", + "Beaver", + "bites", + "elusive", + "virtue", + "gadgets", + "landslide", + "Healthy", + "pits", + "Donnell", + "irony", + "uct", + "practitioners", + "reck", + "governmental", + "atomic", + "motiv", + "polic", + "communicated", + "HS", + "criticize", + "synerg", + "Del", + "Roe", + "inspirational", + "Warning", + "pel", + "nevertheless", + "despair", + "(.", + "fearing", + "grop", + "tree", + "trusts", + "interviewing", + "amic", + "scor", + "ject", + "Another", + "pose", + "depicted", + "Photography", + "Lenovo", + "Epic", + "Boot", + "GI", + "enses", + "Class", + "arity", + "servicing", + "Hann", + "awe", + "overdoses", + "Finnish", + "pav", + "PCs", + "SEC", + "Stro", + "attracts", + "apprehended", + "128", + "unstable", + "Outdoor", + "cloth", + "Ulster", + "visually", + "sculpt", + "sufficiently", + "Kendrick", + "engages", + "knives", + "Gut", + "arbit", + "osition", + "emoji", + "pinpoint", + "remembering", + "rence", + "Vish", + "improperly", + "ranc", + "upstream", + "checkpoint", + "rash", + "eson", + "toes", + "260", + "invalid", + "onions", + "lashed", + "Dong", + "provisional", + "Fern", + "irresponsible", + "actively", + "Known", + "ben", + "Blank", + "actresses", + "paying", + "syrup", + "isman", + "educating", + "Sunday", + "ifiable", + "Post", + "calculation", + "hesitate", + "Increasing", + "reeling", + "Dairy", + "ensing", + "maternity", + "Ø", + "./", + "Elm", + "weddings", + "Yard", + "117", + "Rocket", + "OF", + "treasurer", + "rattled", + "Drop", + "arel", + "Fulton", + "Giant", + "Floor", + "Jet", + "ikk", + "Bucs", + "ostics", + "reme", + "Rouse", + "deliber", + "Ele", + "conducts", + "Blog", + "connected", + "prayed", + "colourful", + "augmented", + "batted", + "relevance", + "Romanian", + "acqu", + "Chel", + "Clo", + "Graves", + "chees", + "Gibbs", + "CLE", + "fertility", + "ambul", + "specs", + "IRA", + "Booth", + "ithe", + "Playoff", + "ammed", + "collaborating", + "lunar", + "confronting", + "attribute", + "King", + "riz", + "casualty", + "acia", + "waters", + "paving", + "caregivers", + "nor", + "reacting", + "Hash", + "squeezed", + "exert", + "Michele", + "Conc", + "Hep", + "sewage", + "wart", + "GY", + "discourage", + "Fir", + "textile", + "Spice", + "Fah", + "complainant", + "instinct", + "camp", + "Edison", + "VIDEOS", + "LM", + "Sands", + "About", + "disk", + "brid", + "muted", + "ACC", + "wre", + "event", + "icons", + "Express", + "udes", + "Beatles", + "color", + "Haas", + "Wolfe", + "YOUR", + "accessibility", + "Cornwall", + "ing", + "atrocities", + "weather", + "Dominion", + "MIL", + "Lara", + "unravel", + "maneuver", + "foam", + "ribe", + "CI", + "candles", + "acs", + ")(", + "coon", + "Purple", + "Governors", + "Keystone", + "Yuk", + "file", + "viol", + "gery", + "370", + "train", + "gunshots", + "olin", + "viruses", + "Tex", + "hours", + "prev", + "Rid", + "ected", + "Vog", + "riers", + "murdering", + "Iz", + "deliberations", + "arming", + "unda", + "rink", + "Drugs", + "idered", + "forge", + "expansive", + "VIEW", + "Bots", + "switches", + "KO", + "atten", + "variants", + "Virtual", + "Coch", + "yon", + "Kai", + "bullied", + "iday", + "version", + "lib", + "Cec", + "igated", + "TRUMP", + "Pod", + "toppled", + "eyeing", + "Patients", + "techn", + "hampered", + "avert", + "Scheme", + "Corm", + "pony", + "zoom", + "abo", + "sleeves", + "lane", + "Lester", + "Dane", + "cough", + "signings", + "HER", + "sibling", + "redemption", + "stockp", + "Algeria", + "padd", + "Brenda", + "uchi", + "transporting", + "speculative", + "Sek", + "abal", + "shipment", + "oker", + "warranty", + "atan", + "blister", + "Celebration", + "wal", + "lac", + "prioritize", + "ression", + "BP", + "collaborated", + "Newsletter", + "Damian", + "Residential", + "gra", + "feasible", + "Crest", + "Bean", + "Sturgeon", + "Tale", + "Contin", + "Mush", + "rocking", + "Mane", + "Humane", + "resistant", + "Fra", + "highest", + "fts", + "amassed", + "Pavilion", + "Skin", + "unfold", + "resur", + "PET", + "model", + "employing", + "rude", + "irrelevant", + "angu", + "Page", + "PN", + "igator", + "Reb", + "Arrest", + "Gund", + "malls", + "zhen", + "wed", + "daring", + "factual", + "Gent", + "informing", + "Stri", + "Lounge", + ".]", + "Tribunal", + "Moines", + "shadows", + "generated", + "fulness", + "heartfelt", + "Livingston", + "Clerk", + "nationalism", + "Miche", + "balls", + "anos", + "agle", + "prejudice", + "evenly", + "swearing", + "exits", + "condemning", + "vanilla", + "club", + "Funding", + "Dover", + "hots", + "fres", + "goodness", + "McKay", + "bulls", + "avia", + "129", + "1947", + "defamation", + "Moran", + "irms", + "Fitz", + "Rossi", + "urated", + "variation", + "Bauer", + "Schro", + "colony", + "Parliamentary", + "ikan", + "stirring", + "Sheldon", + "accessory", + "Utilities", + "nab", + "pract", + "herein", + "Role", + "Mant", + "pharm", + "215", + "NGO", + "Anything", + "Macedonia", + "bree", + "WTO", + "Chicago", + "Protect", + "quarters", + "Grassley", + "Interactive", + "Interview", + "550", + "astronauts", + "freak", + "Integrated", + "indict", + "generators", + "acio", + "Kevin", + "vaccination", + "blockade", + "Sons", + "capita", + "Anita", + "Export", + "Nex", + "Aram", + "zinc", + "revamped", + "selective", + "manipulate", + "Bedford", + "Battery", + "qualifiers", + "lean", + "screw", + "film", + "ror", + "Ellison", + "ombo", + "Ost", + "165", + "slaves", + "Payton", + "barg", + "rugged", + "Winn", + "Hammer", + "UPS", + "Euro", + "unfamiliar", + "distract", + "buffer", + "ledge", + "trunk", + "320", + "122", + "dilemma", + "pra", + "utmost", + "campaigners", + "icular", + "eful", + "�", + "HQ", + "neau", + "sir", + "test", + "Company", + "rescind", + "ardon", + "MG", + "Gov", + "Raz", + "rod", + "fed", + "psych", + "unin", + "Arbor", + "newcomer", + "Edwin", + "raising", + "quist", + "discoveries", + "Steve", + "scramble", + "js", + "acoustic", + "deterioration", + "observing", + "Winning", + "Saban", + "idy", + "overd", + "scouting", + "punitive", + "Shelter", + "mocked", + "dreamed", + "invaluable", + "LP", + "standard", + "recounted", + "Sabres", + "points", + "fringe", + "Barker", + "alian", + "PROV", + "cartel", + "overcrowd", + "tain", + "Year", + "Welfare", + "Chr", + "introduces", + "Doing", + "Glover", + "deteriorating", + "Par", + "attendant", + "Mold", + "Flying", + "ovan", + "optimize", + "chapters", + "dull", + "gay", + "ATP", + "Kah", + "ainer", + "feet", + "joking", + "disadvantage", + "Rep", + "twisted", + "slain", + "comprise", + "restricting", + "dispos", + "shaky", + "embattled", + "owe", + "conscious", + "oken", + "mistaken", + "Dra", + "reservoir", + "spate", + "Scott", + "avor", + "qual", + "amel", + "hunt", + "Chevy", + "claw", + "witch", + "Zimmerman", + "arium", + "rubbish", + "strings", + "doc", + "plaque", + "Cyr", + "flourish", + "worthwhile", + "banners", + "Lemon", + "Rainbow", + "consisted", + "HOW", + "Ñ", + "blogs", + "CLUS", + "eely", + "beast", + "Mai", + "hostility", + "eros", + "foreseeable", + "Corker", + "WEEK", + "visors", + "ressive", + "Viktor", + "bureaucracy", + "256", + "Feel", + "Adventure", + "efficacy", + "Institution", + "Harbaugh", + "Practice", + "Christianity", + "Thanks", + "fridge", + "idel", + "eff", + "vein", + "terms", + "ignorance", + "scream", + "wit", + "Rousse", + "Willow", + "hallway", + "former", + "shooters", + "Reporting", + "gal", + "savvy", + "rand", + "remed", + "Baron", + "inar", + "seizures", + "Thorn", + "Protesters", + "Revolutionary", + "think", + "Cabrera", + "Four", + "Rudd", + "prost", + "Bottom", + "Port", + "nas", + "ifax", + "Wire", + "tokens", + "antis", + "SOU", + "Milk", + "asters", + "shrimp", + "cakes", + "blue", + "ifty", + "View", + "adium", + "fen", + "zyk", + "Emil", + "dismay", + "tilt", + "aska", + "Young", + "predators", + "overshadowed", + "mitt", + "Semin", + "Schiff", + "Clarkson", + "212", + "210", + "vanished", + "mesh", + "Burnett", + "Ment", + "Blind", + "Patriot", + "Vil", + "flick", + "Towns", + "Whites", + "spice", + "Mode", + "nominate", + "wrest", + "Ashes", + "rows", + "Clint", + "gentleman", + "utan", + "athlon", + "Intermediate", + "hews", + "offended", + "Paige", + "Finch", + "Aboriginal", + "positive", + "Stop", + "renting", + "[âĢ¦]", + "Hert", + "vegetation", + "apes", + "Canon", + "appa", + "abst", + "Katz", + "surfing", + "aghan", + "Presidency", + "scaling", + "Sas", + "peanut", + "recommending", + "cious", + "endez", + "eker", + "Kamp", + "sitcom", + "crust", + "women", + "Jes", + "Whe", + "Warwick", + "epit", + "Alc", + "dictate", + "SPORTS", + "Language", + "indicative", + "MacDonald", + "reorgan", + "`", + "ARS", + "liberation", + "bless", + "reflective", + "à¤", + "desires", + "Hank", + "Launch", + "rotating", + "Stones", + "coordinating", + "Zeit", + "skepticism", + "Alam", + "Trout", + "SMS", + "Crescent", + "Teacher", + "fury", + "eyebrows", + "onga", + "Pilot", + "Rutherford", + "interstate", + "established", + "baggage", + "131", + "riks", + "mil", + "neon", + "queer", + "ourced", + "Kash", + "Eleven", + "illes", + "Opportun", + "stre", + "Washington", + "Different", + "exempl", + "boarded", + "rogue", + "DNC", + "rone", + "reversing", + "nine", + "Ivory", + "itating", + "uve", + "fracture", + "255", + "Assessment", + "subjective", + "fluct", + "Jaguar", + "stride", + "reapp", + "Grow", + "against", + "Medina", + "scenes", + "Nieto", + "sou", + "Fleming", + "narcotics", + "Bere", + "Bub", + "Ack", + "vinyl", + "Copy", + "Garland", + "Duty", + "inn", + "merchant", + "activate", + "glowing", + "ettle", + "Bran", + "silk", + "anco", + "TL", + "Furn", + "withheld", + "pulse", + "GU", + "BUS", + "Hyper", + "picnic", + "positives", + "Paramount", + "737", + "enlisted", + "Valerie", + "false", + "Chocolate", + "STAR", + "descended", + "tasty", + "Daesh", + "Ned", + "complimentary", + "depicting", + "Havana", + "college", + "traces", + "undue", + "Sisters", + "aum", + "Courier", + "Ong", + "Sparks", + "ongs", + "Yong", + "URR", + "los", + "horsepower", + "confidence", + "Pett", + "Measure", + "marches", + "zig", + "TOR", + "exported", + "Rak", + "Investigations", + "terminate", + "Tian", + "masters", + "DS", + "outraged", + "Cups", + "Weir", + "exec", + "journeys", + "abide", + "avail", + "Streets", + "fixes", + "cocoa", + "abundant", + "hubs", + "mort", + "robberies", + "Bark", + "precautions", + "hammered", + "ometric", + "mith", + "McCann", + "Jaw", + "Quest", + "McF", + "lob", + "legalized", + "quirky", + "trailers", + "Individual", + "cumulative", + "enlarge", + "convoy", + "olen", + "got", + "landers", + "scanner", + "scans", + "Eg", + "prof", + "hosp", + "Colo", + "err", + "deval", + "Usually", + "bul", + "ummy", + "tandem", + "occupied", + "mandates", + "Swim", + "121", + "ussed", + "EF", + "fries", + "Until", + "rc", + "badge", + "strips", + "magnet", + "archive", + "stan", + "Deadline", + "disposable", + "bob", + "northwestern", + "Jul", + "SAL", + "influencing", + "devil", + "Ellie", + "cms", + "ingo", + "888", + "cosmetic", + "Also", + "yacht", + "lazy", + "merc", + "absorbed", + "harm", + "116", + "subpoena", + "counters", + "Lori", + "randomly", + "nea", + "waves", + "relie", + "Kiss", + "chassis", + "bakery", + "Images", + "Holden", + "amazed", + "alignment", + "Powers", + "labelled", + "staunch", + "signaling", + "senate", + "unconventional", + "Alternative", + "ambassadors", + "VPN", + "atics", + "mosquito", + "Scholarship", + "helpless", + "alone", + "ZA", + "chel", + "constituencies", + "Café", + "hatch", + "Rupert", + "rendering", + "reinstated", + "interval", + "Texas", + "AHL", + "February", + "review", + "gle", + "fals", + "markers", + "governmental", + "Pos", + "arose", + "every", + "rulings", + "obar", + "Govern", + "gren", + "isan", + "marketed", + "Click", + "ord", + "balloons", + "asers", + "Horton", + "pub", + "Aerospace", + "flank", + "molecular", + "bour", + "nuts", + "alliances", + "benchmarks", + "ocate", + "stadt", + "Goodwin", + "lap", + "Factors", + "Never", + "Nem", + "roadside", + "orth", + "exhibited", + "Pearce", + "Olsen", + "postal", + "Liberation", + "reen", + "mary", + "ropes", + "larg", + "gob", + "boys", + "Sax", + "reimbursement", + "Vie", + "Catholics", + "Martial", + "premiered", + "awaits", + "Understanding", + "Belarus", + "Vor", + "ogi", + "iaz", + "victorious", + "ancestors", + "wreckage", + "oppression", + "Childhood", + "width", + "Plymouth", + "Fifty", + "occupancy", + "etts", + "Fiscal", + "lifting", + "Traditional", + "nostalgia", + "Law", + "lays", + "arresting", + "anticipating", + "insults", + "Extension", + "generator", + "ummer", + "ageing", + "bouncing", + "ember", + "WAR", + "Nico", + "Wow", + "Raven", + "flower", + "Crim", + "bh", + "undo", + "burgers", + "roud", + "Atkinson", + "YEAR", + "poorer", + "ICA", + "Schedule", + "stronghold", + "Millennium", + "###", + "ilda", + "GH", + "upscale", + "aldi", + "Resolution", + "swelling", + "grieving", + "Nile", + "Tig", + "ERY", + "ooth", + "BALL", + "ballet", + "bucks", + "UV", + "akin", + "chilling", + "databases", + "GD", + "section", + "hires", + "mul", + "sen", + "Townsend", + "inspected", + "ilic", + "discriminatory", + "fol", + "alcoholic", + "Hoff", + "Carl", + "vicinity", + "lein", + "Eco", + "Govern", + "secrecy", + "aned", + "DUP", + "570", + "sow", + "stalls", + "insulting", + "DT", + "informs", + "fitting", + "Depending", + "Melanie", + "Thom", + "path", + "admired", + "Peter", + "idents", + "ielding", + "Shanahan", + "TD", + "Things", + "sn", + "constituted", + "137", + "derailed", + "Bonnie", + "graffiti", + "earnest", + "compliant", + "blown", + "alle", + "prise", + "focal", + "gentlemen", + "Talks", + "passports", + "deprived", + "dude", + "Nath", + "governed", + "sac", + "castle", + "qv", + "tolerated", + "Sci", + "close", + "Dynamics", + "flashing", + "yk", + "Consolid", + "inherently", + "Forrest", + "Gene", + "Public", + "loser", + "runners", + "prudent", + "pioneering", + "Howe", + "Butter", + "Arabian", + "acha", + "BBQ", + "Mineral", + "destiny", + "retrieve", + "Bav", + "reth", + "oby", + "Grid", + "grievances", + "Tips", + "adamant", + "diets", + "milestones", + "collects", + "Laboratories", + "WC", + "postp", + "dams", + "OEM", + "rumor", + "locking", + "emission", + "queries", + "Jones", + "lang", + "Acqu", + "Medium", + "Treasurer", + "Sept", + "FB", + "integrating", + "bolstered", + "incorporating", + "encers", + "irregularities", + "nom", + "iod", + "Ai", + "sor", + "anked", + "rehears", + "fig", + "Bug", + "hoff", + "trooper", + "galaxy", + "amon", + "Atlas", + "solicit", + "sings", + "Instructions", + "Mig", + "thinking", + "Costco", + "breasts", + "portraits", + "Cock", + "subscriptions", + "pine", + "haunted", + "MED", + "eer", + "ega", + "Za", + "ENN", + "Winners", + "aith", + "safe", + "143", + "Weston", + "Lansing", + "Laurel", + "ocrat", + "ograph", + "matchups", + "Friend", + "digest", + "dimensions", + "azing", + "tipping", + "enrich", + "gart", + "argo", + "outbreaks", + "salvage", + "Erica", + "modules", + "PDF", + "Goods", + "oots", + "2011", + "interrupt", + "radi", + "Simone", + "vell", + "SV", + "extremely", + "stadiums", + "Rox", + "conflicting", + "youthful", + "UM", + "series", + "ded", + "fielding", + "Pre", + "itled", + "streamed", + "apprentices", + "Alec", + "Gap", + "Prem", + "leased", + "deepening", + "bounds", + "rethink", + "Voting", + "Scha", + "blood", + "Reeves", + "bells", + "collector", + "Crimson", + "Wheat", + "207", + "HB", + "BCC", + "sync", + "Anders", + "thanking", + "layoffs", + "foolish", + "custod", + "elephants", + "correlation", + "Harding", + "GPU", + "Barnett", + "ol", + "alarms", + "fluctuations", + "shop", + "commentators", + "Alpine", + "mur", + "biotech", + "unlocked", + "ouri", + "roe", + "Payment", + "POL", + "Guest", + "phrases", + "Built", + "erves", + "nutritional", + "205", + "ourage", + "Related", + "Come", + "SAT", + "gatherings", + "squads", + "organising", + "ministerial", + "kilomet", + "Jump", + "Strength", + "Ferr", + "illustrated", + "Ober", + "extrad", + "limitation", + "idis", + "Months", + "ifts", + "motives", + "maternal", + "bait", + "adversity", + "Twitter", + "Uni", + "grappling", + "bowls", + "Hib", + "Copenhagen", + "sergeant", + "intro", + "scrambled", + "Exc", + "showcases", + "plotting", + "sym", + "Nah", + "berries", + "itching", + "conn", + "istle", + "Beginning", + "asley", + "Meadow", + "Cra", + "supremacist", + "sweats", + "production", + "innon", + "ovo", + "scept", + "drowning", + "Eh", + "decorations", + "sympathetic", + "raction", + "195", + "ripp", + "Notice", + "charging", + "DIY", + "Jin", + "skinny", + "maj", + "whisk", + "congreg", + "RAL", + "volley", + "establishments", + "cite", + "Miss", + "Int", + "iola", + "Bare", + "KING", + "ools", + "private", + "flaw", + "wires", + "ideals", + "oub", + "\"'", + "Compet", + "Statements", + "HDR", + "rm", + "begging", + "uffs", + "dispatch", + "skipped", + "labs", + "hawks", + "expl", + "patriotic", + "ussions", + "portrayal", + "Budapest", + "Cod", + "extingu", + "smart", + "burdens", + "Drama", + "altitude", + "pursuant", + "à¥", + "atari", + "cot", + "hotline", + "ooters", + "Rolls", + "jeopardy", + "oids", + "pageant", + "149", + "distinguish", + "support", + "Highlands", + "Ernst", + "Hole", + "pering", + "Hasan", + "rece", + "irregular", + "disturbed", + "coupon", + "Elijah", + "oise", + "friendships", + "girlfriend", + "rampage", + "arers", + "dispens", + "assion", + "tentative", + "Exploration", + "fashioned", + "Instit", + "themed", + "Kurdistan", + "CAL", + "Sweeney", + "ransom", + "stamps", + "Schwe", + "Lucia", + "124", + "omore", + "motivate", + "Worcester", + "wald", + "CAR", + "iken", + "andro", + "ffic", + "Rehab", + "grou", + "controllers", + "Hai", + "nz", + "artillery", + "Mish", + "registry", + "frontman", + "Charg", + "orneys", + "PRESS", + "perceptions", + "McGee", + "AU", + "mg", + "Off", + "NGOs", + "chemical", + "brun", + "Hav", + "lace", + "202", + "defer", + "injected", + "gluten", + "Rin", + "Avalanche", + "corpor", + "Pamela", + "fills", + "Reve", + "Monument", + "nationalists", + "IQ", + "adden", + "Loop", + "134", + "Reg", + "click", + "bush", + "Kub", + "ipes", + "toggle", + "Rae", + "burgl", + "holistic", + "ronics", + "prominence", + "jack", + "finan", + "icates", + "vel", + "important", + "Thursday", + "chet", + "refunds", + "Elder", + "Owner", + "takeaway", + "Pe", + "Toro", + "Tim", + "fix", + "before", + "Motorola", + "lev", + "Term", + "Sne", + "misinformation", + "Sinai", + "nitrogen", + "203", + "escaping", + "junction", + "Santana", + "Yemeni", + "whipped", + "Stephenson", + "attire", + "Bard", + "atically", + "Faul", + "Sym", + "resh", + "MG", + "Sub", + "Carmen", + "ig", + "Sanford", + "Ya", + "cycle", + "encryption", + "Scal", + "Chest", + "Madonna", + "agin", + "DHS", + "Ced", + "YR", + "truce", + "Bike", + "foes", + "Slovakia", + "adal", + "Rain", + "OPE", + "lockdown", + "unilateral", + "overseen", + "blames", + "barrage", + "aan", + "uds", + "Rust", + "HC", + "cox", + "Allied", + "José", + "pected", + "unp", + "someday", + "deductions", + "icial", + "PRO", + "Intern", + "hemp", + "kilograms", + "nets", + "BACK", + "early", + "outed", + "relegated", + "1958", + "Mustang", + "gamble", + "prostitution", + "Papa", + "inexpensive", + "GHz", + "jerseys", + "misery", + "VIS", + "RAW", + "thri", + "affiliation", + "small", + "flashed", + "coastline", + "gard", + "sv", + "waits", + "itton", + "London", + "accus", + "Charge", + "incub", + "wanna", + "Awareness", + "abies", + "Uh", + "persuaded", + "Thames", + "curated", + "Ī", + "brutally", + "rooftop", + "oy", + "1900", + "bery", + "uphill", + "interacting", + "chilly", + "ERE", + "capsule", + "Saul", + "ocker", + "deserving", + "Bowen", + "Readers", + "Writers", + "artifacts", + "Ranger", + "reau", + "imperson", + "hears", + "Maher", + "neg", + "mantra", + "mull", + "elders", + "Amtrak", + "spouses", + "Hak", + "openness", + "prevailed", + "fortnight", + "Pal", + "ride", + "illustrate", + "dominated", + "trust", + "ī", + "Female", + "Slim", + "desc", + "Kathryn", + "deepen", + "TAIN", + "eredith", + "chanted", + "Hector", + "bread", + "Isa", + "volcanic", + "ah", + "owners", + "aquin", + "melting", + "preschool", + "ocus", + "Mast", + "Myr", + "suppress", + "versatility", + "NEC", + "hoax", + "mutually", + "Neb", + "Wheel", + "kit", + "abl", + "again", + "Sonny", + "rift", + "sweater", + "inund", + "Taco", + "Bout", + "nonprofits", + "modify", + "professionalism", + "Gould", + "Guerrero", + "terribly", + "Benz", + "countered", + "bean", + "Phelps", + "prowess", + "bc", + "feast", + "5000", + "revisit", + "chin", + "agent", + "tones", + "extraction", + "Posts", + "oin", + "attain", + "gardening", + "earned", + "Otto", + "player", + "scams", + "Honolulu", + "Appro", + "HIGH", + "dwell", + "Islam", + "leaders", + "legisl", + "expl", + "Choi", + "frenzy", + "commercially", + "lbs", + "gateway", + "Andersen", + "emia", + "lez", + "residences", + "office", + "Helsinki", + "olia", + "wolf", + "styling", + "Junction", + "Peyton", + "udo", + "Dorothy", + "freshly", + "Julio", + "Sunset", + "Madden", + "issu", + "sounding", + "sports", + "massively", + "Rahman", + "presided", + "Instead", + "136", + "Howell", + "beit", + "prosperous", + "wrongly", + "Raqqa", + "Ces", + "buddy", + "chatting", + "fencing", + "tant", + "ocated", + "ALK", + "snapping", + "euro", + "Ryan", + "Recogn", + "ucked", + "purported", + "Cann", + "intimidating", + "rulers", + "Marse", + "Art", + "Aadhaar", + "vows", + "hunter", + "ourmet", + "Various", + "2009", + "anie", + "compassionate", + "Parking", + "malaria", + "amnesty", + "worsened", + "Titan", + "crossings", + "drug", + "addicted", + "remorse", + "Destiny", + "Dear", + "hur", + "implicated", + "playful", + "ripe", + "sizable", + "crab", + "liqu", + "drib", + "contraction", + "cro", + "Gus", + "doomed", + "mog", + "Monitor", + "Count", + "sadd", + "wrestler", + "restraints", + "raging", + "185", + "tapes", + "mitigation", + "ocratic", + "vib", + "Snowden", + "aldo", + "weights", + "1959", + "ucc", + "Coc", + "Log", + "Stev", + "dealership", + "trademarks", + "iru", + "beneficiary", + "legislator", + "deadlines", + "cosmetics", + "Tammy", + "Combined", + "educator", + "athon", + "combo", + "fu", + "appropriate", + "nington", + "Liberties", + "missions", + "opard", + "Mondays", + "fetch", + "hers", + "jon", + "ukes", + "zek", + "vetting", + "yet", + "facilitating", + "Stras", + "character", + "Heads", + "clim", + "Albuquerque", + "bind", + "concluding", + "Basically", + "rail", + "TCU", + "Depression", + "hem", + "Hue", + "pand", + "scoreboard", + "Av", + "idol", + "compl", + "redesign", + "Jarrett", + "favoured", + "INS", + "propelled", + "evasion", + "widened", + "wastewater", + "nard", + "responsive", + "demographics", + "engine", + "Brewer", + "Baxter", + "ront", + "Colon", + "promoter", + "genres", + "ovsky", + "build", + "urate", + "Cohn", + "design", + "turbulent", + "curtain", + "310", + "Lamp", + "Bonds", + "church", + "deterrent", + "dictatorship", + "acement", + "haul", + "spir", + "conceived", + "stern", + "sit", + "singular", + "Yog", + "conditional", + "ide", + "lund", + "autop", + "BEST", + "Jed", + "rationale", + "alarmed", + "shovel", + "Prob", + "Mao", + "Burgess", + "1953", + "above", + "Manson", + "dismal", + "Frankie", + "tempted", + "underdog", + "ribing", + "ENCY", + "Dele", + "Las", + "places", + "notoriously", + "Akin", + "glut", + "seamlessly", + "recess", + "written", + "TJ", + "occ", + "Territory", + "AIR", + "Diagn", + "vacancies", + "cultivation", + "Aless", + "renamed", + "Mahmoud", + "bright", + "visibly", + "nas", + "erred", + "Carn", + "triggers", + "punishing", + "luc", + "Bett", + "beam", + "Cheng", + "aina", + "determines", + "Gerry", + "shocks", + "stainless", + "defects", + "Cinem", + "torrent", + "resurgence", + "coral", + "blitz", + "Gel", + "stemmed", + "gur", + "lymph", + "zzo", + "spearheaded", + "licences", + "';", + "arbitrary", + "Uzbek", + "thief", + "reaching", + "cand", + "EA", + "Paraly", + "Emerson", + "Sergey", + "Scher", + "Wr", + "rowing", + "3000", + "mighty", + "elight", + "mAh", + "celebr", + "Conclusion", + "Cathy", + "polished", + "uddled", + "ewski", + "fucking", + "interfering", + "landscapes", + "fearful", + "Detention", + "%).", + "TT", + "bleak", + "indebted", + "cheat", + "consolation", + "Pace", + "raine", + "honorary", + "420", + "technician", + "Comprehensive", + "fences", + "wearable", + "Marilyn", + "stru", + "drained", + "Gibraltar", + "lag", + "disorderly", + "proclaimed", + "capacities", + "retains", + "Vid", + "oshi", + "Eid", + "analytical", + "ominium", + "Examiner", + "NAACP", + "ocol", + "rev", + "Rim", + "Woody", + "McKenna", + "Lennon", + "Employ", + "Fort", + "psy", + "sphere", + "oday", + "Chick", + "Compared", + "Iranians", + "Accountability", + "itchie", + "Dickinson", + "flock", + "eclips", + "nat", + "anke", + "Neighborhood", + "141", + "scarce", + "creations", + "lists", + "useless", + "criticisms", + "ruler", + "Hick", + "arya", + "worker", + "alam", + "Angelo", + "otle", + "newsletters", + "erected", + "zip", + "Birthday", + "dogged", + "danced", + "confession", + "vomiting", + "ickers", + "fox", + "deduct", + "stresses", + "poll", + "Radar", + "engagements", + "examiner", + "opportun", + "longevity", + "banana", + "carbon", + "uo", + "LT", + "synagogue", + "blackmail", + "INK", + "fle", + "Gutierrez", + "racket", + "evenings", + "dietary", + "Kok", + "faulty", + "abandoning", + "Flow", + "quest", + "estead", + "bir", + "suicidal", + "Gift", + "Missing", + "Mazda", + "Rib", + "Journey", + "concede", + "brushed", + "Tw", + "andowski", + "Yun", + "Bride", + "zai", + "awatts", + "cha", + "spans", + "SF", + "shells", + "planned", + "Geographic", + "Vent", + "fav", + "interrogation", + "varies", + "Plat", + "operative", + "avid", + "greatness", + "Strait", + "Selling", + "lawful", + "lyn", + "funnel", + "pundits", + "ties", + "pneumonia", + "commencement", + "brisk", + "fires", + "HTML", + "Sevent", + "histor", + "147", + "olls", + "pian", + "Little", + "commercials", + "deteriorated", + "basin", + "prohibition", + "restrictive", + "tom", + "Pulse", + "vale", + "mim", + "Lyons", + "Trinidad", + "data", + "195", + "Pain", + "vor", + "Directorate", + "Wow", + "essential", + "emerges", + "Doors", + "unde", + "archives", + "IX", + "Aman", + "oric", + "Oper", + "nothing", + "142", + "igr", + "rust", + "BYU", + "Bom", + "rift", + "Abs", + "Jenn", + "rookies", + "hoe", + "underage", + "eden", + "roasted", + "enrol", + "erased", + "freeway", + "Sil", + "planner", + "confess", + "Dual", + "Headquarters", + "bottom", + "statistic", + "Push", + "anim", + "ITT", + "executions", + "Hub", + "Stick", + "obscure", + "oven", + "coats", + "unc", + "Morning", + "nit", + "mie", + "curves", + "gew", + "Anniversary", + "members", + "Absolutely", + "apt", + "otional", + "Gin", + "izo", + "pretending", + "arak", + "organise", + "royalties", + "Camden", + "sausage", + "Inst", + "chalk", + "Surf", + "Sunrise", + "moder", + "aido", + "loving", + "lus", + "oblig", + "motions", + "clarification", + "OM", + "bishop", + "exhibitions", + "Rifle", + "Phot", + "HM", + "ATIONAL", + "wid", + "reside", + "PV", + "OOK", + "Tue", + "1200", + "1957", + "espionage", + "APPLIC", + "blasts", + "fter", + "immensely", + "Lots", + "inflammatory", + "anging", + "tumultuous", + "identified", + "stead", + "Ach", + "Ãī", + "bub", + "hler", + "olution", + "shun", + "null", + "unused", + "Obs", + "insol", + "Attack", + "ertain", + "defiant", + "Through", + "Armour", + "simulation", + "UCK", + "influenza", + "onset", + "bored", + "souls", + "referees", + "collaborations", + "Ler", + "creepy", + "analy", + "Effect", + "orting", + "Card", + "dice", + "harvesting", + "235", + "sty", + "McCartney", + "salute", + "UMP", + "herb", + "Abuse", + "Ramadan", + "suck", + "trained", + "Physical", + "iren", + "anches", + "erie", + "hangs", + "cataly", + "intuitive", + "assi", + "techn", + "jugg", + "gameplay", + "apolog", + "fifteen", + "galleries", + "outlines", + "patient", + "Potential", + "ethnicity", + "harbour", + "overthrow", + "Lung", + "warehouses", + "Monitoring", + "mentors", + "sized", + "envisioned", + "gin", + "DT", + "propel", + "Kul", + "ference", + "estic", + "Lego", + "dinners", + "Moe", + "designed", + "Susp", + "Brick", + "qua", + "IDS", + "Bam", + "athe", + "slices", + "bottled", + "thy", + "producing", + "Terror", + "professional", + "Kis", + "erto", + "Vehicles", + "beforehand", + "detrimental", + "weights", + "allowances", + "Williams", + "Syrians", + "Sto", + "cozy", + "reditation", + "ensen", + "Sard", + "roy", + "ooting", + "Reserv", + "ominated", + "emate", + "Tot", + "Carnegie", + "Thib", + "Marshal", + "152", + "mayors", + "inery", + "Fiona", + "Cadillac", + "ivated", + "eagerly", + "Offensive", + "astronaut", + "Vital", + "cane", + "quitting", + "Lone", + "censorship", + "Welch", + "Ud", + "marquee", + "Dip", + "whereby", + "tiger", + "gem", + "conserv", + "presumed", + "Entry", + "ffer", + "Proceed", + "brawl", + "Jaime", + "echo", + "advancements", + "transitional", + "erick", + "bully", + "anan", + "reinvent", + "Letters", + "bricks", + "Smy", + "towering", + "gging", + "299", + "orian", + "dimensional", + "Forty", + "Sinn", + "ushi", + "Surveillance", + "enabled", + "Mous", + "Vive", + "Marcus", + "vom", + "creek", + "lime", + "seismic", + "Fork", + "embroiled", + "marks", + "herald", + "Sonia", + "âĢ¦\"", + "wired", + "obliged", + "Projects", + "lde", + "Riders", + "overcoming", + "Mail", + "Lawn", + "Hawk", + "figure", + "Written", + "ens", + "spacious", + "target", + "Recep", + "SAM", + "entertained", + "ignited", + "CENT", + "ogenic", + "unatt", + "exceeds", + "--------------------------------", + "pillars", + "Borders", + "ickey", + "extinction", + "viability", + "tumors", + "Wilkinson", + "KEY", + "bins", + "Reported", + "Sm", + "Exclusive", + "Chilean", + "info", + "wilderness", + "did", + "absolutely", + "pillar", + "elites", + "Preview", + "ixie", + "Mont", + "ribut", + "dream", + "planners", + "Somerset", + "envis", + "Stall", + "elevate", + "ographies", + "rama", + "Ha", + "amidst", + "oho", + "rejects", + "Jim", + "marginally", + "usher", + "arez", + "Hawth", + "sprink", + "Offer", + "anchored", + "ucking", + "Garn", + "Conserv", + "societal", + "browsing", + "bidder", + "burgh", + "Runner", + "trendy", + "verts", + "imposed", + "Patton", + "lements", + "spicy", + "swe", + "Strike", + "clam", + "Yankee", + "KT", + "Greenwood", + "Ways", + "2050", + "attach", + "Shim", + "meltdown", + "assemble", + "UPDATE", + "scout", + "Brown", + "Kobe", + "postpone", + "liness", + "allo", + "rief", + "Germ", + "FD", + "Reggie", + "Univers", + "Shepard", + "cancell", + "Romeo", + "Warrior", + "ench", + "ifier", + "privileges", + "senses", + "impoverished", + "Postal", + "encer", + "Conrad", + "printer", + "inflicted", + "Gamble", + "Heroes", + "132", + "revisions", + "unsuccessfully", + "Heisman", + "stamped", + "inding", + "Luna", + "reinvest", + "ducers", + "Password", + "Leod", + "compounded", + "',\"", + "ogging", + "probing", + "PBS", + "MU", + "Whenever", + "sped", + "Competitive", + "isans", + "opa", + "cleric", + "vivid", + "à¸", + "126", + "inconvenience", + "udi", + "immersive", + "diversion", + "logs", + "spying", + "inct", + "litres", + "metallic", + "identally", + "FX", + "loudly", + "nursery", + "collectors", + "Kart", + "escalate", + "ringing", + "procedural", + "disrupting", + "Ethiopian", + "CFL", + "illustrates", + "perks", + "official", + "325", + "millennial", + "breadth", + "melted", + "850", + "Bake", + "donald", + "Grac", + "seeded", + "Discount", + "idates", + "drift", + "captive", + "seriousness", + "repercussions", + "disciplines", + "thesis", + "sleeve", + "ses", + "Monday", + "thwart", + "Lic", + "quadru", + "Presbyterian", + "reactors", + "Suzanne", + "ewater", + "lam", + "breastfeeding", + "rats", + "Artists", + "domestically", + "decom", + "Arms", + "basketball", + "scrub", + "Teddy", + "beh", + "Betsy", + "Nursing", + "descriptions", + "127", + "gil", + "itional", + "championed", + "Calling", + "realization", + "Buddy", + "hou", + "Dire", + "Huff", + "lipstick", + "Ray", + "flare", + "belt", + "brightest", + "malfunction", + "Manor", + "saturated", + "rays", + "DW", + "ixed", + "Slovenia", + "seen", + "Cause", + "arios", + "ASE", + "rend", + "TBA", + "lecturer", + "attering", + "affluent", + "CEO", + "breathtaking", + "Giles", + "irth", + "Philips", + "posture", + "TSA", + "heit", + "menace", + "ricks", + "Aden", + "Reich", + "iggle", + "Shutterstock", + "courageous", + "edia", + "Staff", + "divert", + "Cir", + "guessing", + "apers", + "Britons", + "lé", + "convened", + "Serbian", + "richer", + "cock", + "deposited", + "company", + "delic", + "sensitive", + "tank", + "Patty", + "mia", + "onomous", + "cn", + "clamp", + "Academic", + "prosecuting", + "Transparency", + "deflation", + "dashboard", + "Dress", + "lin", + "mu", + "Goodell", + "lav", + "Twelve", + "flavour", + "fiercely", + "bloom", + "Haf", + "Grad", + "LET", + "Seeing", + "oxide", + "menus", + "char", + "adoes", + "combe", + "Street", + "Ridley", + "depicts", + "Pred", + "ÑĢ", + "British", + "bumps", + "lamp", + "Desmond", + "PB", + "frag", + "tin", + "Sharing", + "desperation", + "commuter", + "igrants", + "Shapiro", + "kinda", + "impartial", + "Jewel", + "congratulations", + "compost", + "admiration", + "paycheck", + "Anonymous", + "enger", + "Mer", + "Gospel", + "Eth", + "MH", + "fem", + "Trial", + "depths", + "Applied", + "grit", + "erase", + "sid", + "comm", + "}", + "retreated", + "analysed", + "Regular", + "Pesh", + "ICAL", + "pei", + "Reilly", + "Trib", + "booths", + "drank", + "coma", + "harvested", + "CHAR", + "butterfly", + "sailed", + "Drink", + "eping", + "ATCH", + "Legends", + "insured", + "wholes", + "Bis", + "Shea", + "ighter", + "snakes", + "Gunn", + "Poss", + "dispar", + "bombshell", + "scanning", + "340", + "choice", + "cool", + "\"âĢĶ", + "Theo", + "rine", + "Jacques", + "disadvantaged", + "paramount", + "igate", + "stat", + "anski", + "outsourcing", + "populous", + "binge", + "Organic", + "urban", + "yogurt", + "retweet", + "osen", + "cially", + "215", + "editions", + "burgeoning", + "efully", + "Thousand", + "replacements", + "Amazing", + "rator", + "icy", + "intensify", + "Sen", + "Quincy", + "powers", + "Aur", + "Zion", + "stal", + "pillar", + "Erit", + "Perform", + "aston", + "Eric", + "unh", + "IFF", + "950", + "Engineer", + "Lands", + "dubious", + "fy", + "WI", + "Sv", + "Hendricks", + "Kod", + "outlining", + "Correspond", + "amus", + "worst", + "arter", + "coni", + "hierarchy", + "THAT", + "exce", + "railways", + "masked", + "lene", + "outset", + "avalanche", + "nicknamed", + "702", + "Lee", + "139", + "Sixth", + "365", + "nda", + "accountant", + "obese", + "grape", + "impunity", + "Yorkers", + "guardian", + "icity", + "centrist", + "waterways", + "ursed", + "hopeless", + "header", + "tack", + "ric", + "umn", + "valve", + "tread", + "CST", + "hepatitis", + "ctor", + "RED", + "solitary", + "NW", + "ceremonial", + "foe", + "ling", + "Jason", + "Lisbon", + "1955", + "Heller", + "kin", + "essen", + "turbines", + "shi", + "lodge", + "veterinary", + "Boll", + "Confederation", + "Journalists", + "tug", + "Starr", + "piles", + "Way", + "adel", + "orean", + "oft", + "shortcomings", + "Sheila", + "backbone", + "III", + "Darwin", + "Tunis", + "suspicions", + "disagreements", + "247", + "illery", + "'\"", + "segregation", + "ohl", + "instincts", + "Poo", + "nih", + "parency", + "uddy", + "esting", + "asses", + "Introduction", + "Sirius", + "Local", + "orous", + "rehearsal", + "demol", + "traffickers", + "upsetting", + "heir", + "death", + "Moments", + "Los", + "atmospheric", + "aints", + "Dianne", + "likewise", + "Ming", + "auga", + "firsthand", + "narratives", + "Astron", + "Extreme", + "horns", + "Sana", + "recapt", + "Mist", + "Randolph", + "connect", + "indecent", + "forty", + "jihadists", + "azes", + "dread", + "grapes", + "removes", + "screamed", + "Crus", + "ikers", + "snapshot", + "Calls", + "Cons", + "lettuce", + "Pig", + "urable", + "jured", + "ILY", + "Jessie", + ".).", + "Pay", + "Tra", + "----------------", + "Units", + "Playboy", + "arthritis", + "afforded", + "insk", + "Fake", + "Lies", + "Baltic", + "oyal", + "Vest", + "rusher", + "incorporates", + "MM", + "Dru", + "Ware", + "Sammy", + "Gob", + "Ruk", + "146", + "Crowd", + "duel", + "irts", + "sourcing", + "hp", + "Java", + "bred", + "Refer", + "uninsured", + "slope", + "256", + "regulating", + "fundra", + "inserted", + "Nickel", + "Consumption", + "Romo", + "Atlantic", + "enclave", + "pegged", + "directs", + "mbudsman", + "DES", + "Ob", + "limbs", + "bury", + "ILA", + "stew", + "breeze", + "abrupt", + "Gott", + "Claude", + "genetically", + "rigid", + "Dudley", + "Ner", + "registered", + "entrenched", + "extortion", + "Nurs", + "contingency", + "etter", + "rejo", + "protagonist", + "counselling", + "Vit", + "aware", + "Monsanto", + "GG", + "incarcerated", + "abduction", + "referencing", + "Germany", + "uates", + "reck", + "tram", + "chron", + "mish", + "Ves", + "Tire", + "vandal", + "Crazy", + "Lifetime", + "Spectrum", + "celer", + "motto", + "hang", + "blade", + "gel", + "biography", + "allegiance", + "hod", + "hap", + "ptic", + "acle", + "Blade", + "Boh", + "149", + "chang", + "canned", + "facilitated", + "actor", + "iologist", + "rebuilt", + "awake", + "mayoral", + "Euros", + "dangerously", + "MK", + "replica", + "coinc", + "blog", + "Era", + "relinqu", + "quite", + "ondon", + "rosso", + "tun", + "touchscreen", + "pops", + "ousing", + "efficient", + "148", + "conced", + "although", + "1956", + "mortar", + "Cave", + "Jung", + "urer", + "illusion", + "Berman", + "intend", + "coping", + "Dem", + "tion", + "estation", + "Sounds", + "navigating", + "sperm", + "religions", + "fol", + "heroic", + "FD", + "hesitant", + "asure", + "redeem", + "Adam", + "fireplace", + "vertis", + "Sung", + "290", + "iland", + "Updates", + "OTUS", + "PTSD", + "helmets", + "\"?", + "slashing", + "scouts", + "spelling", + "Initial", + "draw", + "challengers", + "supremacists", + "pilgrims", + "asc", + "Fill", + "Pau", + "jewel", + "Malt", + "icip", + "inhabitants", + "metre", + "ahar", + "Comp", + "atches", + "inv", + "cyclist", + "QC", + "manually", + "Anchorage", + "discarded", + "consolid", + "navig", + "Animals", + "Pole", + "esson", + "1954", + "sorted", + "madness", + "Brigade", + "Genesis", + "dismissing", + "Panasonic", + "dizz", + "Educational", + "KO", + "Pill", + "GIF", + "bol", + "wards", + "controversies", + "Chinese", + "antics", + "reliant", + "Moff", + "ethanol", + "torch", + "rights", + "Habit", + "arton", + "rera", + "Sasha", + "abella", + "proliferation", + "sincerely", + "communication", + "Nay", + "Chattanooga", + "ounces", + "NXT", + "Emir", + "manipulated", + "harassing", + "wat", + "bouts", + "Book", + "hovering", + "Scan", + "ship", + "Angola", + "LC", + "ruins", + "sexist", + "zar", + "pledging", + "ober", + "embold", + "objection", + "boasting", + "MIN", + "herbs", + "gears", + "Ic", + "stre", + "him", + "homicides", + "cki", + "castle", + "counter", + "CAS", + "Reasons", + "Declaration", + "simplify", + "fared", + "escort", + "kidn", + "Hamm", + "nailed", + "accommodations", + "modifications", + "rible", + "wool", + "EDIT", + "2010", + "authentication", + "goat", + "hom", + "federally", + "Rath", + "spiked", + "misrepresent", + "avenue", + "broadcasts", + "Estonia", + "ennes", + "Mare", + "ption", + "Kag", + "circumstance", + "orrow", + "isons", + "Collabor", + "stroll", + "CPS", + "soft", + "iral", + "apo", + "usky", + "poke", + "woo", + "Elena", + "Lastly", + "linemen", + "Canadian", + "Anyway", + "substantive", + "Curt", + "ard", + "Yosh", + "Buchanan", + "revolving", + "specials", + "shrine", + "lumber", + "orchestrated", + "kie", + "azy", + "expiration", + "Daryl", + "Patri", + "better", + "2020", + "Fav", + "OP", + "OTT", + "flush", + "Sikh", + "ecosystems", + "BET", + "eared", + "audio", + "Fahrenheit", + "police", + "incarceration", + "erupt", + "Damien", + "Hague", + "ulz", + "Agents", + "Banner", + "conductor", + "Ajax", + "arson", + "rests", + "eurozone", + "felon", + "curator", + "morning", + "evidenced", + "Neh", + "mattress", + "tast", + "fueling", + "Occup", + "bake", + "Zac", + "meaning", + "Ill", + "Hau", + "Laden", + "bald", + "Mary", + "oky", + "atri", + "tracker", + "OTA", + "catching", + "Underground", + "HuffPost", + "Atkins", + "oglu", + "authorised", + "routines", + "Hof", + "veland", + "langu", + "prot", + "Hyd", + "integ", + "bravery", + "violin", + "delightful", + "ticks", + "iton", + "reap", + "oversized", + "Pitch", + "prized", + "fusion", + "fact", + "acting", + "fullback", + "polite", + "swear", + "confiscated", + "Stud", + "fielded", + "rito", + "covered", + "financial", + "bill", + "HK", + "OTOS", + "loaded", + "marble", + "Diplom", + ".âĢĶ", + "eats", + "backfield", + "timeframe", + "vegetarian", + "swaps", + "Mines", + "igor", + "Lenn", + "DP", + "ordered", + "Shark", + "quant", + "erence", + "ashes", + "Buckley", + "ophobia", + "warranted", + "Rose", + "unreasonable", + "Jav", + "palette", + "joints", + "advent", + "noteworthy", + "Nicol", + "Christensen", + "plummeted", + "ayers", + "defends", + "contended", + "Congratulations", + "kish", + "Hannity", + "groundwater", + "Kramer", + "erect", + "appet", + "Kardash", + "exacerbated", + "explanations", + "vious", + "eport", + "---", + "icism", + "Natasha", + "Geoffrey", + "estro", + "Article", + "incidence", + "provoked", + "elf", + "insistence", + "OUR", + "fertilizer", + "stickers", + "Gators", + "Landing", + "DON", + "sta", + "Robbins", + "pixels", + "Hoy", + "imated", + "Ãī", + "â", + "simpl", + "Other", + "245", + "forcibly", + "'.\"", + "smashing", + "mosquitoes", + "paints", + "debating", + "enty", + "IB", + "leaf", + "Dah", + "referral", + "pired", + "brunch", + "gie", + "vict", + "ribute", + "bloggers", + "gum", + "Admiral", + "France", + "PK", + "Saturn", + "inflated", + "WAR", + "scenic", + "usal", + "their", + "contends", + "pathways", + "inis", + "awarding", + "misled", + "eternal", + "examinations", + "poker", + "safest", + "childcare", + "aday", + "preceding", + "Collective", + "respectable", + "ographical", + "oak", + "00000", + "Corridor", + "oran", + "133", + "mushrooms", + "gaard", + "Omega", + "Naturally", + "anim", + "captains", + "tang", + "lobbyists", + "Sug", + "succ", + "249", + "ENG", + "134", + "solic", + "Added", + "Suicide", + "FULL", + "Strauss", + "Diesel", + "tempting", + "acist", + "Delivery", + "quiz", + "PARK", + "collisions", + "restrained", + "purpose", + "Changes", + "absentee", + "probes", + "hib", + "cul", + "petty", + "necess", + "cues", + "OME", + "inadvertently", + "urity", + "Stuff", + "FG", + "wrestlers", + "paste", + "Roku", + "cardboard", + "aires", + "variables", + "Saras", + "Fif", + "invests", + "Discover", + "Fix", + "Thomas", + "Lunch", + "lv", + "camera", + "Step", + "resumes", + "Sacred", + "Shooting", + "noble", + "slopes", + "ont", + "twists", + "Very", + "bigotry", + "Tib", + "mos", + "warrior", + "broadcasters", + "ubiquitous", + "ameda", + "chess", + "Special", + "conver", + "deleg", + "endant", + "foil", + "lush", + "taxed", + "Mag", + "ahs", + "tablespoons", + "scription", + "clamation", + "Certain", + "Diversity", + "hairst", + "Brewery", + "shedding", + "Cla", + "penis", + "Murder", + "Park", + "uner", + "iments", + "OVER", + "hus", + "tabloid", + "Chart", + "vouchers", + "Coord", + "methane", + "Fisheries", + "Kham", + "includes", + "Superman", + "ensed", + "isure", + "Amazon", + "vacated", + "heet", + "roast", + "legalize", + "Tut", + "signage", + "init", + "thefts", + "202", + "static", + "chants", + "Bob", + "discretionary", + "endurance", + "collegiate", + "corridors", + "slack", + "Lash", + "Az", + "Series", + "nonpartisan", + "McGill", + "uneven", + "ulsive", + "eu", + "pil", + "fisheries", + "onslaught", + "fiction", + "holding", + "cheated", + "traumat", + "lasting", + "multitude", + "Thr", + "Breast", + "1600", + "Matth", + "diminish", + "FTC", + "gram", + "Resident", + "fading", + "marginalized", + "Lite", + "Carlton", + "erad", + "Welcome", + "Faw", + "iddy", + "particip", + "cz", + "texted", + "suites", + "Forever", + "rendition", + "rait", + "Prague", + "sponsoring", + "compos", + "Beacon", + "144", + "pupil", + "intricate", + "athleticism", + "optimization", + "loot", + "polit", + "Ott", + "Whatever", + "uno", + "Constable", + "esville", + "lookout", + "Aircraft", + "spo", + "corrobor", + "hiatus", + "Knowing", + "Hamp", + "spe", + "storing", + "shakes", + "uran", + "sickness", + "liber", + "Administrative", + "pleasing", + "Equal", + "Conversation", + "algae", + "lobbyist", + "Helena", + "ptions", + "faire", + "Gone", + "Wiggins", + "Robert", + "listens", + "Daisy", + "sticky", + "sale", + "Marijuana", + "SSD", + "Tool", + "once", + "Harmon", + "mobile", + "detain", + "Money", + "flawless", + "forced", + "guru", + "airspace", + "Archie", + "Gender", + "Meat", + "abilities", + "BD", + "Open", + "outsider", + "issue", + "learns", + "natural", + "vinegar", + "SUB", + "Recon", + "blers", + "sniff", + "suppression", + "saf", + "urger", + "bunker", + "asaki", + "Spartan", + "Tok", + "rav", + "foc", + "Sean", + "etric", + "ballpark", + "Herb", + "BM", + "Publishing", + "roadmap", + "pered", + "predator", + "Blockchain", + "validity", + "Glou", + "Yamaha", + "adop", + "swamp", + "complied", + "Ky", + "Greg", + "casts", + "john", + "Bosnia", + "cinematic", + "Tavern", + "frustrations", + "eryl", + "fairy", + "UNCH", + "Tus", + "Corp", + "Nug", + "closed", + "exercised", + "urden", + "digitally", + "137", + "Victims", + "reluctance", + "ELL", + "Tribe", + "chall", + "whiskey", + "ogl", + "mater", + "Bac", + "apartheid", + "MBA", + "mot", + "Ire", + "®,", + "Chic", + "timed", + "Dome", + "efer", + "observer", + "unky", + "Kant", + "undrafted", + "simplicity", + "onds", + "stoked", + "1949", + "ransomware", + "Pow", + "Angelo", + "Ambrose", + "adjusted", + "Guard", + "138", + "Kaplan", + "stri", + "cries", + "NF", + "atro", + "avocado", + "illian", + "sculptures", + "elevation", + "inspires", + "generals", + "arb", + "chell", + "Journalism", + "Hybrid", + "Caller", + "vec", + "Lu", + "resemble", + "bys", + "erving", + "antz", + "widen", + "vised", + "Ev", + "diagn", + "Makes", + "cer", + "Pats", + "single", + "sche", + "struct", + "dissolved", + "timeout", + "enhancement", + "CF", + "indust", + "Ded", + "Zo", + "CB", + "pesticides", + "Rubin", + "George", + "opal", + "motel", + "critical", + "collapsing", + "Shal", + "tex", + "complementary", + "oust", + "Flu", + "exporting", + "differential", + "north", + "FG", + "spoon", + "sha", + "dismantle", + "elta", + "jar", + "space", + "Smart", + "mere", + "Ð", + "Gillespie", + "Lo", + "Mead", + "capacity", + "Issue", + "050", + "Vall", + "disgr", + "meme", + "pard", + "compensated", + "Ket", + "major", + "Bren", + "heed", + "131", + "cm", + "dazzling", + "Cheese", + "monumental", + "yielding", + "Read", + "grinding", + "Ang", + "defiance", + "intimidated", + "310", + "outsiders", + "houn", + "Ma", + "ĸ", + "Forget", + "Sans", + "unfolding", + "Sap", + "Lak", + "sectarian", + "Daddy", + "oxy", + "hitting", + "detectors", + "Ree", + "broaden", + "slaying", + "suspending", + "investig", + "Tuesday", + "antibiotic", + "Shiite", + "igi", + "External", + "Photographer", + "erratic", + "NJ", + "Dock", + "outweigh", + "rants", + "lobster", + "reactor", + "unrealistic", + "Audrey", + "Yor", + "Anyone", + "fraught", + "е", + "Wester", + "fc", + "Dunham", + "Lug", + "allow", + "139", + "parity", + "horizontal", + "ijuana", + "civilization", + "Gins", + "smokers", + "Diabetes", + "Five", + "DG", + "underscores", + "elabor", + "Lub", + "Devil", + "154", + "Guarant", + "Pandora", + "excav", + "accuser", + "revolt", + "instructors", + "ire", + "ographic", + "CLE", + "expedition", + "ould", + "striving", + "south", + "onis", + "Swed", + "MY", + "Levin", + "carp", + "Architects", + "{", + "covert", + "cooled", + "Staten", + "specializing", + "Hazel", + "len", + "ighty", + "brilliantly", + "Phil", + "lament", + "Australia", + "203", + "ticking", + "adjud", + "roommate", + "Sheet", + "capital", + "167", + "endeavor", + "aver", + "dues", + "Cycl", + "oried", + "Va", + "loading", + "premie", + "regimes", + "Aly", + "perennial", + "consoles", + "ironic", + "ichael", + "vigorously", + "transmit", + "gary", + "eking", + "jails", + "Episcopal", + "eddy", + "idle", + "safeguards", + "dwindling", + "NOR", + "torn", + "Evangel", + "Plastic", + "Term", + "forwarded", + "avage", + "refrigerator", + "arna", + "Guinness", + "Candy", + "botched", + "seller", + "pul", + "grades", + "oshenko", + "earth", + "nette", + "traps", + "tarn", + "militar", + "Ariel", + "tubes", + "ulo", + "Water", + "edin", + "marvel", + "chenko", + "Elk", + "spect", + "coe", + "Illustrated", + "ruthless", + "etermined", + "dys", + "breaching", + "gee", + "Nick", + "cruiser", + "civ", + "dou", + ";", + "deb", + "Asheville", + "biting", + "yo", + "Courtesy", + "roses", + "Consequently", + "revis", + "confinement", + "next", + "produced", + "moratorium", + "kne", + "eties", + "plethora", + "celeb", + "FIN", + "departures", + "Wynne", + "abilia", + "Courts", + "olis", + "cereal", + "blended", + "333", + "Lun", + "repe", + "mathematics", + "pharmacies", + "Center", + "whist", + "pine", + "perm", + "customary", + "hormones", + "cleansing", + "confidentiality", + "mascot", + "slippery", + "mediation", + "podcasts", + "coating", + "conveyed", + "gir", + "Nurse", + "DM", + "lured", + "orted", + "olig", + "ritz", + "INF", + "tirelessly", + "doorstep", + "tomb", + "withholding", + "irling", + "hog", + "156", + "gau", + "chem", + "raid", + "trolls", + "182", + "Columb", + "tissues", + "naive", + "lect", + "Central", + "Sign", + "168", + "bribe", + "Doll", + "Tripoli", + "funk", + "plaza", + "mechanic", + "mem", + "monkey", + "grid", + "tainted", + "Nicaragua", + "pelling", + "Xia", + "ammers", + "orth", + "ICAN", + "rant", + "diary", + "Harrington", + "imply", + "Qaeda", + "worsen", + "crafting", + "Shir", + "coincided", + "snatched", + "ileen", + "sei", + "surgeons", + "directed", + "compulsory", + "nowadays", + "LI", + "Rebel", + "lions", + "JR", + "scar", + "Respons", + "scroll", + "Erd", + "iety", + "\";", + "Bone", + "Rumble", + "KS", + "Laur", + "kell", + "Birds", + "agic", + "simmer", + "runaway", + "162", + "auna", + "dialog", + "louder", + "esque", + "RR", + "bloss", + "caliber", + "nery", + "hauled", + "bacterial", + "Vanity", + "Programs", + "omew", + "Mama", + "arr", + "dod", + "Jarvis", + "FIRST", + "injections", + "Ballard", + "medically", + "angan", + "Newfoundland", + "fracking", + "bast", + "outing", + "mercury", + "watershed", + "Amateur", + "153", + "escal", + "painter", + "creat", + "perceive", + "gent", + "attacks", + "worked", + "importing", + "Indian", + "convict", + "clad", + "budding", + "ambient", + "Witness", + "letes", + "buffet", + "needles", + "coding", + "choke", + "correspondence", + "gods", + "dances", + "steadfast", + "cert", + "roaming", + "between", + "weak", + "Jer", + "jandro", + "discouraged", + "fruition", + "Ø", + "Kop", + "ULL", + "efe", + "imble", + "obb", + "ulla", + "accredited", + "lectures", + "bil", + "why", + "greeting", + "Boost", + "mailed", + "troop", + "frig", + "rese", + "scratched", + "Stars", + "Railroad", + "Idol", + "succumbed", + "Weeks", + "ffe", + "jihadist", + "ITION", + "threads", + "Generally", + "medieval", + "quotas", + "Ferry", + "rique", + "prod", + "Educ", + "rive", + "ensued", + "Cy", + "infring", + "prank", + "frontline", + "completes", + "upe", + "manageable", + "poems", + "otten", + "igne", + "threat", + "Dri", + "LINK", + "Calif", + "Dos", + "ulent", + "aids", + "slips", + "umped", + "styled", + "disproportionately", + "Dish", + "Uncle", + "andel", + "recharge", + "rators", + "SPR", + "guarded", + "Greatest", + "Skills", + "Nob", + "Desk", + "Cros", + "writ", + "query", + "ORTS", + "bundled", + "gib", + "eth", + "iesta", + "evade", + "dict", + "straight", + "Met", + "present", + "diff", + "dere", + "Spl", + "repr", + "Beard", + "vain", + "appointing", + "Visual", + "caps", + "gado", + "Rican", + "Pose", + "endor", + "222", + "Lear", + "constructing", + "Dan", + "Spears", + "Therapy", + "pta", + "rehabilit", + "risked", + "Guer", + "HF", + "301", + "liking", + "modular", + "eree", + "MAT", + "Homeless", + "stove", + "erd", + "hash", + "Achilles", + "Beta", + "incl", + "gunned", + "Crab", + "Mara", + "invaded", + "ulatory", + "ATA", + "angering", + "onso", + "allocate", + "garment", + "itudes", + "Huang", + "staples", + "Alban", + "trough", + "upright", + "tie", + "exploits", + "Vaughan", + "Darrell", + "assortment", + "Chill", + "learners", + "aqu", + "explode", + "Chong", + "bt", + "opl", + "altern", + "151", + "fur", + "ULT", + "HOU", + "Memory", + "boosts", + "ynes", + "priv", + "timeless", + "curtail", + "Cary", + "Hud", + "exclus", + "275", + "fry", + "Vera", + "defied", + "Dust", + "envision", + "Philipp", + "enhancements", + "LIB", + "ggy", + "Azure", + "esis", + "charismatic", + "coincide", + "inged", + "Choose", + "sizeable", + "136", + "pronounce", + "Positive", + "ideally", + "echoes", + "cottage", + "encrypted", + "Prime", + "á", + "flashes", + "Group", + "501", + "heat", + "atility", + "Testing", + "pex", + "WT", + "154", + "annah", + "compromising", + "inactive", + "disparity", + "gruesome", + "Feather", + "Mandal", + "thereof", + "Producer", + "profiling", + "logistical", + "cornerstone", + "Claudia", + "Congress", + "Dill", + "ophone", + "cameo", + "Cutler", + "craz", + "throw", + "Kasich", + "exploiting", + "Seas", + "agles", + "Geological", + "Stub", + "Ups", + "MER", + "mem", + "itution", + "understandably", + "contractual", + "warming", + "qi", + "Sky", + "whelming", + "curse", + "Aren", + "265", + "Gree", + "presiding", + "Works", + "stones", + "appalling", + "plex", + "dj", + "aunting", + "imag", + "sexism", + "Vert", + "Rag", + "Bliss", + "posium", + "div", + "experimenting", + "Ass", + "Lago", + "worthiness", + "Berk", + "Disneyland", + "exaggerated", + "iliation", + "FP", + "principals", + "Miami", + "ropri", + "PLE", + "iona", + "Pokemon", + "apse", + "bubbles", + "INC", + "Caps", + "Browne", + "sing", + "café", + "ceilings", + "frame", + "Irwin", + "ATS", + "dated", + "protester", + "taps", + "Oslo", + "Ù", + "concentrations", + "distributions", + "glucose", + "Rudolph", + "towels", + "âĸº", + "neighbourhoods", + "induction", + "glaring", + "annexation", + "unsustainable", + "Tend", + "thumbs", + "iegel", + "cript", + "gor", + "closure", + "thought", + "paddle", + "emulate", + "diameter", + "tailor", + "Corpor", + "icable", + "Prin", + "administer", + "Judd", + "Colleg", + "aund", + "Pond", + "NOTE", + "combating", + "invention", + "Oculus", + "Repl", + "iscal", + "trilogy", + "anian", + "ATT", + "Coke", + "DL", + "Lup", + "living", + "advertise", + "Connie", + "amping", + "sung", + "ORY", + "Tet", + "splits", + "reconnect", + "lou", + "mut", + "ulator", + "strap", + "swallow", + "rote", + "exec", + "ffen", + "Combine", + "Treat", + "sorrow", + "Notably", + "Sever", + "rette", + "wherein", + "transitioning", + "trout", + "cockpit", + "crawl", + "ferv", + "liquids", + "tsp", + "atell", + "measles", + "jug", + "Ac", + "KD", + "Moose", + "vans", + "chain", + "Papua", + "plet", + "Wednesday", + "lynn", + "chery", + "budget", + "Tony", + "Bacon", + "stirred", + "Specialist", + "counterfeit", + "а", + "differentiate", + "muscular", + "Theodore", + "looms", + "XX", + "ottage", + "benches", + "Municip", + "Po", + "Heck", + "scars", + "Nim", + "ÙĬ", + "Ingredients", + "ecological", + "AWS", + "dispose", + "mattered", + "720", + "patriotism", + "Grind", + "curved", + "opia", + "Liqu", + "evangelical", + "tto", + "Material", + "Showtime", + "BS", + "checkpoints", + "crippling", + "Balance", + "stress", + "bearing", + "216", + "Guards", + "linebackers", + "offending", + "sands", + "umbnail", + "atorial", + "liberties", + "GW", + "Pulitzer", + "Alvin", + "FAC", + "Strategies", + "reiter", + "Restaur", + "Lithuania", + "Swanson", + "terror", + "Maurit", + "paradise", + "zzle", + "owment", + "WP", + "sodium", + "futuristic", + "dots", + "Anthony", + "Though", + "stripes", + "orig", + "ultz", + "340", + "KK", + "umer", + "ivery", + "placebo", + "democrat", + "submerged", + "Hidden", + "pieces", + "asteroid", + "Graphic", + "advert", + "sil", + "dreaming", + "nationality", + "fostering", + "daughter", + "Savings", + "mischief", + "Clair", + "Bundy", + "blatant", + "tabs", + "qa", + "severe", + "attered", + "greed", + "resembles", + "nominal", + "ineligible", + "wealth", + "fax", + "payers", + "displacement", + "itute", + "unpleasant", + "Pom", + "lif", + "edo", + "NP", + "Inter", + "cohort", + "Stacy", + "Dai", + "histories", + "alin", + "273", + "dram", + "Kand", + "expectancy", + "ansson", + "limbo", + "Polar", + "divine", + "oused", + "shel", + "Problem", + "achment", + "âĸł", + "shoot", + "antam", + "Herz", + "157", + "preventive", + "keye", + "Sing", + "characteristic", + "casually", + "Taiwanese", + "md", + "Hubbard", + "imon", + "sect", + "148", + "martyr", + "stud", + "congrat", + "SWAT", + "Theory", + "INAL", + "opping", + "ply", + "Kindle", + "uu", + "Lith", + "kaya", + "Activity", + "uously", + "Jeb", + "tell", + "Spin", + "Explorer", + "folded", + "Canterbury", + "Stur", + "miniature", + "multif", + "Pressure", + "angling", + "Overse", + "resides", + "impressions", + "authored", + "265", + "allergies", + "143", + "Ji", + "sticker", + "Accord", + "caste", + "separates", + "Fein", + "Daily", + "179", + "Scores", + "Auction", + "hea", + "disclosing", + "Tacoma", + "verse", + "Beg", + "fabrics", + "aez", + "attachment", + "isy", + "Christ", + "addictive", + "vir", + "Week", + "Plum", + "croft", + "itivity", + "Exhibition", + "bruised", + "mimic", + "rers", + "anal", + "unintended", + "pall", + "atts", + "Warn", + "slows", + "WH", + "embro", + "nec", + "168", + "285", + "ologic", + "hob", + "Peel", + "Mill", + "eps", + "robbers", + "Dahl", + "semble", + "omics", + "toe", + "Loch", + "reproduction", + "Cullen", + "implants", + "wow", + "STATE", + "vt", + "depleted", + "breweries", + "hateful", + "gast", + "hollow", + "radically", + "ographed", + "Fog", + "onian", + "Sequ", + "disrespectful", + "Dis", + "Exper", + "pron", + "Amelia", + "Sage", + "bath", + "transformative", + "tremendously", + "pillow", + "Normal", + "Cont", + "Medic", + "educated", + "redesigned", + "kneeling", + "inh", + "roofs", + "handmade", + "protracted", + "Isn", + "Capacity", + "squash", + "Vega", + "fats", + "Certified", + "ointed", + "pricey", + "Basil", + "freezer", + "scent", + "pizz", + "Ard", + "distractions", + "violently", + "Hess", + "func", + "undert", + "rejuven", + "disbelief", + "cluded", + "named", + "Failure", + "kus", + "hostages", + "Sahara", + "1944", + "Leary", + "Prel", + "enza", + "Ally", + "Kak", + "counselors", + "Gale", + "Hok", + "Sold", + "hacker", + "hun", + "bung", + "declares", + "infringement", + "OOD", + "doub", + "jam", + "allergy", + "Shipping", + "medic", + "accommod", + "documenting", + "companions", + "modelling", + "carriage", + "Cherokee", + "tresp", + "taxable", + "Activities", + "Crane", + "bots", + "Russo", + "stocked", + "ervation", + "coffin", + "aign", + "guards", + "onwards", + "frank", + ".*", + "unic", + "cens", + "enic", + "ruit", + "rained", + "adapting", + "aments", + "stagnant", + "azaar", + "Harlem", + "158", + "ysis", + "braking", + "dipping", + "clan", + "Shu", + "props", + "qualified", + "mistakenly", + "Stalin", + "addicts", + "CALL", + "ropolis", + "aten", + "pec", + "Dro", + "Fellowship", + "Supporting", + "loc", + "uben", + "499", + "Bro", + "pots", + "chunks", + "wr", + "Colonial", + "Architecture", + "constrained", + "envelop", + "Ironically", + "aban", + "apparatus", + "cue", + "borne", + "Roz", + "ilton", + "theoretical", + "Watching", + "fuck", + "Silk", + "STE", + "bler", + "POST", + "Upton", + "summons", + "Cum", + "KL", + "relaxation", + "Duff", + "incumb", + "Redd", + "stature", + "canv", + "added", + "remedies", + "ISO", + "Decker", + "afloat", + "startling", + "Bethlehem", + "realizes", + "find", + "Ara", + "phased", + "arov", + "halting", + "Window", + "dentist", + "tumble", + "validation", + "carve", + "IPS", + "irrit", + "Essential", + "fluids", + "rons", + "implant", + "nuisance", + "Shelley", + "Gemini", + "pharmac", + "iction", + "taped", + "Governments", + "ruly", + "scant", + "prominently", + "reim", + "unning", + "arted", + "Matters", + "1918", + "Pros", + "atel", + "Battalion", + "onduct", + "talk", + "Tinder", + "Instant", + "Kern", + "buckets", + "Groups", + "metaphor", + "cloud", + "String", + "Ohio", + "caffeine", + "Old", + "definite", + "Nikola", + "Lords", + "icol", + ")?", + "enjoyment", + "famine", + "definitions", + "Jem", + "Check", + "aiding", + "Mé", + "renewables", + "sightings", + "footed", + "Box", + "goats", + "shack", + "AX", + "Monk", + "Graduate", + "meats", + "handle", + "147", + "rys", + "unsub", + "Pont", + "uble", + "440", + "eyel", + "thro", + "creep", + "^^^^", + "popcorn", + "compression", + "sal", + "ouf", + "repairing", + "Think", + "doubtful", + "Looks", + "taller", + "sul", + "sf", + "give", + "Gau", + "revered", + "EMBER", + "sloppy", + "ersen", + "vitamins", + "Improvement", + "progresses", + "diploma", + "semb", + "ustain", + "chant", + "bumped", + "sabotage", + "nant", + "rabbit", + "dividing", + "Defender", + "lik", + "irrespective", + "cade", + "Ster", + "touch", + "EMA", + "parted", + "BAR", + "hung", + "annoyed", + "hinder", + "examines", + "oan", + "Boe", + "aggreg", + "Chu", + "UCS", + "IGHTS", + "pez", + "UNESCO", + "windshield", + "Martin", + "withhold", + "does", + "bruising", + "deterior", + "bourg", + "Towers", + "JD", + "England", + "equivalents", + "razor", + "reassuring", + "ident", + "208", + "reath", + "ceans", + "patrolling", + "eve", + "pots", + "itative", + "sided", + "sofa", + "unborn", + "aug", + "perpetual", + "effect", + "represented", + "rails", + "Summers", + "MOR", + "Slow", + "Expert", + "shameful", + "audits", + "Sl", + "Burr", + "adow", + "WAY", + "anic", + "Islamists", + "Stranger", + "pse", + "amaz", + "Peggy", + "Seventh", + "screenplay", + "Griff", + "Ireland", + "142", + "neural", + "Fernand", + "ainment", + "Migration", + "ureen", + "SCH", + "Sullivan", + "Wag", + "REG", + "420", + "inky", + "Newspaper", + "School", + "Ok", + "Krishna", + "480", + "erald", + "skipping", + "harrowing", + "158", + "rogen", + "betrayal", + "culmination", + "Circ", + "211", + "stro", + "Trace", + "heaviest", + "td", + "Henri", + "epend", + "RB", + "arella", + "umbai", + "crem", + "Distribut", + "ruff", + "screams", + "scathing", + "girls", + "tiles", + "Evil", + "usp", + "knowledgeable", + "restitution", + "WiFi", + "itiner", + "exper", + "oris", + "Pokémon", + "iane", + "produ", + "Achievement", + "brunt", + "Surgery", + "pragmatic", + "Ber", + "Kejriwal", + "cus", + "consensual", + "acet", + "Secondly", + "divul", + "uca", + "busted", + "emies", + "Mou", + "217", + "excludes", + "Samoa", + "lofty", + "Sic", + "Remem", + "dn", + "eradicate", + "pies", + "scenery", + "ATTLE", + "WAS", + "innovate", + "Everest", + "synonymous", + "izen", + "euth", + "FIA", + "ITIES", + "Suddenly", + "foray", + "pell", + "ÄŁ", + "licensed", + "fra", + "blasting", + "autical", + "Blizzard", + "orer", + "chili", + "Sylvia", + "except", + "tec", + "Resistance", + "young", + "usions", + "iotic", + "Dreams", + "Archives", + "unleash", + "Pract", + "likened", + "ga", + "disappearing", + "unnoticed", + "frightened", + "arms", + "CAD", + "coloured", + "Signs", + "oing", + "vodka", + "ruption", + "otions", + "isal", + "Become", + "swoop", + "reating", + "choking", + "unforgettable", + "258", + "packs", + "345", + "Autumn", + "ther", + "399", + "Faculty", + "1933", + "Normally", + "orge", + "Tess", + "Chrom", + "scripts", + "biking", + "Act", + "grazing", + "Labrador", + "Ley", + "wandering", + "fend", + "Polk", + "Keane", + "Beef", + "elope", + "Approximately", + "1952", + "personal", + "historians", + "McDonnell", + "must", + "LES", + "iking", + "therm", + "humane", + "crowdfunding", + "Benefits", + "Land", + "analog", + "agency", + "Crowley", + "births", + "obj", + "fren", + "Salmon", + "bies", + "reve", + "216", + "betrayed", + "induced", + "acles", + "trad", + "forgiven", + "earners", + "208", + "xen", + "unle", + "necklace", + "gravel", + "salads", + "grooming", + "California", + "possessed", + "proclamation", + "sequences", + "ream", + "FOX", + "arkin", + "TRAN", + "purs", + "Loans", + "sacrificed", + "iceberg", + "Phill", + "galvan", + "smugglers", + "formation", + "onson", + "Vaughn", + "doctrine", + "Eyes", + "unmanned", + "states", + "determin", + "almost", + "eviction", + "tid", + "ARR", + "cooks", + "Bad", + "Camb", + "linear", + "229", + "Cooke", + "Purch", + "join", + "Cult", + "Refugee", + "slamming", + "ðŁij", + "pedal", + "Veronica", + "landowners", + "Yel", + "Workshop", + "antic", + "dysfunction", + "229", + "culturally", + "infuri", + "Eck", + "sem", + "wired", + "Werner", + "lov", + "Jasper", + "vehemently", + "Spy", + "lift", + "Nab", + "Pound", + "Hanna", + "leveled", + "WOOD", + "tm", + "Kitt", + "conve", + "nat", + "jog", + "IVER", + "memes", + "seaw", + "ector", + "sprayed", + "vaccinated", + "Europe", + "mustard", + "Mahm", + "214", + "Research", + "iminary", + "concerted", + "Detroit", + "kios", + "plummet", + "visuals", + "247", + "228", + "development", + "Pascal", + "acial", + "Seasons", + "TL", + "480", + "Reader", + "expulsion", + "choked", + "devotion", + "STAT", + "urred", + "fascinated", + "stealth", + "NL", + "booster", + "Kat", + "Priebus", + "aux", + "Hate", + "Thing", + "abnormal", + "calmly", + "dedicate", + "cause", + "isolate", + "Pai", + "suspensions", + "poisoned", + "ission", + "prohibiting", + "353", + "banks", + "kissed", + "Begin", + "atis", + "LI", + "shaft", + "Guth", + "Boo", + "cinnamon", + "verbally", + "Rabbi", + "monsters", + "done", + "Clyde", + "spar", + "Cage", + "Persons", + "305", + "Mons", + "jealous", + "swirling", + "know", + "prote", + "cruising", + "duly", + "chapel", + "groove", + "bps", + "Kelvin", + "iom", + "aer", + "bomb", + "Christian", + "gigs", + "+.", + "Wei", + "farmland", + "otally", + "equitable", + "CBO", + "chool", + "amara", + "wealthiest", + "Means", + "235", + "Uk", + "steps", + "raham", + "nerg", + "clad", + "sled", + "Morrow", + "152", + "Rece", + "plausible", + "bisexual", + "artments", + "veh", + "Loft", + "bly", + "CONC", + "automatic", + "masterpiece", + "Springer", + "tendencies", + "Ro", + "resentment", + "adversely", + "bandwidth", + "DAV", + "tun", + "puppies", + "Bundes", + "Hort", + "Garfield", + "enlist", + "mont", + "gd", + "rooting", + "Dream", + "fulfillment", + "chal", + "182", + "prop", + "159", + "courtyard", + "iard", + "Sle", + "operative", + "publishes", + "Proposition", + "critique", + "redist", + "wang", + "Nep", + "DD", + "bonding", + "141", + "Assault", + "-'", + "lodging", + "itters", + "cigarettes", + "__", + "Laf", + "GF", + "Anat", + "Stephan", + "214", + "Kass", + "viz", + "piling", + "fugitive", + "Currency", + "Crypto", + "faux", + "Ping", + "Lia", + "igl", + "adversaries", + "YPG", + "Comb", + "Yar", + "heny", + "overhe", + "Fest", + "emy", + "Ever", + "370", + "secretive", + "SEN", + "MEM", + "PRESS", + "Birth", + "kos", + "precarious", + "irting", + "UI", + "occupying", + "olute", + "periodic", + "eon", + "iens", + "RH", + "Win", + "playbook", + "exodus", + "Skinner", + "orderly", + "Ved", + "ouses", + "escal", + "benign", + "bots", + "Whis", + "appra", + "FOR", + "Chromebook", + "_____", + "990", + "athed", + "spirited", + "illi", + "bicycles", + "orse", + "ifestyle", + "orno", + "Dept", + "JA", + "nausea", + "pervasive", + "velop", + "commun", + "Universities", + "remnants", + "disarm", + "Boots", + "prin", + "...\"", + "quila", + "cautiously", + "uper", + "onto", + "din", + "velocity", + "conspiring", + "MX", + "emphasizing", + "âĸ", + "Stam", + "spices", + "airplanes", + "uty", + "culture", + "Petr", + "glor", + "Excel", + "Speech", + "harmless", + "Pend", + "Crossing", + "Document", + "ramifications", + "Croatian", + "Killer", + "multim", + "discontinued", + "cherished", + "Maker", + "aspers", + "Blooming", + "Mata", + "offic", + "settlers", + "Plenty", + "Institutes", + "Arpaio", + "Pool", + "Subst", + "380", + "decidedly", + "ollah", + "Den", + "Jiang", + "Amos", + "Grand", + "Turns", + "meyer", + "conducive", + "poignant", + "abortion", + "notebook", + "shelling", + "common", + "Pavel", + "humid", + "inappropriately", + "????", + "soar", + "dynasty", + "researched", + "Yon", + "maple", + "wedge", + "mass", + "TM", + "USE", + "eln", + "gloss", + "rigan", + "steen", + "DeV", + "debacle", + "Christmas", + "tweaks", + "grab", + "profoundly", + "campaigner", + "Seal", + "iteration", + "sigh", + "unfounded", + "framing", + "recognizable", + "seizing", + "legal", + "proportions", + "omers", + "rek", + "screenshot", + "itsu", + "OG", + "Ying", + "Mississ", + "295", + "landsl", + "psychiatrist", + "sov", + "arine", + "Ju", + "flo", + "apple", + "hof", + "wig", + "ENT", + "enthusiast", + "Such", + "Artificial", + "happy", + "oton", + "Fram", + "Remove", + "smear", + "jer", + "topp", + "imbalance", + "Words", + "coffers", + "olina", + "rigged", + "uction", + "idding", + "dispensaries", + "dermat", + "shutter", + "idental", + "continu", + "humility", + "bulbs", + "207", + "lass", + "Beirut", + "Ult", + "urry", + "NEWS", + "feminine", + "simulated", + "charger", + "mom", + "Creed", + "wolves", + "essions", + "created", + "ifiers", + "dissemin", + "Darling", + "umann", + "marrying", + "shred", + "avin", + "budgetary", + "medicinal", + "ulin", + "seys", + "agues", + "extracted", + "Flower", + "continents", + "Wish", + "divides", + "Ding", + "insulation", + "respect", + "ABS", + "reconcile", + "keep", + "ILD", + "genome", + "410", + "Sweep", + "harass", + "frantic", + "EE", + "dad", + "aperture", + "rought", + "hugs", + "drying", + "overrun", + "Space", + "periodically", + "brightness", + "atched", + "kee", + "ITS", + "Spokane", + "Seaf", + "desks", + "Eisen", + "OPS", + "cider", + "acceler", + "Athlet", + "2008", + "Guid", + "Manip", + "mould", + "misguided", + "brow", + "managerial", + "hugged", + "furnish", + "Harmony", + "Hebrew", + "typh", + "decreases", + "impetus", + "contagious", + "unch", + "209", + "swell", + "Huffington", + "pubs", + "adequ", + "amoto", + "rir", + "pristine", + "anx", + "Secure", + "enrichment", + "VAL", + "summed", + "confidently", + "Profit", + "Frog", + "Lena", + "FUN", + "bruises", + "uproar", + "coll", + "Impro", + "flair", + "146", + "Brend", + "166", + "enhances", + "Dent", + "degener", + "proponents", + "Inspired", + "ramps", + "wisely", + "Western", + "tart", + "steered", + "treason", + "dropping", + "transc", + "Scarlett", + "Ezekiel", + "pivot", + "esame", + "Show", + "discontent", + "Judith", + "Putting", + "blessings", + "hardcore", + "tray", + "discern", + "oley", + "ouk", + "wil", + "intolerance", + "157", + "Relative", + "Lynd", + "whistleblower", + "incon", + "Tao", + "indefinite", + "guardians", + "agon", + "Instruments", + "existential", + "AAF", + "vind", + "brazen", + "condition", + "ratified", + "fam", + "Hin", + "Michaels", + "204", + "Kats", + "ITS", + "ISON", + "prone", + "boiling", + "prolong", + "noticing", + "resident", + "brance", + "Folk", + "desserts", + "uton", + "Web", + "Longh", + "Reef", + "Going", + "Carb", + "Sur", + "complete", + "Sloan", + "Clubs", + "Sadd", + "shrugged", + "edible", + "Typ", + "thal", + "Rocks", + "Clive", + "kidding", + "Crom", + "Turks", + "Wak", + "eyewitness", + "Hass", + "collar", + "succeeding", + "insert", + "224", + "Bret", + "neurological", + "rewrite", + "imil", + "ultimate", + "Jeremiah", + "liaison", + "pedd", + "direct", + "Yi", + "MAD", + "Orion", + "oyd", + "LOC", + "release", + "investigates", + "Apache", + "û", + "Vend", + "cynical", + "Helm", + "Movies", + "tops", + "sinister", + "unparalleled", + "spikes", + "overlap", + "enstein", + "hypocrisy", + "Plus", + "expansions", + "vow", + "detonated", + "fellowship", + "solicitor", + "Newtown", + "mony", + "Lod", + "Developers", + "ateg", + "ibus", + "crumbling", + "Wein", + "Klan", + "gio", + "Phys", + "Antarctica", + "368", + "seam", + "automobiles", + "TEAM", + "bern", + "manic", + "sanct", + "equals", + "Est", + "incentiv", + "Hawking", + "nin", + "resonate", + "bid", + "telescope", + "endon", + "Vacc", + "regretted", + "1300", + "Forestry", + "BOOK", + "groundwork", + "essays", + "Indo", + "Pierre", + "Chau", + "apologies", + "killers", + "Moroccan", + "0001", + "336", + "Ra", + "parcels", + "leaned", + "thankfully", + "Split", + "lobbied", + "Degree", + "risking", + "assy", + "supplemental", + "little", + "eclectic", + "206", + "ealing", + "206", + "repo", + "hose", + "ayn", + "lux", + "believer", + "')", + "Hide", + "vance", + "Einstein", + "depos", + "fray", + "ki", + "internship", + "Hou", + "Vis", + "stare", + "Breed", + "option", + "visionary", + "mins", + "bitten", + "ancies", + "Shake", + "template", + "liner", + "muster", + "appro", + "Mubarak", + "esty", + "mong", + "actory", + "headphone", + "Prec", + "waive", + "Ron", + "Hearing", + "imperfect", + "sealing", + "locating", + "culminated", + "chio", + "channel", + "lust", + "Lowell", + "woods", + "soak", + "forbidden", + "detached", + "unct", + "Hunger", + "Patient", + "Polo", + "Saharan", + "Jon", + "athered", + "Signal", + "Six", + "statistically", + "ITH", + "artment", + "CU", + "hates", + "qual", + "capitalist", + "ATES", + "Desc", + "handcuffed", + "indulge", + "Religious", + "German", + "housing", + "dismantling", + "conventions", + "dain", + "chairs", + "loos", + "knowingly", + "Var", + "husbands", + "eez", + "asion", + "Issa", + "swollen", + "1946", + "headlined", + "Chelsea", + "ignorant", + "peripheral", + "Note", + "axe", + "nicotine", + "Sanctuary", + "1917", + "withdrawals", + "uits", + "Hot", + "reimburse", + "probably", + "Adapt", + "industrial", + "answer", + "orus", + "Mell", + "Talk", + "contemplating", + "omas", + "taxis", + "encompasses", + "rations", + "Latvia", + "humiliating", + "loft", + "tight", + "rium", + "login", + "Bulletin", + "turtles", + "EAR", + "349", + "Radio", + "Bord", + "151", + "kk", + "pocket", + "dove", + "348", + "temptation", + "Coy", + "those", + "Dest", + "ishly", + "rn", + "mammals", + "Tub", + "arial", + "Persian", + "daddy", + "Zen", + "ps", + "]", + "Field", + "adiq", + "meaningless", + "primer", + "1942", + "!", + "625", + "fashionable", + "Theft", + "HAVE", + "christ", + "peril", + "repealing", + "buff", + "odor", + "stalking", + "Dems", + "iences", + "unilaterally", + "odies", + "Quite", + "bloodshed", + "infect", + "reminders", + "chop", + "evapor", + "877", + "horrified", + "Fruit", + "rams", + "insecure", + "cester", + "Nationwide", + "mocking", + "Ret", + "complying", + "sav", + "ali", + "Family", + "Ĩ", + "dishonest", + "incorrectly", + "LOAD", + "Gand", + "ourcing", + "obby", + "Petersen", + "Something", + "ravaged", + "limited", + "rituals", + "Knowledge", + "Utility", + "doom", + "sheds", + "Gael", + "Millennials", + "Monthly", + "domination", + "rapport", + "spot", + "Prest", + "HA", + "ushes", + "tact", + "Richard", + "gritty", + "Does", + "TNT", + "downfall", + "Wood", + "Prediction", + "Pour", + "Fraud", + "Syndrome", + "166", + "literal", + "addict", + "Loud", + "hens", + "Accounts", + "distance", + "classmate", + "salv", + "unlucky", + "partying", + "Kou", + "SNAP", + "%-", + "delegate", + "strikers", + "Slate", + "articulate", + "390", + "inqu", + "discredit", + "Priv", + "ploy", + "Marketplace", + "Tune", + "visor", + "wrestle", + "kindly", + "Collect", + "circ", + "Remain", + "192", + "contin", + "325", + "severed", + "isations", + "muddy", + "taxing", + "Represent", + "Sty", + "rology", + "Judges", + "Bronze", + "Applic", + "arrow", + "consuming", + "Featuring", + "spies", + "noises", + "Colony", + "lost", + "opp", + "deem", + "Garc", + "icent", + "ptroller", + "liest", + "outward", + "User", + "intimidate", + "156", + "jab", + "ANGE", + "Jay", + "Poverty", + "ACA", + "rife", + "faint", + "Acceler", + "tall", + "UNITED", + "Fighter", + "Gilmore", + "sod", + "amura", + "predictive", + "polish", + "DD", + "fabricated", + "Dag", + "fatty", + "plague", + "exhib", + "Advent", + "1941", + "ERSON", + "initely", + "loneliness", + "Equality", + "untrue", + "onlook", + "fragmented", + "ruce", + "distrust", + "scal", + "Cors", + "robbing", + "cultural", + "clusion", + "Obi", + "sels", + "Evidence", + "Sac", + "fragments", + "flipping", + "Rabbit", + "disproportionate", + "Creat", + "labeling", + "Gri", + "161", + "Editors", + "holm", + "adr", + "Ĭ", + "tailed", + "renters", + "noodles", + "competence", + "panc", + "uration", + "acids", + "confid", + "rival", + "AAA", + "kson", + "recreate", + "153", + "164", + "Olympia", + "Unlimited", + "Shock", + "Teaching", + "Houses", + "resso", + "Maw", + "replen", + "protestors", + "bey", + "surve", + "emphasizes", + "223", + "Esther", + "Nikol", + "prosecutions", + "Freed", + "poss", + "OTE", + "Prayer", + "squarely", + "tir", + "adv", + "bogus", + "wrongful", + "embell", + "seldom", + "possesses", + "Er", + "Alternatively", + "instituted", + "rr", + "vocational", + "eval", + "Comics", + "stumbling", + "335", + "dragon", + "vine", + "services", + "crit", + "irens", + "layered", + "orb", + "dominates", + "Marx", + "period", + "avering", + "brigade", + "chem", + "Evolution", + "Suk", + "209", + "Malk", + "tallest", + "recogn", + "Craw", + "ell", + "Caesar", + "php", + "Survivors", + "sd", + "itsch", + "ambo", + "ashore", + "acular", + "rost", + "murderer", + "casts", + "Economist", + "Weapons", + "nostalgic", + "Skip", + "REAM", + "Pa", + "journals", + "Sitting", + "Union", + "Att", + "Maxim", + "purportedly", + "respecting", + "MAX", + "seed", + "juicy", + "Gallup", + "mileage", + "adier", + "bod", + "DER", + "summers", + "icult", + "ipl", + "Deng", + "smells", + "ivory", + "255", + "Id", + "DEN", + "159", + "Due", + "Lighting", + "Surely", + "sund", + "Kessler", + "immigrant", + "tragedies", + "Oxy", + "Fixed", + "Balk", + "oriented", + "pher", + "kitchens", + "hips", + "tweak", + "tuna", + "Cla", + "dislike", + "ussy", + "outnumbered", + "plumbing", + "cogn", + "Throw", + "TER", + "urally", + "Murd", + "creamy", + "residing", + "otics", + "fingerprints", + "!,", + "paused", + "Milo", + "homosexuality", + "responsibly", + "iop", + "UCT", + "succeeds", + "CRE", + "Thatcher", + "currents", + "arises", + "waterproof", + "amp", + "Claims", + "177", + "subpoen", + "vig", + "Neuro", + "blur", + "Paint", + "campus", + "toughness", + "Button", + "Neal", + "DEN", + "Nir", + "Axel", + "EEP", + "pint", + "agile", + "odor", + "essentials", + "Mov", + "Venezuel", + "exchanging", + "Negative", + "Mil", + "Key", + "buzzing", + "Stew", + "rebuke", + "depl", + "Koz", + "163", + "shines", + "NZ", + "carnage", + "cases", + "warmed", + "Greenwich", + "College", + "needy", + "301", + "Mü", + "culation", + "440", + "425", + "atories", + "satisfactory", + "Fib", + "Elim", + "developed", + "vacations", + "peculiar", + "vets", + "onest", + "Pug", + "lifestyles", + "zzi", + "provoke", + "bah", + "arger", + "Virt", + "Sales", + "annel", + "Meth", + "ivating", + "revoke", + "Agenda", + "Ich", + "sensit", + "Azerbai", + "Bombay", + "uncon", + "river", + "apr", + "actic", + "Subaru", + "banquet", + "contradict", + "tek", + "Football", + "igent", + "reintrodu", + "Insight", + "systematically", + "boun", + "Fishing", + "stri", + "OB", + "stair", + "Wall", + "Allow", + "caramel", + "169", + "cafes", + "calcium", + "169", + "portraying", + "discriminate", + "unrestricted", + "mant", + "scarcity", + "feminism", + "JJ", + "Oversight", + "Cue", + "inexperienced", + "drafts", + "1939", + "nm", + "forest", + "Honour", + "ceramic", + "downstairs", + "boon", + "morality", + "horrifying", + "Rad", + "justice", + "mosques", + "curfew", + "surrogate", + "reimb", + "enth", + "pressure", + "beam", + "whirlwind", + "Recession", + "Tours", + "clusters", + "Quant", + "Jonathan", + "project", + "777", + "NOAA", + "abis", + "deficiencies", + "suicides", + "foothold", + "Yah", + "imeter", + "URN", + "cultivate", + "noisy", + "1951", + "pressuring", + "Deals", + "Prophet", + "Wikipedia", + "INESS", + "Shine", + "Called", + "Sole", + "Zhou", + "asphalt", + "armac", + "Scorp", + "Unknown", + "PAT", + "Heart", + "guessed", + "sushi", + "heartbeat", + "concent", + "eret", + "plin", + "weeds", + "bombed", + "Terrorism", + "Rich", + "blades", + "haunt", + "storefront", + "thwarted", + "access", + "Lydia", + "LINE", + "pregnancies", + "ripping", + "Believe", + "spoken", + "inian", + "sed", + "Brass", + "econom", + "current", + "voc", + "modeled", + "peppers", + "otech", + "Option", + "Connell", + "isel", + "compel", + "juveniles", + "NET", + "EXP", + "paradigm", + "Des", + "204", + "employed", + "durability", + "245", + "billionaires", + "violent", + "Cooperative", + "TOP", + "Garry", + "Soldiers", + "dared", + "voucher", + "blends", + "gue", + "adventurous", + "organisms", + "gaze", + "crap", + "Coach", + "omon", + "Wheels", + "Grayson", + "recy", + "grave", + "allergic", + "reef", + "beginnings", + "Ruff", + "clout", + "structed", + "315", + "Georgian", + "say", + "springs", + "Asus", + "repaid", + "Guys", + "ticket", + "unb", + "Certificate", + "STORY", + "cin", + "passions", + "mediocre", + "lackluster", + "vernight", + "kids", + "Wife", + "politics", + "Himal", + "oddy", + "ensus", + "Gustav", + "binding", + "Individuals", + "maize", + "hoop", + "Changing", + "lessen", + "arranging", + "Fukushima", + "Trying", + "Mage", + "skeleton", + "Tec", + "289", + "recl", + "FIL", + "Gs", + "Odyssey", + "Processing", + "ilion", + "subsidized", + "abdomen", + "analyse", + "music", + "clean", + "unfinished", + "downloads", + "morally", + "218", + "trib", + "Keep", + "SER", + "FY", + "aust", + "discovers", + "GROUP", + "Machines", + "eroded", + "ominous", + "brightly", + "IME", + "wicked", + "Trou", + "visions", + "Kay", + "reported", + "bog", + "Quin", + "Sigma", + "urned", + "ixon", + "harming", + "checkout", + "inet", + "much", + "cherish", + "Byrd", + "Samson", + "WP", + "orders", + "boa", + "bron", + "oki", + "RR", + "suitcase", + "feathers", + "Christy", + "Islamic", + "amusement", + "ISS", + "intensive", + "Qaida", + "neurons", + "wagon", + "Tek", + "dolls", + "Shoot", + "underestimate", + "streamlined", + "fractures", + "cathedral", + "eliminates", + "helle", + "citrus", + "risis", + "impecc", + "istries", + "Hog", + "vote", + "pas", + "assign", + "Songs", + "Miracle", + "kas", + "zynski", + "crane", + "adulthood", + "Benefit", + "Grimes", + "payday", + "ablished", + "centerpiece", + "hassle", + "Appalachian", + "follow", + "290", + "RL", + "Doe", + "acclaim", + "levied", + "tossing", + "carrots", + "Darius", + "161", + "offspring", + "Jury", + "TPP", + "CAP", + "environmentalists", + "rays", + "267", + "Ser", + "captivity", + "appellate", + "Electricity", + "Enough", + "232", + "fisher", + "brilliance", + "praises", + "aunch", + "solicitation", + "adolescent", + "inferior", + "checks", + "Set", + "mutations", + "Latinos", + "License", + "Ame", + "hirt", + "Chun", + "deeds", + "ldon", + "mammoth", + "turtle", + "rule", + "Ken", + "voyage", + "gram", + "conquer", + "retaliate", + "PJ", + "Viking", + "safegu", + "ordinary", + "Arbit", + "Digest", + "Die", + "bureaucratic", + "honorable", + "cafeteria", + "RAF", + "Places", + "Klu", + "Cam", + "Biology", + "Cycling", + "imore", + "stripping", + "warriors", + "bursting", + "lapse", + "versa", + "clicked", + "ogh", + "\"âĢ¦", + "diligently", + "Miy", + "Corpus", + "redef", + "176", + "Instrument", + "OECD", + "stro", + "microwave", + "Santa", + "pars", + "Social", + "iffe", + "itability", + "Equ", + "nud", + "legged", + "Tud", + "lav", + "interpreter", + "alcohol", + "imposition", + "dwelling", + "1400", + "].\"", + "Iw", + "RM", + "555", + "paralyzed", + "mind", + "rans", + "adin", + "French", + "liar", + "Represent", + "strapped", + "orate", + "rigging", + "interrog", + "sparse", + "ento", + "Them", + "baseless", + "buildup", + "undecided", + "isms", + "abduct", + "flowed", + "prestige", + "hacks", + "panicked", + "Cast", + "Krish", + "umat", + "antique", + "bitters", + "entitlement", + "standby", + "Ten", + "said", + "Conditions", + "events", + "obey", + "shortest", + "etting", + "concentrating", + "Needs", + "234", + "intrigued", + "enting", + "Xen", + "Alger", + "seekers", + "anish", + "172", + "âĢij", + "silicon", + "standardized", + "Fountain", + "essel", + "approves", + "sucked", + "gone", + "Briggs", + "brother", + "artisan", + "Continuing", + "vir", + "submarines", + "Ink", + "program", + "Nexus", + "Coco", + "conceptual", + "matt", + "aughters", + "baths", + "beaut", + "Emerald", + "Parties", + "248", + "completely", + "esan", + "diarrhea", + "1100", + "borg", + "Broken", + "reiterate", + "sorting", + "ONS", + "177", + "admin", + "Mandatory", + "symptom", + "paced", + "Remember", + "abdominal", + "swapped", + "transitions", + "IFA", + "pretty", + "JC", + "allotted", + "Shows", + "Arthur", + "soften", + "dozen", + "Mah", + "extinguished", + "reelection", + "deployments", + "sturdy", + "downright", + "jams", + "Optim", + "humiliation", + "cd", + "bunk", + "sie", + "NAT", + "ilies", + "implying", + "<", + "homepage", + "242", + "ey", + "dict", + "slender", + "forehead", + "Cecil", + "shrunk", + "Exit", + "expressly", + "seals", + "Thiel", + "umni", + "damning", + "VS", + "ulum", + "BBC", + "URES", + "inhal", + "font", + "workplaces", + "PUBLIC", + "Horror", + "Bs", + "arta", + "Bread", + "stret", + "ethos", + "stabilized", + "convers", + "Inqu", + "judgments", + "Contemporary", + "221", + "zombie", + "VD", + "misunderstanding", + "spam", + "Papers", + "crocod", + "ENA", + "Juven", + "Abram", + "bursts", + "atto", + "turbulence", + "tty", + "sexual", + "waning", + "community", + "Government", + "transpl", + "??", + "Getting", + "Rare", + "prime", + "looting", + "validate", + "Creating", + "Corruption", + "spit", + "Favorite", + "Kar", + "adaptive", + "ART", + "torso", + "Ident", + "subdivision", + "azo", + "consequently", + "rotate", + "Wit", + "estab", + "managed", + "Bound", + "skim", + "198", + "Corona", + "âĿ", + "wording", + "buck", + "iph", + "patrick", + "Help", + "flying", + "racer", + "fisherman", + "____", + "ackers", + "persisted", + "myths", + "garn", + "ologue", + "Apprentice", + "hereby", + "vulgar", + "Ginger", + "trait", + "Idea", + "figur", + "Schwarzenegger", + "Safari", + "178", + "Asians", + "775", + "Triangle", + "demons", + "Ov", + "anime", + "Broad", + "molecule", + "deposition", + "biodiversity", + "modern", + "wallets", + "NH", + "planes", + "rats", + "Seed", + "174", + "umed", + "touting", + "gre", + "SEAL", + "perpetrator", + "Gerrard", + "allocations", + "worsh", + "payment", + "bett", + "Issues", + "ennis", + "eering", + "MV", + "yi", + "hak", + "167", + "orchestr", + "224", + "sup", + "leukemia", + "osures", + "575", + "noticeably", + "paramilitary", + "THERE", + "waged", + "igrated", + "documentaries", + "senseless", + "bark", + "genetics", + "Albania", + "Crypt", + "SEO", + "nightly", + "faults", + "279", + "Ferdinand", + "Sylv", + "calam", + "Muller", + "Spielberg", + "Boy", + "Urs", + "rug", + "colonies", + "Funk", + "lyric", + "ATT", + "anni", + "NB", + "thorn", + "pertinent", + "188", + "partic", + "Head", + "Pad", + "Palestinian", + "Barg", + "anical", + "beaut", + "onge", + "gigantic", + "travel", + "downloading", + "Contin", + "whe", + "plane", + "Wil", + "IDA", + "Ele", + "PAL", + "beams", + "Proud", + "ramer", + "independents", + "translator", + "Brah", + "Trooper", + "aylor", + "pson", + "guise", + "differing", + "topple", + "ichen", + "Seymour", + "deg", + "Mixed", + "involuntary", + "countdown", + "Narc", + "Adults", + "coaster", + "342", + "Acquisition", + "mone", + "penchant", + "Brian", + "Gh", + "Pres", + "enei", + "reefs", + "Maver", + "devised", + "IMP", + "vict", + "agility", + "Payments", + "respected", + "tuning", + "FACE", + "actions", + "yell", + "Leaving", + "snowy", + "Saudi", + "formations", + "airborne", + "deed", + "ooks", + "namesake", + "punishable", + "agg", + "oths", + "Famous", + "Deposit", + "induce", + "189", + "hesitation", + "Browse", + "ople", + "reys", + "henko", + "secretaries", + "intersections", + "diminishing", + "ints", + "1934", + "Investigative", + "Mexicans", + "Mahar", + "ibur", + "stocking", + "gross", + "asbestos", + "agitation", + "BST", + "Overall", + "heats", + "Span", + "imped", + "trusting", + "Pet", + "egregious", + "comedians", + "zin", + "WIN", + "chats", + "exploding", + "Tort", + "embraces", + "neut", + "verson", + "ouncing", + "Fiber", + "baker", + "unstoppable", + "Dial", + "cars", + "Marc", + "164", + "volt", + "ceased", + "EFF", + "promoters", + "circuits", + "excise", + "seminars", + "Tiny", + "Important", + "Tup", + "outburst", + "SOC", + "WWII", + "merging", + "highly", + "Gmail", + "ozy", + "KB", + "laboratories", + "knit", + "Closed", + "surrounds", + "Vet", + "cere", + "vard", + "Deadpool", + "text", + "infusion", + "cuc", + "Atl", + "bustling", + "Settings", + "193", + "ryan", + "184", + "186", + "swat", + "rane", + "epidem", + "lando", + "testifying", + "moistur", + "Tens", + "exemplary", + "Pump", + "forcefully", + "Fare", + "complicate", + "Fe", + "Di", + "Thy", + "compartment", + "Fiesta", + "Would", + "fitted", + "cull", + "comedic", + "cyl", + "whichever", + "stic", + "213", + "spills", + "plasma", + "disguise", + "Compass", + "Immun", + "scarf", + "disperse", + "reckon", + "Taste", + "root", + "GAME", + "xx", + "homophobic", + "dimin", + "/#", + "178", + "gems", + "lio", + "informed", + "ample", + "XT", + "repression", + "Takes", + "habitats", + "mountainous", + "McH", + "ENC", + "Mobil", + "reel", + "TI", + "authorize", + "Accept", + "Metall", + "CCC", + "wetlands", + "Witch", + "heading", + "intervals", + "Witt", + "hene", + "comforting", + "ollen", + "ERN", + "ooky", + "etch", + "assailant", + "announced", + "elin", + "plate", + "920", + "eating", + "induced", + "Igor", + "Amph", + "patented", + "posing", + "extraordinarily", + "fearless", + "mortem", + "Draw", + "Rend", + "Son", + "ridden", + "Advantage", + "305", + "roared", + "Str", + "radioactive", + "slur", + "Rear", + "affles", + "Pon", + "ost", + "umbs", + "Slack", + "athom", + "baby", + "213", + "Spending", + "Accordingly", + "clocks", + "archs", + "smugg", + "mastermind", + "Klaus", + "alpha", + "spoiled", + "264", + "Pod", + "flared", + "composure", + "CAM", + "restruct", + "tasted", + "Kimber", + "upheaval", + "CHAR", + "Geo", + "itations", + "begged", + "UX", + "Authorities", + "Engel", + "HOME", + "ratt", + "quickest", + "475", + "Sting", + "ICO", + "yu", + "defy", + "Prince", + "cards", + "overtake", + "retrieved", + "Navajo", + "pastry", + "Lange", + "entrusted", + "Cull", + "aler", + "dinosaurs", + "bragging", + "Alley", + "meier", + "Assuming", + "ana", + "omatic", + "Brend", + "acted", + "exhaustive", + "unfit", + "Several", + "gap", + "tet", + "228", + "Sk", + "302", + "deflect", + "179", + "226", + "adorned", + "Spread", + "thirds", + "Semi", + "descend", + "accumulate", + "flavours", + "invoked", + "Ange", + "profess", + "unks", + "Kickstarter", + "ENTS", + "Rw", + "chatter", + "POS", + "collaborators", + "EW", + "Markus", + "impair", + "bolt", + "glue", + "loosely", + "SUM", + "hydraulic", + "predatory", + "Charles", + "cond", + "spawned", + "Fr", + "174", + "tame", + "aggrav", + "christ", + "true", + "ivable", + "hen", + "Kut", + "skyrocket", + "eg", + "veterinarian", + "Stats", + "Kit", + "biologist", + "Spe", + "antenna", + "sust", + "fill", + "payload", + "227", + "livestream", + "ORN", + "Abel", + "deception", + "ussen", + "Britain", + "partisan", + "browse", + "melan", + "172", + "Numerous", + "Mansion", + "assailants", + "£", + "olerance", + "directives", + "Integ", + "zers", + "duct", + "Honestly", + "Immediately", + "ixty", + "diagnose", + "implication", + "iPads", + "testers", + "riots", + "respons", + "XP", + "pes", + "875", + "199", + "Poe", + "303", + "ailments", + "Carrier", + "eject", + "restroom", + "Drive", + "manufact", + "compens", + "glossy", + "recovers", + "thinner", + "descendants", + "antle", + "Beaut", + "competitive", + "Robotics", + "pretext", + "233", + "flanked", + "âĻ", + "guts", + "wee", + "accents", + "mc", + "grapp", + "Nathaniel", + "Mikhail", + "obligated", + "manoeuv", + "echoing", + "189", + "Device", + "isd", + "loopholes", + "behold", + "Merry", + "funn", + "nuanced", + "667", + "ELY", + "Tasmania", + "Saddam", + "quizz", + "military", + "cient", + "outlaw", + "Audit", + "Boom", + "crim", + "asured", + "Apps", + "Kush", + "onica", + "amput", + "signed", + "MEN", + "Rosenberg", + "vide", + "Direction", + "fountain", + "TW", + "CARE", + "reassured", + "Food", + "depressing", + "Whilst", + "reatment", + "spelled", + "hipp", + "Peach", + "hound", + "Harry", + "catalogue", + "Commun", + "nurture", + "rush", + "Population", + "NTS", + "Electrical", + "rounded", + "blending", + "223", + "alities", + "ilation", + "eas", + "estate", + "narrowing", + "Treasure", + "192", + "whims", + "robber", + "soaked", + "nian", + "congest", + "Yosemite", + "notes", + "icer", + "Guardians", + "Frozen", + "187", + "handcuffs", + "Someone", + "enshr", + "gency", + "Cube", + "printers", + "undercut", + "Solution", + "rosis", + "Humanity", + "sucks", + "Sick", + "Tax", + "tablespoon", + "Trin", + "Archive", + "Mom", + "SAY", + "drifting", + "Farage", + "forging", + "WM", + "Eleanor", + "USH", + "emph", + "careless", + "spew", + "insensitive", + "awhile", + "cit", + "opened", + "Fem", + "vapor", + "downt", + "ylene", + "clut", + "culp", + "1990", + "disgruntled", + "Students", + "uttering", + "gyn", + "vre", + "rapes", + "division", + "Calendar", + "tal", + "icts", + "caliber", + "Fighters", + "Unc", + "163", + "Rogue", + "registrations", + "undermines", + "Punch", + "dramas", + "176", + "slider", + "Flore", + "ر", + "bru", + "inelli", + "disparities", + "ا", + "referrals", + "Charges", + "breeds", + "MEP", + "288", + "mouths", + "sideways", + "believers", + "ppard", + "hotter", + "underestimated", + "jelly", + "525", + "CMS", + "Weiner", + "guarding", + "ampl", + "Kidd", + "UF", + "orient", + "max", + "Ash", + "wander", + "..........", + "Dempsey", + "Token", + "chat", + "Justin", + "equipped", + "BI", + "sins", + "nond", + "ursion", + "coc", + "mailing", + "Architect", + "haunting", + "pont", + "ascertain", + "wig", + "skysc", + "arg", + "Italians", + "/?", + "----------------------------------------------------------------", + "Precision", + "EPA", + "hotly", + "circumvent", + "Ecc", + "merch", + "akov", + "unab", + "heres", + "subcommittee", + "Discuss", + "Challenger", + "crafted", + "canine", + "osphere", + "spider", + "teachings", + "atos", + "universally", + "turbine", + "LO", + "MAG", + "passers", + "roundup", + "denounce", + "Spiegel", + "until", + "shaved", + "disdain", + "Nazi", + "newfound", + "spontaneous", + "mash", + "Dispatch", + "sunrise", + "ogged", + "fuss", + "eas", + "acci", + "Targ", + "hash", + "lict", + "misc", + "Sched", + "guy", + "linger", + "warm", + "ipel", + "Gork", + "dispatcher", + "315", + "finely", + "reliably", + "rupt", + "negligent", + "endorsements", + "Orient", + "electro", + "haired", + "physique", + "wine", + "adolescents", + "184", + "alth", + "validated", + "izzard", + "Peck", + "emblem", + "status", + "Jungle", + "orius", + "eccentric", + "folding", + "poor", + "THC", + "appers", + "scripted", + "239", + "Preferred", + "digital", + "sharper", + "portrays", + "rative", + "238", + "183", + "uneasy", + "RI", + "vil", + "171", + "spoil", + "Pricing", + "Hardware", + "188", + "horrendous", + "ostensibly", + "nah", + "gadget", + "ADS", + "coat", + "exhausting", + "draining", + "arate", + "Bulgarian", + "emo", + "hier", + "guitars", + "ieties", + "assed", + "Yaz", + "aggress", + "BG", + "vik", + "neatly", + "pixel", + "intimacy", + "Rug", + "512", + "narrated", + "mast", + "Nos", + "Hung", + "reciation", + "Chandra", + "bios", + "Ended", + "lique", + "Cambod", + "worrisome", + "EQ", + "novelist", + "Dynamic", + "MIC", + "disposed", + "brackets", + "haircut", + "Lana", + "lull", + "billboard", + "Reverend", + "NAV", + "borgh", + "adrenaline", + "seeming", + "PCB", + "Bridgewater", + "squirrel", + "262", + "write", + "stabilization", + "wild", + "secession", + "packet", + "AMES", + "licted", + "malnutrition", + "claimed", + "charred", + "tragically", + "Published", + "repealed", + "Sawyer", + "Mormon", + "resolution", + "Saud", + "Henry", + "discontin", + "snag", + "danger", + "mixes", + "upbringing", + "limb", + "Fantastic", + "Sim", + "Augustine", + "Greeks", + "cod", + "Historically", + "mire", + "register", + "Kund", + "debilitating", + "Chat", + "Tau", + "ï", + "lower", + "pie", + "430", + "nascent", + "375", + "bum", + "WI", + "Netflix", + "whether", + "dearly", + "eff", + "PRES", + "landmarks", + "culminating", + "migrate", + "balanced", + "regulars", + "modification", + "dips", + "Redmond", + "ationally", + "atsu", + "philosophical", + "typing", + "unreal", + "boiled", + "blight", + "dru", + "Gaddafi", + "nour", + "sequential", + "augment", + "Euras", + "Wiley", + "endar", + "acronym", + "esteem", + "Majesty", + "grips", + "obsolete", + "nos", + "Made", + "ogie", + "Liver", + "Donetsk", + "dynam", + "tel", + "bring", + "knit", + "firepower", + "prepaid", + "Raphael", + "sensing", + "720", + "WN", + "Nor", + "puted", + "bureaucrats", + "Adjust", + "intensely", + "sunscreen", + "Ho", + "Yelp", + "PU", + "Serge", + "Cyp", + "ELF", + "Guns", + "teamwork", + "Bib", + "Maintenance", + "perate", + "wiping", + "charcoal", + "ordan", + "International", + "behaving", + "softened", + "Increased", + "unfl", + "470", + "informative", + "novelty", + "avoidance", + "teasing", + "matic", + "maid", + "Pell", + "counterterrorism", + "Gabe", + "ications", + "Connection", + "Inquiry", + "isin", + "orama", + "corpse", + "practitioner", + "itto", + "UA", + "forestry", + "lic", + "revolves", + "calculating", + "puppet", + "ulously", + "Pebble", + "Dep", + "upholding", + "carving", + "wartime", + "envy", + "encro", + "Punk", + "Administ", + "ucha", + "battleground", + "lol", + "uable", + "unheard", + "Spur", + "phony", + "carc", + "Sut", + "pollutants", + "Cr", + "vigorous", + "355", + "Marriage", + "staffed", + "fecture", + "Arabs", + "supported", + "manpower", + "Satellite", + "None", + "queues", + "insightful", + "interchange", + "Rel", + "solemn", + "smuggled", + "upt", + "171", + "parallels", + "intelligence", + "punk", + "recycle", + "decorative", + "shar", + "arrell", + "iances", + "Bolivia", + "strengthens", + "430", + "hardships", + "signalling", + "unthinkable", + "READ", + "tad", + "picked", + "armor", + "cores", + "Matrix", + "dj", + "evolutionary", + "Bermuda", + "OE", + "organized", + "relentlessly", + "sol", + "Mamm", + "pounding", + "Weather", + "rab", + "sweets", + "funding", + "HUD", + "Soldier", + "reed", + "released", + "containment", + "alid", + "Nikon", + "cervical", + "ign", + "alias", + "optimized", + "asserting", + "AFTER", + "flatt", + "dinosaur", + "Refugees", + "Anch", + "adjustable", + "roaring", + "pilgrimage", + "cowboy", + "entails", + "ractions", + "EY", + "undy", + "Kuh", + "inges", + "Terra", + "Escape", + "rundown", + "striped", + "KN", + "ocations", + "IDENT", + "IGH", + "avoids", + "Moh", + "LS", + "lbs", + "Attempt", + "triangle", + "climax", + "hp", + "allot", + "learning", + "JFK", + "Justice", + "OUT", + "HER", + "Lect", + "trench", + "edar", + "reservoirs", + "uid", + "rf", + "162", + "interfered", + "emit", + "these", + "444", + "Leather", + "essing", + "Eighth", + "uckle", + "Breaking", + "unresolved", + "goose", + "252", + "platform", + "atus", + "complexion", + "BUS", + "struct", + "middle", + "Sat", + "WHERE", + "LB", + "redible", + "vered", + "Louis", + "Baz", + "Eye", + "safety", + "hypothetical", + "bowel", + "untouched", + "312", + "Pric", + "astounding", + "meet", + "Aaron", + "Woo", + "236", + "Shape", + "drifted", + "tile", + "Grim", + "undeniable", + "..", + "radius", + "ovarian", + "Seriously", + "verning", + "assertions", + "oxic", + "231", + "Viz", + "Jackson", + "Sno", + "boycot", + "okingly", + "ousse", + "proclaimed", + "blazing", + "inefficient", + "fig", + "booze", + "259", + "agus", + "statement", + "locom", + "tacos", + "memos", + "gender", + "Ort", + "263", + "intervening", + "Soc", + "University", + "Pis", + "Returns", + "PAN", + "ultrasound", + "coherent", + "tracking", + "rieved", + "383", + "qualitative", + "uld", + "Giovanni", + "storylines", + "darkest", + "velvet", + "RIP", + "compatibility", + "troll", + "CN", + "Found", + "Ou", + "tease", + "vested", + "provocation", + "improvised", + "activation", + "unte", + "Monteneg", + "JOHN", + "React", + "polluted", + "217", + "mushroom", + "disconnected", + "Voices", + "asu", + "sensory", + "REE", + "monarchy", + "173", + "doing", + "involved", + "Jonah", + "toxins", + "tv", + "academia", + "IQ", + "Mor", + "Straight", + "RN", + "âĹı", + "pear", + "187", + "endeavors", + "Turbo", + "ducks", + "Ramsay", + "outpatient", + "comprehend", + "UNE", + "briefings", + "total", + "migr", + "always", + "moot", + "Rider", + "biblical", + "Form", + "curry", + "exquisite", + "385", + "244", + "attendants", + "cabinets", + "nton", + "Baby", + "Honestly", + "FIRE", + "211", + "itech", + "Prosper", + "chops", + "odic", + "Rod", + "job", + "orset", + "Ary", + "obic", + "Nil", + "isable", + "orche", + "trivial", + "Zy", + "XP", + "endorsing", + "LIM", + "adish", + "237", + "Laws", + "heid", + "Signature", + "Vern", + "Bland", + "ansk", + "repository", + "Petra", + "Enter", + "truths", + "bordering", + "penn", + "simplified", + "zn", + "Cree", + "181", + "Hi", + "Greenberg", + "prematurely", + "Sass", + "wrecked", + "heinous", + "415", + "Turn", + "zl", + "amental", + "Braz", + "fing", + "Angle", + "Phantom", + "agra", + "Shack", + "homegrown", + "alright", + "AME", + "KN", + "clicks", + "manned", + "Scope", + "extras", + "clinicians", + "321", + "African", + "juices", + "refere", + "****", + "ambling", + "since", + "voic", + "QB", + "Atmospheric", + "Mat", + "perpetrated", + "Steps", + "Fit", + "silenced", + "bonded", + "quantify", + "Houston", + "ocracy", + "freeing", + "pipe", + "corn", + "rones", + "ooked", + "Suz", + "unaccount", + "196", + "logos", + "Furious", + "Spart", + "urst", + "itri", + "Zub", + "Actual", + "slee", + "gag", + "metabolism", + "Designed", + "pedigree", + "coolest", + "âĿ", + "iuses", + "Yellowstone", + "informant", + "ushered", + "Garg", + "thel", + "Hop", + "repetitive", + "flag", + "unmarked", + "Brave", + "incur", + "reading", + "ppel", + "lah", + "ateurs", + "286", + "Atomic", + "appliance", + ")'", + "traditional", + "dads", + "regimen", + "infrared", + "dotted", + "tails", + "horrors", + "uments", + "dub", + "lighting", + "unearthed", + "assisted", + "Spiel", + "trial", + "persever", + "MAX", + "icing", + "Energy", + "1943", + "move", + "Error", + "liter", + "Cly", + "Ari", + "granite", + "cropped", + "RD", + "REM", + "TX", + "displeasure", + "Comfort", + "unsettling", + "scratching", + "866", + "eton", + "560", + "commonplace", + "reproduced", + "ggie", + "schooling", + "reprim", + "darling", + "huge", + "Dante", + "cp", + "heastern", + "educ", + "Digital", + "wrath", + "watering", + "Tail", + "degradation", + "530", + "usive", + "Xu", + "AH", + "classy", + "SET", + "criminally", + "dependent", + "Alps", + "notwithstanding", + "familiarity", + "APP", + "aurus", + "gments", + "Mid", + "epilepsy", + "resemblance", + "brush", + "333", + "liberated", + "Beng", + "Lans", + "traff", + "ihu", + "establish", + "cort", + "Rick", + "plugged", + "onement", + "Accounting", + "reconstruct", + "Pop", + "incapable", + "aho", + "Dexter", + "pitted", + "bathing", + "dun", + "explor", + "Midnight", + "activ", + "iann", + "likely", + "acons", + "owicz", + "negativity", + "freel", + "ewitness", + "inj", + "Stephen", + "shredded", + "prepar", + "Script", + "correctional", + "commits", + "hai", + "activity", + "Imp", + "stumble", + "cache", + "Promise", + "precinct", + "multicultural", + "substitutes", + "shortened", + "ovable", + "fasting", + "infused", + "bulldo", + "alm", + "adjoining", + "multiplayer", + "Alien", + "pund", + "ethyl", + "bliss", + "Decision", + "bab", + "angrily", + "another", + "oled", + "ainted", + "Priest", + "draped", + "Personally", + "stomp", + "Wolfgang", + "oste", + "itches", + "hoops", + "JO", + "sche", + "Zan", + "cleans", + "climbs", + "electronically", + "243", + "ocy", + "gall", + "REAL", + "murky", + "modernization", + "tub", + "Really", + "lax", + "doubted", + "yden", + "Prevent", + "UTERS", + "override", + "SAF", + "coun", + "excerpts", + "motivations", + "decency", + "astronomers", + "orical", + "altering", + "232", + "described", + "omic", + "exh", + "knocks", + "Riot", + "Purs", + "equal", + "pleting", + "llan", + "SOL", + "iator", + "ILE", + "WM", + "defences", + "forearm", + "Toronto", + "526", + "acne", + "thirteen", + "itiz", + "akable", + "charges", + "inaction", + "bred", + "deficiency", + "intrigue", + "opoly", + "Camer", + "Melt", + "unlawfully", + "penetrate", + "Used", + "Dirty", + "excerpt", + "Yen", + "CARD", + "cher", + "Challenges", + "ieves", + "ambush", + "Data", + "eeks", + "giveaway", + "pawn", + "transf", + "renched", + "moderately", + "numbered", + "Integrity", + "HOU", + "HDMI", + "Royal", + "LT", + "Dirk", + "izon", + "227", + "disagrees", + "Ninth", + "increment", + "Glory", + "suff", + "artery", + "Employee", + "bum", + "Editorial", + "Kh", + "Premiere", + "Weld", + "Included", + "mathematical", + "exponentially", + "handwritten", + "MAS", + "indiscrim", + "nutrient", + "Selection", + "219", + "hyd", + "deton", + "æ", + "dark", + "Fidel", + "monkeys", + "nutritious", + "headlights", + "oller", + "piring", + "Defenders", + "drown", + "elong", + "floats", + "graduate", + "prosper", + "Named", + "Eating", + "ECK", + "establishment", + "XM", + "soaking", + "278", + "listener", + "simultaneous", + "olutions", + "payer", + "customize", + "ROCK", + "altar", + "Exercise", + "anky", + "Profession", + "sever", + "Merchant", + "RF", + "Combat", + "legality", + "fledged", + "diapers", + "lves", + "lur", + "ignores", + "Protocol", + "representations", + "Blumenthal", + "Lime", + "romptu", + "besieged", + "dl", + "sighting", + "Parm", + "Server", + "Benghazi", + "estival", + "playlist", + "Ung", + "Quantum", + "compromises", + "Survivor", + "Mobility", + "bounty", + "ophers", + "ISA", + "need", + "uese", + "orn", + "218", + "530", + "buddies", + "agendas", + "Feldman", + "Ãĸ", + "BMC", + "Serve", + "Ent", + "KH", + "INT", + "littered", + "visitation", + "mist", + "dupl", + "routed", + "Amount", + "Dev", + "Conv", + "slams", + "Veterinary", + "bold", + "186", + "DOT", + "builder", + "decay", + "Hemp", + "pelled", + "mankind", + "Tonight", + "effortlessly", + "BUT", + "hostilities", + "formerly", + "alon", + "Crash", + "humane", + "mayhem", + "Budd", + "disinformation", + "226", + "prototypes", + "__", + "IVERS", + "izzy", + "Might", + "Pip", + "pour", + "INO", + "LL", + "wiret", + "resorted", + "Tanaka", + "DOES", + "Earlier", + "HO", + "moniker", + "Fang", + "Hua", + "bered", + "adding", + "194", + "STR", + ".\")", + "cop", + "Flags", + "Colleges", + "Uz", + "sparks", + "paradox", + "Marie", + "Strong", + "strawberry", + "nurturing", + "fax", + "Tor", + "killer", + "burse", + "attachments", + "pup", + "exhaustion", + "whisky", + "isu", + "ologically", + "iership", + "lamps", + "shuff", + "centralized", + "Needless", + "grenade", + "router", + "optics", + "ivering", + "pioneers", + "Hug", + "handguns", + "010", + "bailed", + "uana", + "197", + "distorted", + "Essentially", + "Silent", + "comparative", + "Music", + "MUS", + "Bur", + "Comet", + "Winchester", + "IGN", + "Mod", + "Candidate", + "dysfunctional", + "Celeb", + "hitch", + "api", + "idiot", + "unsupported", + "gat", + "inker", + "redevelop", + "dwind", + "forgetting", + "Rost", + "remembrance", + "Na", + "mopolitan", + "berries", + "marital", + "Vol", + "Closing", + "Hindus", + "itism", + "rover", + "mysteries", + "Nig", + "ucing", + "fabrication", + "garments", + "wield", + "Compton", + "357", + "oxide", + "chron", + "Thought", + "comed", + "Epstein", + "BART", + "orative", + "Kahn", + "adan", + "APH", + "cum", + "loophole", + "GoPro", + "osit", + "specification", + "APR", + "drains", + "conserve", + "Morse", + "calorie", + "Cheney", + "station", + "evangel", + "spraying", + "lections", + "enclosure", + "commanded", + "Organizations", + "imb", + "mins", + "Tobias", + "Ve", + "Nau", + "183", + "Guantanamo", + "173", + "requisite", + "derivative", + "populism", + "cultivated", + "lord", + "uler", + "DEA", + "inally", + "demonstr", + "trip", + "Firefox", + "246", + "confirmed", + "Anne", + "tamp", + "Household", + "amous", + "Meet", + "dashed", + "pire", + "inex", + "loosen", + "272", + "famous", + "Heard", + "hindsight", + "depot", + "Cutting", + "Mouse", + "geological", + "number", + "OUN", + ".,\"", + "moderation", + "UNHCR", + "domains", + "eco", + "crater", + "510", + "kid", + "cylinders", + "Classes", + "Kn", + "carcin", + "Hunting", + "irit", + "ARP", + "anting", + "Marino", + "RESP", + "ifle", + "239", + "fman", + "theoretically", + "distraught", + "staircase", + "expel", + "lord", + "behaviours", + "prescribing", + "ographs", + "Newly", + "patiently", + "skyline", + "udos", + "repertoire", + "hover", + "mint", + "clears", + "kale", + "Sco", + "Coulter", + "pancreat", + "pu", + "995", + "incompetent", + "2007", + "gripping", + "enable", + "reinforcing", + "Fee", + "education", + "Kuro", + "bowed", + "shave", + "Mean", + "xi", + "inciting", + "atters", + "ecstatic", + "hog", + "clauses", + "subt", + "behaved", + "tains", + "Liverpool", + "strives", + "Kev", + "Framework", + "defined", + "recounts", + "array", + "tips", + "artificially", + "fits", + "Clearly", + "mediate", + "unseen", + "thugs", + "Lent", + "1938", + "genital", + "Sonic", + "Warehouse", + "pler", + "unm", + "packets", + "MET", + "ealous", + "ographers", + "labou", + "Core", + "+,", + "parable", + "strat", + "invitations", + "souven", + "billboards", + "Regulations", + "dwarf", + "toler", + "prose", + "estates", + "metabolic", + "Suff", + "Firstly", + "polio", + "chick", + "Daughter", + "substant", + "Identity", + "umbers", + "Facts", + "frust", + "dissip", + "Deck", + "Hy", + "Birch", + "hurled", + "democracy", + "nered", + "eper", + "cerebral", + "181", + "halves", + "abit", + "balance", + "Tibet", + "handheld", + "Dough", + "programmed", + "hw", + "outlawed", + "Serious", + "ironically", + "manipulating", + ")\"", + "juries", + "fragrance", + "crete", + "HHS", + "cience", + "cosmic", + "foreclosure", + "percentages", + "Bus", + "enticing", + "extra", + "Shy", + "Â¥", + "headsets", + "imensional", + "lux", + "residual", + "mantle", + "SJ", + "Peaks", + "Finger", + "unfolds", + "anity", + "resettlement", + "Weak", + "Been", + "198", + "angels", + "Farn", + "peace", + "capac", + "hue", + "lust", + "traumatic", + "laun", + "strawberries", + "herbal", + "conversions", + "Held", + "prescribe", + "Its", + "Dartmouth", + "fashioned", + "460", + "BLE", + "international", + "lumin", + "plantation", + "ilde", + "490", + "euph", + "disgust", + "aspire", + "medical", + "socialism", + "dissolve", + "Wal", + "admittedly", + "sewing", + "Acer", + "tul", + "facilit", + "grandma", + "Feeling", + "obst", + "Franz", + "Palin", + "Increase", + "gets", + "Imam", + "âĢİ", + "coincides", + "urrence", + "lifes", + "Lab", + "Ham", + "angelo", + "Wild", + "vetoed", + "ventilation", + "olid", + "Summer", + "facade", + "neys", + "WOM", + "Benny", + "Married", + "squ", + "Reflect", + "return", + "elia", + "olding", + "refine", + "Madness", + "innacle", + "posts", + "287", + "fruit", + "274", + "icator", + "Voy", + "unsett", + "fant", + "treaties", + "crystals", + "hijacked", + "words", + "Released", + "Save", + "cannon", + "anomaly", + "beacon", + "crippled", + "bundles", + "untreated", + "happiest", + "galaxies", + "occupational", + "416", + "Dar", + "crank", + "appropriation", + "asking", + "mens", + "detector", + "skewed", + "poke", + "254", + "hypertension", + "apolog", + "evaluations", + "blocks", + "pow", + "GEN", + "scalp", + "arrogant", + "AIDS", + "ority", + "redirect", + "derogatory", + "lateral", + "495", + "rolley", + "brew", + "babys", + "muff", + "Requ", + "dime", + "wonderfully", + "treasures", + "NES", + "ponds", + "impulse", + "detecting", + "grin", + "brid", + "shoved", + "purge", + "irteen", + "OTHER", + "ÙĦ", + "irsch", + "Occ", + "193", + "fodder", + "wrote", + "meric", + "posal", + "winters", + "Juice", + "hub", + "contrasting", + "Brazil", + "flashy", + "uffer", + "technology", + "Children", + "catapult", + "owsky", + "Eclipse", + "abeth", + "Particip", + "laud", + "Quiet", + "simulations", + "sacrificing", + "preaching", + "voicing", + "itizen", + "gn", + "sans", + "285", + "Robot", + "1936", + "sham", + "Kislyak", + "GCC", + "tale", + "Shades", + "sediment", + "conveniently", + "Give", + "mounted", + "peel", + "Jun", + "Eisenhower", + "diplom", + "Preservation", + "affirm", + "taboo", + "Garr", + "Apply", + "prim", + "ausp", + "textbook", + "forfeit", + "icides", + "undis", + "DJ", + "\"...", + "Xperia", + "furry", + "Australian", + "preach", + "paramed", + "196", + "agos", + "RIP", + "408", + "Quarterly", + "Quentin", + "deft", + "Vlad", + "massive", + "apore", + "questionnaire", + "secution", + "Tunnel", + "Assist", + "BILITY", + "everything", + "vich", + "comparatively", + "heng", + "ETH", + "iPod", + "insurgent", + "testosterone", + "191", + "moons", + "gripped", + "strang", + "pects", + "SERVICE", + "numb", + "measurable", + "dismantled", + "depict", + "retake", + "Light", + "aquatic", + "useum", + "judicial", + "****", + "rosters", + "certain", + "hypothesis", + "2002", + "Snow", + "pounded", + "Zel", + "Trem", + "iversity", + "219", + "Jen", + "Adventures", + "cylinder", + "banging", + "balk", + "analy", + "Hust", + "ookie", + "Returning", + "pods", + "analysis", + "Truman", + "org", + "sar", + "dred", + "Telecommunications", + "Sven", + "carry", + "LOVE", + "parting", + "asar", + "utations", + "itic", + "actu", + "bananas", + "Nights", + "410", + "Still", + "tweaked", + "went", + "toddlers", + "irted", + "paed", + "Wink", + "viewpoint", + "Helic", + "handshake", + "poaching", + "rounding", + "268", + "NVIDIA", + "squat", + "towed", + "handler", + "conspir", + "additionally", + "CENT", + "Ãľ", + "article", + "Tough", + "NM", + "Rem", + "stunts", + "ILS", + "LM", + "Connect", + "Paragu", + "complexities", + "hugging", + "abolish", + "ricting", + "Items", + "temples", + "Seat", + "Rubber", + "indic", + "Vitamin", + "citations", + "armored", + "---------------", + "Neo", + "ippy", + "Que", + "rag", + "lov", + "630", + "adept", + "orbit", + "253", + "412", + "butterflies", + "outl", + "Cycle", + "aesthetics", + "Twitch", + "405", + "factor", + "ðŁij", + "Circus", + "Posted", + "introductory", + "Stack", + "atoes", + "furn", + "Hond", + "bipolar", + "Aging", + "inches", + "incompetence", + "aloud", + "Imagine", + "separ", + "manip", + "ophobic", + "inion", + "bek", + "quer", + "Armen", + "humorous", + "mundane", + "apologizing", + "pioneered", + "303", + "282", + "calming", + "orious", + "760", + "stitches", + "throttle", + "spinach", + "urities", + "Cologne", + "ripple", + "Cs", + "Cent", + "Should", + "affinity", + "amount", + "MISS", + "sage", + "amusing", + "snatch", + "clair", + "Guess", + "bench", + "Moj", + "nuclear", + "fid", + "VM", + "GN", + "brainer", + "curled", + "bushes", + "icably", + "creeping", + "veil", + "ALS", + "ESPN", + "ulsion", + "GTX", + "ANN", + "complicit", + "assault", + "IOR", + "polymer", + "estimating", + "277", + "alog", + "glimps", + "reinforces", + "textbooks", + "dictated", + "Reyn", + "latable", + "Orth", + "520", + "trickle", + "Wrong", + ".[", + "Designer", + "304", + "Inner", + "rave", + "ppa", + "Gim", + "swath", + "carts", + "atlantic", + "persists", + "Developer", + "goodies", + "isive", + "Inf", + "Saving", + "loop", + "tions", + "abusers", + "clot", + "mesmer", + "deg", + "skirts", + "257", + "unreliable", + "COMM", + "194", + "fledgling", + "administ", + "Israeli", + "Barbie", + "Jeanne", + "generously", + "Struct", + "Zap", + "vetted", + "Violet", + "),", + "embarrass", + "bang", + "Provider", + "getting", + "alg", + "unconditional", + "Hulk", + "Wad", + "utation", + "pointless", + "deprivation", + "starving", + "Impossible", + "Stir", + "knack", + "anse", + "securely", + "ply", + "395", + "Pack", + "liv", + "ridden", + "alks", + "308", + "male", + "bitterly", + "irrational", + "Members", + "ported", + "qq", + "ractor", + "inflict", + "Boehner", + "thickness", + "dome", + "Influ", + "heap", + "mirrored", + "constituent", + "fertile", + "vaping", + "266", + "riages", + "embassies", + "persu", + "MacArthur", + "issions", + "Main", + "aths", + "onne", + "circ", + "sweating", + "quartered", + "sax", + "540", + "reputable", + "satire", + "pastors", + "ventional", + "Mic", + "female", + "pity", + "appropri", + "voc", + "hei", + "imperial", + "corrective", + "resent", + "tempered", + "differs", + "Hamilton", + "saddle", + "grenades", + "Quart", + "onymous", + "til", + "depiction", + "disreg", + "petitioner", + "fret", + "Ens", + "Emer", + "540", + "opathy", + "vertisements", + "sketches", + "venth", + "automate", + "jihad", + "iping", + "tert", + "Sop", + "ships", + "deceptive", + "Pryor", + "Gorge", + "Meridian", + "rero", + "affected", + "lame", + "660", + "rub", + "Hello", + "Numbers", + "269", + "marg", + "Fran", + "640", + "cath", + "winter", + "Mosque", + "reckoning", + "Imaging", + "mutation", + "Mild", + "kidnap", + "nav", + "ferocious", + "dusty", + "Cele", + "Foss", + "regrett", + "lymp", + "coli", + "stereo", + "foresee", + "alties", + "resusc", + "Full", + "wash", + "INST", + "Pars", + "coated", + "HT", + "discord", + "reforming", + "CAN", + "blink", + "lubric", + "mishand", + "ensible", + "existent", + "secondary", + "Doesn", + "terrorist", + "riff", + "custom", + "DET", + "reusable", + "CRA", + "Scalia", + "accelerator", + "propag", + "MID", + "ework", + "looted", + "oscope", + "eners", + "ruction", + "barr", + "viewership", + "lends", + "obil", + "Roots", + "Came", + "ibel", + "globalization", + "lab", + "information", + "coordin", + "glitch", + "worms", + "slurs", + "contemplated", + "Penal", + "191", + "221", + "exposes", + "248", + "ASP", + "dependency", + "urga", + "pdf", + "vibr", + "clone", + "ossible", + "Utt", + "serv", + "Levant", + "maybe", + "MU", + "Lunar", + "bystanders", + "capitals", + "preacher", + "thin", + "underscore", + "('", + "medd", + "autobiography", + "persistence", + "arming", + "appalled", + "contradictory", + "reciproc", + "takedown", + "tan", + "necessities", + "itans", + "Alas", + "segregated", + "Responsibility", + "SHOW", + "ISIS", + "pengu", + "umb", + "HO", + "HB", + "Chou", + "alluded", + "harms", + "bara", + "WOR", + "Sorry", + "starvation", + "spilling", + "carb", + "annis", + "Garrison", + "millionaire", + "ifling", + "Cancel", + "imprint", + "borrower", + "455", + "Cic", + "exposures", + "dest", + "unn", + "802", + "adherence", + "prints", + "weary", + "waging", + "1937", + "Kepler", + "%;", + "defective", + "Reps", + "Granted", + "disco", + "Ranking", + "erno", + "archaeological", + "sq", + "capit", + "fleets", + "inventor", + "iffin", + "spotting", + "SHARES", + "309", + "Hard", + "save", + "241", + "Thinking", + "XY", + "havens", + "messed", + "crop", + "perme", + "timelines", + "Garage", + "plateau", + "together", + "fox", + "failings", + "Tight", + "Physics", + "Scholars", + "pans", + "Fall", + "hull", + "GER", + "bourbon", + "ceived", + "steroids", + "hamb", + "interpretations", + "cush", + "Chair", + "informational", + "aryn", + "woven", + "amen", + "Bre", + "refreshed", + "York", + "Blast", + "Editor", + "motivating", + "Reason", + "Florida", + "dreaded", + "stationary", + "bil", + "doors", + "slightest", + "combustion", + "fascination", + "straps", + "scribed", + "exhibiting", + "simplest", + "Gar", + "progressives", + "claim", + "ocket", + "exoner", + "NETWORK", + "Brad", + "197", + "nightmares", + "illust", + "among", + "Greenpeace", + "oval", + "blocker", + "3000", + "Memor", + "mids", + "confuse", + "YN", + "cow", + "dispensary", + "telling", + "entail", + "neurolog", + "broth", + "pron", + "Answer", + "thank", + "intersect", + "clinging", + "Killing", + "cohesion", + "categorized", + "tangled", + "ASC", + "Arsenal", + "Automatic", + "580", + "sac", + "shady", + "consumer", + "hetically", + "NV", + "overl", + "holes", + "Donation", + "tera", + "score", + "library", + "smoother", + "coasts", + "intercourse", + "unfavorable", + "erb", + "Hel", + "biases", + "inheritance", + "suppressed", + "Recommend", + "iculture", + "ighting", + "inguished", + "idences", + "operated", + "hors", + "shrug", + "aila", + "Consortium", + "veins", + "uria", + "Smithsonian", + "AX", + ")âĢĶ", + "given", + "JC", + "reneg", + "princip", + "extinct", + "Golden", + "ASON", + "statutes", + "292", + "GOOD", + "Greenland", + "Rasmussen", + "ATHER", + "deserted", + "Hitchcock", + "qualifies", + "dreadful", + "supers", + "tendon", + "oter", + "Fate", + "restrooms", + "igating", + "Sher", + "Name", + "orph", + "Critical", + "rox", + "defunct", + "canoe", + "biscuits", + "womb", + "808", + "istar", + "roar", + "aundering", + "iewicz", + "NM", + "Chamberlain", + "233", + "Coat", + "999", + "aft", + "lurking", + "Pist", + "follower", + "careg", + "ÙĨ", + "Thin", + "ZZ", + "GI", + "Vintage", + "painstaking", + "gloom", + "tbsp", + "whim", + "Mask", + "rugged", + "writings", + "stantial", + "luence", + "ordable", + "akia", + "assassinated", + "Wind", + "demeanor", + "Night", + "rape", + "Bringing", + "shields", + "Antarctic", + "fruitful", + "Buster", + "Lois", + "302", + "Style", + "RIS", + "dissatisfaction", + "ulp", + "Laser", + "disposition", + "Ank", + "absorbing", + "276", + "volcan", + "leftover", + "yah", + "Vaj", + "unsolved", + "oland", + "stained", + "pathetic", + "ylan", + "knots", + "immigration", + "ieving", + "Coming", + "Commerce", + "Hurt", + "drawn", + "axis", + "dye", + "Nora", + "Portal", + "suspense", + "Exactly", + "powering", + "Clock", + "drawer", + "Spike", + "hallmark", + "aber", + "Trainer", + "UV", + "redundant", + "Tour", + "designate", + "redress", + "Ub", + "cake", + "oded", + "kings", + "iates", + "coupons", + "extremes", + "Elect", + "citation", + "directory", + "transpired", + "cele", + "gence", + "5000", + "ostic", + "raining", + "Sight", + "videos", + "phthal", + "llor", + "appraisal", + "detox", + "electing", + "ordinances", + "lifespan", + "Ref", + "illuminated", + "forfe", + "Making", + "Worst", + "TP", + "fullest", + "ISIL", + "Rates", + "yeast", + "sett", + "Yok", + "innie", + "edition", + "Goldstein", + "unaff", + "god", + "zo", + "rums", + "opaque", + "Hist", + "Yesterday", + "AMS", + "aband", + "005", + "illary", + "Splash", + "accrued", + "Ell", + "nominating", + "Broadcast", + "Whip", + "ARM", + "unnecessarily", + "brown", + "429", + "ansky", + "extravagant", + "Malley", + "wage", + "exempted", + "typo", + "esports", + "Stru", + "Python", + "saint", + "CSI", + "Powder", + "disguised", + "Subway", + "precursor", + "Wizard", + "Johnson", + "icas", + "defaults", + "!).", + "ebra", + "jected", + "unaccompanied", + "HH", + "proced", + "clinical", + "mitigating", + "Soup", + "Funny", + "344", + "Hall", + "scalable", + "shimmer", + "understatement", + "zeb", + "icus", + "retract", + "IDER", + "ieft", + "iii", + "Emperor", + "voltage", + "343", + "Rest", + "Butcher", + "laced", + "salty", + "fourteen", + "oxy", + "raged", + "forg", + "caveat", + "ponder", + "process", + "ghosts", + "Goose", + "didn", + "stood", + "amation", + "villains", + "contract", + "booted", + "Didn", + "Salon", + "lewd", + "Fritz", + "organis", + "puzzles", + "RX", + "curtains", + "Package", + "rebate", + "spokes", + "occupant", + "fooled", + "appy", + "yourselves", + "maths", + "630", + "bos", + "Heb", + "APS", + "bulletin", + "pests", + "lum", + "HAS", + "users", + "idated", + "palpable", + "Feature", + "PKK", + "detriment", + "bamboo", + "immersed", + "Dud", + "ion", + "icc", + "Iris", + "Beats", + "improbable", + "funer", + "sprung", + "Lieberman", + "STA", + "venge", + "treacherous", + "preced", + "sniper", + "GOLD", + "SUR", + "Nic", + "ROB", + "Camp", + "hooks", + "oling", + "bolst", + "339", + "heter", + "bracelet", + "breat", + "307", + "Trader", + "Pixar", + "hist", + "menacing", + "grizz", + "294", + "illustrious", + "transact", + "spoiler", + "WORK", + "Road", + "blackout", + "encomp", + "proven", + "Friendship", + "entrances", + "professions", + "insin", + "recorder", + "formulation", + "govern", + "painfully", + "Repe", + "eeds", + "cru", + "Dir", + "triumphant", + "ignition", + "xy", + "intrusion", + "EAR", + "RES", + "ration", + "Taken", + "cages", + "peg", + "commem", + "680", + "Rite", + "folder", + "vertically", + "cheeks", + "pick", + "crispy", + "squeezing", + "Bene", + "Trailer", + "KM", + "acceptable", + "Setting", + "supernatural", + "Ez", + "venom", + "Frey", + "pulp", + "Had", + "centered", + "metics", + "Kent", + "DOI", + "kr", + "WHEN", + "takeoff", + "isf", + "uko", + "quasi", + "veggies", + "pesticide", + "stimulating", + "acknowledgement", + "attained", + "Background", + "281", + "317", + "Trees", + "detractors", + "announcer", + "joyful", + "Elf", + "istration", + "phi", + "progressively", + "mini", + "contraception", + "asca", + "ishops", + "misunderstood", + "initiating", + "Conversely", + "338", + "080", + "idation", + "Goes", + "improv", + "swapping", + "Vict", + "devoid", + "fighter", + "Mori", + "voy", + "Elev", + "Aim", + "trustworthy", + "Leg", + "675", + "Possible", + "Crunch", + "Rings", + "phony", + "bladder", + "Chall", + "Spot", + "oak", + "Was", + "FAM", + "AGA", + "Fifa", + "enclosed", + "anthrop", + "faith", + "Aux", + "gracious", + "roller", + "downtime", + "swing", + "camouflage", + "Costs", + "liv", + "ricular", + "Uran", + "disapproval", + "propriet", + "bits", + "mafia", + "SCHOOL", + "Prepar", + "button", + "Almost", + "pastoral", + "Dove", + "Hol", + "imposes", + "Dram", + "lys", + "SAS", + "wiring", + "271", + "Models", + "outpost", + "etics", + "insulted", + "Mongolia", + "overth", + "Haw", + "Homer", + "itta", + "raining", + "evidently", + "raphic", + "impact", + "franch", + "2100", + "approximate", + "cartoons", + "backups", + "umbing", + "forceful", + "Shad", + "surges", + "perf", + "dele", + "quieter", + "Horowitz", + "DX", + "anners", + "Ninja", + "Script", + "Elise", + "collect", + "grading", + "Bethesda", + "Kids", + "Telephone", + "preferring", + "reconcil", + "mango", + "Hail", + "Citizenship", + "Master", + "cular", + "stuffing", + "Alive", + "ALLY", + "chi", + "Dynam", + "Rosenthal", + "purity", + "temp", + "HAL", + "employ", + "plentiful", + "Comed", + "stacks", + "Huge", + "Older", + "sclerosis", + "ONY", + "filmmaking", + "chance", + "Cry", + "workflow", + "Personnel", + "awed", + "Column", + "uncomp", + "discriminated", + "pts", + "allev", + "Kinn", + "meal", + "novice", + "crest", + "hearty", + "lowers", + "inqu", + "Playoffs", + "Hyp", + "autos", + "indec", + "nighttime", + "reflex", + "306", + "disciplinary", + "ophe", + "contact", + "achievable", + "slab", + "Message", + "VMware", + "Dia", + "REG", + "confisc", + "Mechan", + "phenomena", + "sequencing", + "shaming", + "compilation", + "Ages", + "mastered", + "agony", + "restrain", + "Lyme", + "Which", + "Barney", + "Concept", + "superheroes", + "Psychology", + "reminis", + "violence", + "Lead", + "Da", + "VEN", + "ERC", + "Voter", + "betray", + "savage", + "driver", + "IFT", + "Chain", + "angler", + "'-", + "lain", + "Ratt", + "bis", + "iverse", + "densely", + "uncom", + "unsuspecting", + "stimulation", + "diff", + "skins", + "Riding", + "ategic", + "Understand", + "occup", + "Cooking", + "schizophrenia", + "Koen", + "comrades", + "HY", + "fab", + "Rowling", + "Allen", + "JUL", + "embryos", + "UU", + "CAT", + "tidy", + "finger", + "Cake", + "rightfully", + "religious", + "407", + "Gal", + "408", + "grievance", + "swallowed", + "251", + "283", + "Barcl", + "opter", + "pedoph", + "cured", + "establishes", + "increasing", + "tics", + "articles", + "unethical", + "authored", + "anchors", + "Contra", + "ventured", + "Coh", + "puff", + "heddar", + "omission", + "dich", + "ceed", + "scares", + "doctoral", + "293", + "Unt", + "dop", + "Injury", + "ificantly", + "Rift", + "Orders", + "mobilize", + "particularly", + "chilled", + "Reports", + "redibly", + "Guru", + "valleys", + "textures", + "reuse", + "roit", + "unts", + "irreversible", + "warships", + "pus", + "peeled", + "thirst", + "grapple", + "busters", + "nort", + "Dates", + "Safe", + "birthplace", + "hemoth", + "vile", + "306", + "Ram", + "activated", + "Aero", + "butcher", + "Knock", + "disturb", + "totality", + "tted", + "legit", + "cking", + "nikov", + "favoring", + "lang", + "rightful", + "orum", + "!!!!", + "Minute", + "postings", + "Java", + "510", + "microbes", + "sixteen", + "entimes", + "bulb", + "goalt", + "humiliated", + "ansom", + "roach", + "grouping", + "hari", + "cler", + "stared", + "Symptoms", + "basil", + "Whenever", + "Whoever", + "Oil", + "Jericho", + "Alm", + "Pol", + "Hur", + "upro", + "Spo", + "hammer", + "Mur", + "Torch", + "frequencies", + "Expansion", + "paralysis", + "igon", + "Sail", + "silently", + "revolver", + "stockpile", + "pessimistic", + "ESA", + "disclaim", + "democracies", + "Tales", + "Angry", + "Whitman", + "Ori", + "transitioned", + "behind", + "LAN", + "cav", + "Jazeera", + "KC", + "Inspect", + "irty", + "Ain", + "Orig", + "obscene", + "dormant", + "harb", + "Wiz", + "Adolf", + "vic", + "denouncing", + "ye", + "aques", + "omn", + "assemblies", + "nosis", + "admon", + "anguish", + "vag", + "YE", + "Macro", + "rubbing", + "replicated", + "Moon", + "Guitar", + "centimeters", + "amily", + "Ames", + "chlorine", + "Perhaps", + "partisans", + "soc", + "vagina", + "trove", + "YES", + "therapists", + "nods", + "hanged", + "ridge", + "haz", + "macOS", + "ske", + "Shia", + "steril", + "almond", + "Rockefeller", + "intrinsic", + "Certainly", + "sublime", + "Earn", + "abet", + "frameworks", + "ogical", + "ilst", + "ipal", + "rescuing", + "Watergate", + "231", + "Nano", + "ighthouse", + "olph", + "312", + "healed", + "Tomb", + "subst", + "sulph", + "Newsp", + "Lama", + "venue", + "387", + "productive", + "NEED", + "minus", + "Pages", + "cand", + "Clover", + "Forensic", + "ryn", + "ogle", + "ocr", + "vaccinations", + "cies", + "Mek", + "unaffected", + "fetal", + "Dino", + "hemisphere", + "froze", + "Peg", + "microscope", + "moderates", + "GEN", + "Hawai", + "stagn", + "Absolutely", + "practice", + "IBLE", + "cture", + "Ashe", + "condoms", + "poked", + "training", + "intermedi", + "347", + "cardinal", + "Spoon", + "supp", + "previews", + "Service", + "Beam", + "transcend", + "Fresh", + "Sure", + "4000", + "idential", + "Coinbase", + "workings", + "PI", + "passionately", + "decisively", + "Inspection", + "invoke", + "stain", + "cleaners", + "regulates", + "shone", + "EVERY", + "istance", + "map", + "redu", + "occupies", + "procure", + "acket", + "roman", + "illeg", + "leaps", + "yond", + "yarn", + "LTD", + "CONTR", + "Restoration", + "CDs", + "drinkers", + "Jordanian", + "abl", + "disparate", + "primed", + "Firearms", + "artz", + "indispensable", + "Ter", + "fright", + "markedly", + "roam", + "Jurassic", + "feder", + "pepp", + "DV", + "pancakes", + "sweet", + "unmatched", + "assembling", + "Ultimately", + "endeavour", + "luckily", + "bitch", + "elegance", + "eers", + "drop", + "credit", + "scourge", + "Minimum", + "impatient", + "hunted", + "Goddard", + "Kal", + "mined", + "calves", + "234", + "plank", + "injecting", + "Kaufman", + "Compliance", + "tone", + "345", + "dazz", + "Clarks", + "comprehens", + "pist", + "rhythms", + "reserv", + "337", + "IDF", + "shouts", + "midt", + "323", + "soothing", + "administr", + "gloomy", + "futile", + "Prohibition", + "upon", + "Anglic", + "seeking", + "dodge", + "Ds", + "Grants", + "editor", + "Inquis", + "1929", + "decl", + "Ports", + "Cure", + "DPRK", + "oct", + "vocabulary", + "cling", + "298", + "peac", + "antibodies", + "dor", + "Worse", + "smelled", + "leash", + "MED", + "disinteg", + "truthful", + "salesman", + "squares", + "susp", + "craving", + "wizard", + "moral", + "Qué", + "Anything", + "falsehood", + "ARI", + "coworkers", + "thy", + "outher", + "brushing", + "Protest", + "MF", + "abba", + "lead", + "Exhibit", + "Ga", + "Franks", + "dictates", + "illegal", + "relayed", + "ploy", + "اÙĦ", + "Documents", + "tint", + "Yuan", + "depended", + "Mir", + "Introdu", + "recourse", + "oqu", + "TED", + "differentiated", + "Walls", + "sentimental", + "antis", + "retion", + "comes", + "WORLD", + "coax", + "Tatt", + "Gingrich", + "2006", + "Brut", + "Second", + "posed", + "shots", + "313", + "idian", + "alking", + "dens", + "gif", + "akings", + "keywords", + "chast", + "adversary", + "nick", + "iasis", + "Legisl", + "coff", + "Oriental", + "Morg", + "HAR", + "legalizing", + "banter", + "Tart", + "TRI", + "antagon", + "GF", + "oler", + "UFO", + "Therefore", + "Osama", + "Structure", + "apps", + "pee", + "Somehow", + "Overwatch", + "Casual", + "dishon", + "SEE", + "ctive", + "andering", + "Transformation", + "Andy", + "Fever", + "spectator", + "lash", + "protector", + "apy", + "exhilar", + "aroo", + "mamm", + "bystand", + "acky", + "digestive", + "amplified", + "alpha", + "continue", + "Low", + "disgusted", + "356", + "script", + "generational", + "Passenger", + "sight", + "cout", + "hone", + "ulse", + "ignite", + "284", + "gow", + "binary", + "incess", + "Review", + "607", + "Surprise", + "irritation", + "Barth", + "Gum", + "videot", + "Fres", + "asons", + "collaborator", + "fal", + "Gon", + "settles", + "regular", + "miscarriage", + "cube", + "subord", + "Registered", + "notions", + "zzy", + "revert", + "OFF", + "hasht", + "PNG", + "unimaginable", + "builders", + "Taylor", + "PAY", + ").", + "238", + "LAST", + "MAS", + "illustrations", + "parody", + "dispersed", + "Roses", + "estimation", + "Gets", + "Patrick", + "CHA", + "misdem", + "agate", + "alter", + "geo", + "enormously", + "arrogance", + "pert", + "meta", + "Juno", + "iov", + "imov", + "chores", + "acan", + "Paris", + "313", + "Lewis", + "willingly", + "ERA", + "encaps", + "ilk", + "nodes", + "enzyme", + "want", + "tolerant", + "condos", + "asserts", + "canon", + "scanned", + "bishop", + "perched", + "util", + "Bonus", + "create", + "Fuk", + "motif", + "contemplate", + "BEN", + "imir", + "academ", + "uvian", + "Ideas", + "CY", + "ants", + "prostitutes", + "2005", + "Spring", + "Barrel", + "Aunt", + "Ludwig", + "Herm", + "PRO", + "obiles", + "rack", + "STER", + "ucket", + "mun", + "419", + "ICES", + "cardio", + "trenches", + "Nation", + "yahoo", + "burd", + "nost", + "appropriations", + "Chili", + "Josh", + "GW", + "oppressed", + "BEFORE", + "murderous", + "Pen", + "achable", + "rive", + "culmin", + "defin", + "Mord", + "idate", + "Chim", + "ource", + "Electro", + "orthy", + "calendars", + "regation", + "retrospect", + "Tribal", + "Hes", + "cran", + "creditor", + "fibers", + "note", + "idays", + "Sebast", + "Kitty", + "plainly", + "LAPD", + "trumpet", + "Appropriations", + "Hill", + "Veget", + "296", + "lated", + "othes", + "ibrarian", + "Listen", + "nex", + "WHO", + "shampoo", + "claimants", + "isol", + "unchecked", + "mov", + "umo", + "Lens", + "discreet", + "respectfully", + "reclaimed", + "Hatt", + "thus", + "Flo", + "summ", + "phas", + "Haitian", + "strife", + "abound", + "verted", + "patronage", + "449", + "prelim", + "Zhu", + "Revel", + "adic", + "minded", + "Stability", + "resembling", + "vending", + "ischer", + "kisses", + "superiority", + "infinite", + "ISC", + "880", + "appease", + "VO", + "404", + "ECH", + "gam", + "River", + "metal", + "determination", + "Cook", + "buds", + "(%)", + "Created", + "strut", + "425", + "verte", + "Orb", + "weaving", + "261", + "flyers", + "spons", + "Covenant", + "570", + "intangible", + "BJ", + "Stead", + "Brune", + "pain", + "independent", + "Ball", + "witch", + "Ion", + "pupp", + "Cash", + "Convert", + "impede", + "broad", + "onew", + "synergy", + "coined", + "620", + "ivalent", + "Infect", + "Aqua", + "Together", + "Chemistry", + "URL", + "ampion", + "declarations", + "affirmative", + "umper", + "Tarant", + "stereotype", + "bookstore", + "incre", + "chipset", + "angst", + "Jose", + "laus", + "heater", + "ipers", + "eminent", + "hook", + "sticks", + "Coul", + "mildly", + "SG", + "worm", + "disable", + "perfume", + "ISTER", + "gathers", + "Lotus", + "hyp", + "actus", + "distinctly", + "fifth", + "!),", + "Crunch", + "cohesive", + "fortunately", + "ninety", + "cartels", + "empl", + "Direct", + "commuting", + "SX", + "ractive", + "translating", + "AQ", + "slay", + "abuse", + "Proc", + "Cantor", + "Tas", + "Sir", + "Thom", + "CHRIST", + "receptive", + "Cornel", + "Arab", + "grammar", + "handlers", + "alloy", + "thinly", + "adem", + "proponent", + "PVC", + "stump", + "tom", + "rets", + "iciency", + "780", + "311", + "Clapper", + "ITAL", + "Ùħ", + "narrator", + "blond", + "intermittent", + "collabor", + "646", + "metast", + "regeneration", + "Legendary", + "genitals", + "bartender", + "atson", + "Okay", + "passages", + "substituted", + "orr", + "ALTH", + "artic", + "ascent", + "matured", + "terminology", + "served", + "Deliver", + "attic", + "anges", + "renaissance", + "bleed", + "claimer", + "onse", + "Sec", + "particle", + "aneous", + "ateur", + "zeal", + "Pets", + "Working", + "Respect", + "sermon", + "Provided", + "filibuster", + "abolished", + "reviewed", + "cription", + "revers", + "atered", + "435", + "whe", + "ometown", + "UFC", + "products", + "Winter", + "304", + "sporadic", + "orough", + "EB", + "Agric", + "MTA", + "wic", + "powerless", + "carrot", + "ww", + "absorption", + "Typhoon", + "Turkey", + "proclaim", + "hikers", + "practise", + "/$", + "fingertips", + "baff", + "vu", + "ans", + "plug", + "acquaintance", + "itement", + "ihar", + "reluctantly", + "forc", + "guarant", + "Wanted", + "Walk", + "addle", + "unders", + "Fred", + "tides", + "Bai", + "countering", + "raper", + "ursions", + "Flav", + "pared", + "raised", + "Ñı", + "Diff", + "reload", + "ourses", + "Burning", + "wand", + "ledger", + "coughing", + "Loren", + "Nazis", + "compile", + "Eight", + "icultural", + "yy", + "1932", + "Run", + "AIN", + "attractiveness", + "Omn", + "confer", + "compliance", + "embed", + "Steven", + "2001", + "decre", + "prompts", + "Hare", + "leaping", + "slaughtered", + "forfeiture", + "342", + "Charl", + "CDC", + "ographically", + "duplicate", + "distracting", + "examination", + "peas", + "catchy", + "dives", + "Ada", + "Hay", + "enthusiastically", + "funky", + "kay", + "EVA", + "psychologists", + "ancestry", + "iyah", + "ifter", + "nob", + "518", + "rouse", + "chord", + "cone", + "barracks", + "Royale", + "Integration", + "trolling", + "Synt", + "andals", + "Grain", + "Neck", + "618", + "rapist", + "pins", + "witty", + "dehydration", + "arlane", + "immoral", + "accum", + "McAuliffe", + "slow", + "injust", + "1700", + "carbs", + "intel", + "Non", + "isks", + "Tre", + "interviewer", + "sam", + "delve", + "admirable", + "ROM", + "Hispanics", + "impart", + "underrated", + "victimized", + "Psych", + "ppings", + "610", + "pole", + "diner", + "Scale", + "unforeseen", + "surprisingly", + "opus", + "COURT", + "juggling", + "Facilities", + "Aid", + "HPV", + "crawling", + "flu", + "etary", + "Harriet", + "329", + "Sod", + "Biological", + "birth", + "ribed", + "pulses", + "396", + "eways", + "Alma", + "nov", + "015", + "ricane", + "agna", + "Ak", + "Claim", + "pref", + "interfaces", + "ADHD", + "604", + "ZE", + "venture", + "ascend", + "Gou", + "priceless", + "redo", + "kw", + "Conf", + "mah", + "poets", + "stalk", + "encamp", + "hopped", + "melody", + "JECT", + "eming", + "bewild", + "aternal", + "uchs", + "dit", + "Transmission", + "Lake", + "atoms", + "Thoughts", + "ilts", + "volume", + "socioeconomic", + "atisf", + "narr", + "zinski", + "ymes", + "episode", + "inherit", + "intending", + "arenas", + "uras", + "burning", + "334", + "teenth", + "sophistication", + "screenshots", + "autistic", + "lip", + "paper", + "monopol", + "799", + "forms", + "ocrats", + "pineapple", + "begs", + "persecuted", + "subscribed", + "elic", + "PRESIDENT", + "297", + "preferential", + "pyramid", + "convergence", + "wob", + "Project", + "Aluminum", + "JPM", + "BAT", + "dolphins", + "018", + "healthy", + "CG", + "Effective", + "worm", + "Eas", + "olicited", + "USE", + "Caval", + "swirl", + "spaghetti", + "inward", + "Republican", + "publicized", + "economical", + "salsa", + "Titanic", + "dot", + "contro", + "Bangl", + "iban", + "Klux", + "hinges", + "610", + "valves", + "profits", + "Wonder", + "orient", + "sque", + "privatization", + "Obama", + "Thousands", + "Tasman", + "maze", + "eem", + "survives", + "istant", + "enriched", + "encl", + "compliments", + "Shoes", + "insanity", + "consider", + "agog", + "baffled", + "°", + "WordPress", + "qus", + "usual", + "stall", + "Deb", + "Rothschild", + "esche", + "soph", + "ambiguous", + "negative", + "discouraging", + "Alexander", + "319", + "summon", + "ipation", + "000000", + "minimalist", + "enraged", + "777", + "planetary", + "throughput", + "temperament", + "NIC", + "ileged", + "minster", + "PLEASE", + "exagger", + "Description", + "agitated", + "immortal", + "renders", + "charisma", + "sequ", + "majorities", + "freaking", + "Advice", + "embodies", + "stable", + "customization", + "started", + "Autism", + "participates", + "UTC", + "Marco", + "oddly", + "antiqu", + "Pear", + "Fey", + "certify", + "disillusion", + "Physicians", + "obl", + "855", + "elim", + "335", + "Ol", + "Sear", + "nuances", + "past", + "Sa", + "Slov", + "filtered", + "analogy", + "formulate", + "armies", + "puls", + "fters", + "ilipp", + "HOT", + "485", + "Afghans", + "topical", + "Bunny", + "seeing", + "eloqu", + "kidneys", + "DEM", + "pent", + "hus", + "stores", + "Protestant", + "Comm", + "label", + "Kings", + "Purpose", + "âĢ¦..", + "accumulating", + "calling", + "giveaways", + "predicament", + "typ", + "traveler", + "003", + "impro", + "fac", + "mapped", + "itious", + "masculinity", + "tantal", + "DJs", + "viewpoints", + "Burn", + "Wii", + "pak", + "EB", + "hinge", + "facets", + "photographic", + "compiling", + "decks", + "articulated", + "Federal", + "crim", + "llah", + "fiasco", + "LIST", + "oute", + "Draper", + "Laos", + "climbers", + "raph", + "Dek", + "WAY", + "greets", + "oppressive", + "otor", + "otiation", + "\":[", + "Record", + "mining", + "Town", + "favorably", + "Youtube", + "William", + "lan", + "âĢ²", + "Spec", + "tranquil", + "Client", + "oln", + "celona", + "realistically", + "misplaced", + "Bie", + "bye", + "Yo", + "465", + "Madagascar", + "oplan", + "arist", + "confines", + "ï", + "awks", + "piracy", + "unwelcome", + "Intel", + "paranoid", + "CLAIM", + "blush", + "united", + "motivational", + "VII", + "diabetic", + "antiv", + "dissect", + "bestselling", + "fluffy", + "Remote", + "vert", + "Correct", + "colossal", + "contrasts", + "circa", + "Damage", + "unrel", + "discrepancy", + "CIS", + "CLASS", + "ilty", + "synopsis", + "emed", + "cakes", + "ibal", + "inea", + "ienced", + "implicit", + "LOOK", + "silhouette", + "affiliated", + "Halo", + "377", + "lyr", + "Vide", + "herent", + "badges", + "plays", + "orea", + "jammed", + "cancer", + "Yep", + "racted", + "Disability", + "footh", + "friends", + "bloated", + "Bet", + "Antioch", + "introdu", + "annexed", + "ivism", + "Flickr", + "pants", + "interruption", + "645", + "Ily", + "Oss", + "AMA", + "politely", + "natives", + "rushes", + "enges", + "Harm", + "destroyer", + "Estimates", + "transforms", + "invariably", + "cac", + "iency", + "599", + "constitutionally", + "rappers", + "Settlement", + "icz", + "hardened", + "citizens", + "circling", + "trapping", + "guaranteeing", + "690", + "agher", + "arcade", + "fanc", + "slapping", + "OPS", + "masse", + "pudding", + "Jac", + "Graphics", + "uptake", + "?,", + "Fair", + "Satan", + "uffy", + "Guatem", + "Transaction", + "unlocking", + "LINE", + "apprehens", + "glean", + "291", + "exacerbate", + "Trave", + "Trop", + "Supp", + "queens", + "cart", + "scrolling", + "ox", + "cone", + "Matthew", + "DIRECT", + "backer", + "thyroid", + "Sarah", + "EDIT", + "Activision", + "352", + "reinforcements", + "ding", + "plush", + "peanuts", + "Fant", + "Pediatrics", + "accommodating", + "Practices", + "Answer", + "racial", + "Constant", + "740", + "strength", + "apist", + "synthes", + "Leap", + "Fabric", + "brainstorm", + "obia", + "conception", + "tuberculosis", + "majestic", + "Titus", + "Tee", + "likeness", + "SEA", + "lite", + "950", + "sufficient", + "trem", + "harshly", + "redacted", + "welding", + "perplex", + "poetic", + "insignificant", + "ware", + "wandered", + "mete", + "START", + "weaponry", + "opsy", + "shadow", + "obsc", + "hare", + "OPEN", + "diligent", + "Girls", + "initials", + "Start", + "Brookings", + "ombs", + "lashes", + "essor", + "gravy", + "Ubuntu", + "Tree", + "435", + "cellar", + "aquarium", + "Podesta", + "361", + "Controller", + "eru", + "reasonable", + "permissions", + "725", + "administering", + "flirt", + "fleeting", + "asive", + "subcontract", + "fascist", + "cabbage", + "science", + "boiler", + "ioned", + "integrates", + "residue", + "KEY", + "wi", + "squared", + "Unless", + "mute", + "Tuc", + "verb", + "Gary", + "experimentation", + "fee", + "chini", + "marrow", + "Balt", + "nodded", + "tn", + "missionary", + "OTO", + "optimum", + "555", + "whipping", + "aunts", + "Scene", + "characterize", + "retrospective", + "utilizes", + "hastily", + "older", + "PW", + "sleepy", + "020", + "Acid", + "ridiculously", + "gigg", + "649", + "crus", + "Shame", + "Torn", + "finding", + "IPS", + "plat", + "ometers", + "amphib", + "ellow", + "Species", + "commercial", + "virgin", + "darn", + "sorely", + "respondent", + "ray", + "CONS", + "unequivocally", + "server", + "drip", + "Razor", + "Ban", + "HMS", + "hijab", + "Muss", + "sandy", + "aversion", + "overarching", + "ultr", + "Iraqis", + "uninterrupted", + "routing", + "undone", + "independence", + "gra", + "ysics", + "inflammatory", + "cussion", + "Definitely", + "elastic", + "peer", + "Giov", + "Mandarin", + "scratches", + "physicist", + "bestowed", + "usually", + "OULD", + "igration", + "Human", + "Dead", + "osph", + "bott", + "doctoral", + "bending", + "configurations", + "psych", + "db", + "UD", + "arteries", + "orically", + "blasphemy", + "jj", + "checking", + "adian", + "IRD", + "Dialogue", + "shielded", + "Vox", + "Dave", + "turb", + "Massive", + "BMI", + "NF", + "uced", + "ickle", + "ishable", + "embody", + "ÙĪ", + "Senior", + "Result", + "try", + "egu", + "401", + "Loyal", + "perilous", + "dissu", + "mythology", + "Wax", + "Jesus", + "Motorsport", + "advis", + "Aki", + "ISM", + "tested", + "plag", + "riches", + "OCT", + "Locke", + "BG", + "460", + "rawl", + "Termin", + "295", + "chopping", + "KT", + "converts", + "Ask", + "alse", + "Keynes", + "refuted", + "rabbits", + "bilingual", + "urse", + "Salad", + "odiac", + "solidly", + "Dam", + "pp", + "rities", + "Rah", + "itness", + "sixty", + "332", + "cold", + "hindered", + "clipped", + "receptor", + "Homs", + "dusk", + "archae", + "LR", + "rods", + "257", + "Sith", + "Pumpkin", + "ellation", + "WD", + "decriminal", + "usable", + "cheerful", + "Inform", + "brushes", + "vier", + "Brush", + "590", + "boost", + "guided", + "MJ", + "satirical", + "ortion", + "efficiency", + "strands", + "Wilde", + "reproduce", + "verage", + "lug", + "hist", + "offer", + "collapses", + "clerks", + "airstrike", + "IPP", + "iscover", + "nefarious", + "stripe", + "bona", + "ocon", + "punishments", + "ITED", + "Altern", + "testing", + "eerie", + "erous", + "caves", + "condemns", + "Dropbox", + "inese", + "axis", + "Registry", + "Mong", + "bullies", + "docks", + "Alter", + "rella", + "446", + "Dare", + "virtues", + "dont", + "Value", + "ENE", + "received", + "seaf", + "476", + "ilon", + "Kits", + "rarity", + "nurt", + "skin", + "UL", + "Regiment", + "terior", + "hate", + "Estimated", + "Silence", + "organism", + "Signed", + "IA", + "bite", + "thicker", + "eyeb", + "journalistic", + "Disp", + "margin", + "Dri", + "complexes", + "imaginary", + "refuel", + "meticulous", + "Dub", + "haze", + "860", + "proverbial", + "ozone", + "cale", + "resent", + "discrete", + "boats", + "343", + "RET", + "sailor", + "hair", + "gear", + "malt", + "peach", + "Rabb", + "699", + "318", + "Verge", + "Fin", + "Mighty", + "ierce", + "403", + "disenfranch", + "bass", + "nice", + "sinks", + "Laugh", + "367", + "Zur", + "travers", + "Mystery", + "onsense", + "Monarch", + "leapt", + "ergy", + "porate", + "display", + "ilet", + "endemic", + "Bern", + "pulmonary", + "broch", + "Manziel", + "Lyn", + "Repe", + "lda", + "hands", + "troublesome", + "Jordan", + "UTION", + "ALP", + "LEG", + "reconnaissance", + "RNA", + "letters", + "Younger", + "LW", + "Sensor", + "388", + "wielding", + "spr", + "ancestral", + "331", + "OTH", + "Axis", + "irement", + "Compact", + "voice", + "percussion", + "endeav", + "Kate", + "JACK", + "Magnus", + "interconnected", + "Traff", + "demon", + "ardent", + "Somers", + "andum", + "346", + "heartedly", + "ayne", + "Design", + "melon", + "Carib", + "1935", + "intention", + "cape", + "cend", + "organic", + "373", + "Revival", + "BLACK", + "aspiration", + "yellow", + "bodied", + "crave", + "Intelligent", + "Unique", + "tab", + "386", + "Ness", + "Official", + "Stay", + "creat", + "iliary", + "rified", + "Pok", + "abolition", + "Ka", + "Courage", + "Dickens", + "rophic", + "FAR", + "furnished", + ".âĢĵ", + "rete", + "vaginal", + "hner", + "LONG", + "imates", + "Liter", + "Measures", + "Belg", + "\"-", + "Raider", + "enario", + "rification", + "FISA", + "Stab", + "nar", + "mund", + "Tenn", + "wakes", + "charg", + "okers", + "assment", + "siph", + "ludicrous", + "670", + "compositions", + "pinnacle", + "Rankings", + "Telescope", + "secure", + "ib", + "aptly", + "paste", + "JUST", + "RD", + "herry", + "sung", + "mig", + "naires", + "migrated", + "Base", + "amazingly", + "unregulated", + "published", + "PIT", + "Missile", + "extreme", + "Alone", + "skilled", + "Ramp", + "camer", + "flyer", + "brewers", + "Reference", + "MOV", + "Lep", + "entitle", + "ivals", + "PIN", + "batches", + "unexplained", + "energies", + "blurred", + "enged", + "orig", + "WF", + "olves", + "Picks", + "Twice", + "arranted", + "membrane", + "Moonlight", + "sulfur", + "purposely", + "fumes", + "(#", + "onics", + "ivities", + "rollers", + "flattering", + "felt", + "intoxication", + "Bridge", + "Fallout", + "creatively", + "psychologically", + "despicable", + "gae", + "820", + "VERS", + "tidal", + "carbohydrates", + "strip", + "gravitational", + "feds", + "Zhao", + "legates", + "307", + "String", + "Repair", + "1928", + "orses", + "atography", + "Boston", + "asymm", + "Somebody", + "Van", + "Sovereign", + "notoriety", + "simulate", + "Discussion", + "Transition", + "copying", + "antage", + "Rodrig", + "indifference", + "580", + "astronomical", + "screws", + "840", + "inates", + "Streaming", + "entit", + "Literature", + "369", + "805", + "OTS", + "о", + "img", + "inness", + "reverber", + "partition", + "Short", + "moist", + "spoof", + "Desire", + "orce", + "crammed", + "unfor", + "Pan", + "ingen", + "relat", + "Mother", + "Gn", + "altern", + "resurg", + "cramped", + "Citadel", + "laureate", + "analys", + "nuns", + "Tie", + "activ", + "Surprisingly", + "Protective", + "Redemption", + "endlessly", + "fists", + "spl", + "Kron", + "Examples", + "Especially", + "prejud", + "Schwar", + "237", + "Plants", + "UNDER", + "lasers", + "sher", + "goddess", + "wipes", + "409", + "GTA", + "hybrids", + "rowd", + "MILL", + "NUM", + "Geek", + "TWO", + "Timbers", + "resembled", + "GRE", + "Bring", + "compressed", + "Oral", + "379", + "wrench", + "LCS", + "homosexual", + "Kelly", + "hump", + "Sicily", + "perished", + "aos", + "doesn", + "scrib", + "Charlie", + "shuffle", + "372", + "cedented", + "402", + "tiers", + "interacted", + "HG", + "Jere", + "BRA", + "DOC", + "things", + "faiths", + "girlfriends", + "fortified", + "develop", + "Kus", + "iability", + "rase", + "iotics", + "Chern", + "boxes", + "abol", + "idan", + "emon", + "Judaism", + "Situation", + "Grimm", + "gou", + "Victim", + "backer", + "animosity", + "Horizons", + "Kazakh", + "grossly", + "Tac", + "yg", + "366", + "cheaply", + "formulated", + "Dangerous", + "offensive", + "sauces", + "keyboards", + "666", + "canopy", + "Inc", + "astered", + "iesel", + "adv", + "currency", + "scapego", + "plings", + "BDS", + "strangely", + "today", + "Egyptians", + "coron", + "often", + "Transformers", + "Afterwards", + "reated", + "poisonous", + "geographically", + "mell", + "Cross", + "deductible", + "Zionist", + "cutter", + "RP", + "Imag", + "overflow", + "358", + "ADD", + "bones", + "flattened", + "GREEN", + "laure", + "haps", + "Cellular", + "kens", + "363", + "Smash", + "Speak", + "Maiden", + "greedy", + "Manit", + "facet", + "GPA", + "racks", + "popular", + "322", + "Bars", + "avement", + "359", + "pomp", + "registers", + "Fs", + "Loving", + "Taxi", + "concert", + "Archae", + "curls", + "Spit", + "LIFE", + "invade", + "rolog", + "wreck", + "conflicted", + "970", + "exiled", + "chew", + "udging", + "exper", + "Ft", + "rius", + "Xer", + "~", + "bandwagon", + "Fore", + "Cat", + "overflowing", + "radios", + "Much", + "facilitates", + "Caf", + "Qing", + "Use", + "mang", + "pissed", + "Outer", + "within", + "Schr", + "Sherlock", + "336", + "casc", + "chens", + "incent", + "cultivating", + "ampions", + "wasteful", + "adays", + "sets", + "LF", + "watching", + "abandonment", + "Jesuit", + "legislatures", + "regnancy", + "Colt", + "interns", + "undertook", + "IPA", + "Install", + "nsics", + "washer", + "beginners", + "Diseases", + "limp", + "ESA", + "Basically", + "prud", + "LED", + "grease", + "ousel", + "rotten", + "Cele", + "facts", + "Louie", + "ISI", + "481", + "sett", + "toug", + "Reck", + "OUNT", + "Fou", + "inhibitor", + "gru", + "bane", + "1980", + "Panc", + "superficial", + "authoritative", + "VOL", + "790", + "crusade", + "airy", + "emphatically", + "flourishing", + "416", + "heroine", + "inx", + "anch", + "stretched", + "Regener", + "Ancient", + "evaluate", + "antibody", + "Eston", + "Aeg", + "boldly", + "TN", + "Percentage", + "747", + "rapt", + "Edited", + "Earth", + "phal", + "XXX", + "arling", + "Religion", + "503", + "forces", + "endpoint", + "Miller", + "Ba", + "disappears", + "andre", + "connector", + "407", + "TOUR", + "aura", + "Razer", + "UPDATE", + "calib", + "original", + "Monkey", + "Ir", + "exacerb", + "killing", + "forb", + "native", + "poking", + "veiled", + "mails", + "alphabet", + "awkwardly", + "Names", + "spiders", + "Param", + "Colour", + "unification", + "Pione", + "offend", + "scoff", + "SAR", + "Buildings", + "edes", + "Ake", + "firmware", + "Madison", + "policy", + "Computing", + "RW", + "fluent", + "dece", + "swore", + "restaur", + "presses", + "ophon", + "philosopher", + "ften", + "intruder", + "leng", + "Cowboy", + "cled", + "meticulously", + "Pair", + "END", + "capsules", + "auxiliary", + "verses", + "sheltered", + "explorer", + "Wolverine", + "auts", + "inhibitors", + "Peng", + "Valve", + "imar", + "chuck", + "Recording", + "ardu", + "Test", + "interven", + "chrome", + "months", + "tap", + "Manz", + "format", + "Balkans", + "annex", + "uder", + "AAC", + "disturbances", + "354", + "asms", + "Tad", + "puting", + "fateful", + "imen", + "audi", + "Newsweek", + "Around", + "retribution", + "sugars", + "escapes", + "legitim", + "Proof", + "misogyn", + "cit", + "clutching", + "exist", + "revol", + "discs", + "discrimination", + "stout", + "aline", + "Random", + "364", + "apprehension", + "mockery", + "fossils", + "Stress", + "benefic", + "exc", + "lude", + "Small", + "gh", + "observes", + "SUP", + "brewer", + "ESP", + "omitted", + "multiple", + "minimizing", + "taco", + "indifferent", + "medi", + "available", + "252", + "sanity", + "Cookie", + "mostly", + "near", + "NASA", + "lowly", + "seless", + "obsess", + "itous", + "Dispatch", + "canyon", + "briefs", + "Say", + "Nato", + "Spend", + "242", + "Ethernet", + "matte", + "Stim", + "hetics", + "flourished", + "389", + "McA", + "695", + "overr", + "torment", + "pirate", + "Johann", + "roversial", + "Unemployment", + "breakers", + "Messages", + "tones", + "tagging", + "frog", + "Jewish", + "messenger", + "exasper", + "ernaut", + "narrower", + "Catalyst", + "Secrets", + "adj", + "Fug", + "aura", + "therape", + "mber", + "caliphate", + "retreating", + "Comput", + "burying", + "ail", + "griev", + "lins", + "825", + "tten", + "ifully", + "Trials", + "igma", + "1914", + "coordinates", + "ocusing", + "Feng", + "Whale", + "shorten", + "correctness", + "evil", + "network", + "reactive", + "assuming", + "Laksh", + "games", + "ruining", + "excluding", + "annels", + "º", + "rubbed", + "aleb", + "flex", + "iped", + "Limit", + "allowed", + "DMV", + "LD", + "stamina", + "conduct", + "mislead", + "lib", + "Eminem", + "payoff", + "kernel", + "sweeps", + "sonic", + "Kodi", + "unique", + "surrog", + "Michigan", + "attest", + "dummy", + "Stellar", + "Squadron", + "Hait", + "Spirits", + "605", + "Hemisphere", + "legram", + "Rack", + "opol", + "freshwater", + "cession", + "abort", + "LOG", + "fuzzy", + "crystall", + "illation", + "Freddy", + "salvation", + "juxtap", + "weekly", + "usha", + "456", + "660", + "Glacier", + "negatives", + "illegitimate", + "Protein", + "Moore", + "Der", + "infancy", + "Again", + "ALD", + "Leon", + "Ideally", + "fresh", + "730", + "gamb", + "screwed", + "wow", + "embodied", + "Cinderella", + "341", + "Piano", + "broccoli", + "mats", + "Zheng", + "cream", + "anut", + "Zig", + "Columb", + "Tibetan", + "Death", + "stren", + "Vertical", + "ratification", + "principally", + "ELD", + "forbid", + "amalg", + "blind", + "auri", + "stery", + "barley", + "FBI", + "Hex", + "925", + "Domin", + "oat", + "swayed", + "KKK", + "Taxes", + "ker", + "eeper", + "Awakens", + "Pix", + "KING", + "dc", + "Ren", + "legitimately", + "Triumph", + "Sites", + "Sai", + "tl", + "painted", + "Waiting", + "starting", + "parents", + "Duo", + "eele", + "upper", + "Investig", + "eighteen", + "correlated", + "Cascade", + "acca", + "Alph", + "Polic", + "EVs", + "worthless", + "Indust", + "auld", + "Yiannopoulos", + "Ezra", + "morphed", + "originating", + "mania", + "sparing", + "extrem", + "cre", + "ults", + "mare", + "classified", + "parachute", + "mistrust", + "ONT", + "Mind", + "thru", + "707", + "Twain", + "melodies", + "Danger", + "DPS", + "derive", + "dissolution", + "childbirth", + "415", + "fork", + "solid", + "loads", + "CGI", + "378", + "Shed", + "Face", + "comet", + "iceps", + "Reduction", + "Fly", + "jp", + "Animation", + "Luke", + "abiding", + "devise", + "Ae", + "flux", + "bras", + "fracturing", + "inventive", + "Granger", + "sap", + "inducing", + "reviewers", + "Officers", + "WHY", + "amplify", + "entr", + "slit", + "457", + "reformed", + "Phi", + "tempt", + "contradiction", + "585", + "Maced", + "371", + "kinson", + "robe", + "Hunters", + "astern", + "criminal", + "jew", + "decentralized", + "bands", + "avatar", + "Barrier", + "characterization", + "student", + "gays", + "specialize", + "Judging", + "initiation", + "shove", + "pirates", + "fictitious", + "Poker", + "Elsa", + "TECH", + "handedly", + "glued", + "clinically", + "inaccessible", + "deregulation", + "prohib", + "dangling", + "noses", + "stash", + "اØ", + "ESH", + "monstrous", + "crept", + "Charm", + "beh", + "shuts", + "236", + "imedia", + "445", + "Du", + "afar", + "Rout", + "flares", + "Utah", + "808", + "jewels", + "2004", + "recal", + "Gas", + "Excellent", + "pitfalls", + "Drawing", + "viously", + "angered", + "changes", + "pasture", + "talking", + "inequ", + "bicycl", + "Cost", + "423", + "bard", + "anterior", + "ecast", + "CHR", + "397", + "masters", + "706", + "Finish", + "Yet", + "study", + "Cogn", + "loaf", + "spatial", + "Parad", + "batch", + "vents", + "spins", + "Addiction", + "condone", + "proble", + "English", + "Romans", + "Saying", + "Kling", + "Universal", + "ivist", + "skirm", + "2500", + "263", + "aired", + "Martian", + "Compensation", + "lation", + "Salam", + "LGBT", + "Dart", + "strike", + "vasive", + "ILLE", + "imaginative", + "Euph", + "Financial", + "holog", + "orah", + "crit", + "Oswald", + "512", + "Uri", + "discrepancies", + "beads", + "Shots", + "Mem", + "hunts", + "subtly", + "470", + "Vigil", + "sew", + "Burma", + "igm", + "ighed", + "swe", + "251", + "deceit", + "physi", + "iflower", + "Cert", + "chewing", + "rax", + "MER", + "icient", + "Les", + "390", + "perjury", + "filtering", + "770", + "poppy", + "bland", + "Nasa", + "orbiting", + "Ripple", + "otal", + "Ryu", + "Shap", + "Jian", + "piv", + "Neptune", + "rary", + "unavoidable", + "guideline", + "waterfall", + "inators", + "Logic", + "Plug", + "role", + "alterations", + "Sett", + "Feld", + "freezes", + "bedrock", + "VIEW", + "ovation", + "needless", + "IU", + "ignant", + "Confeder", + "316", + "fine", + "jars", + "gotten", + "Bron", + "mindfulness", + "imating", + "hysteria", + "hurried", + "infantry", + "NYU", + "tags", + "Penn", + "tracing", + "Swing", + "Io", + "reckoned", + "Recall", + "Version", + "314", + "ecology", + "armoured", + "resonance", + "970", + "vigilance", + "rede", + "Bohem", + "chau", + "Devi", + "tru", + "))", + "Put", + "flavored", + "Clown", + "Senate", + "Scandinavian", + "mable", + "Residents", + "Franchise", + "precincts", + "Prem", + "Neutral", + "coal", + "delinqu", + "Mus", + "UME", + "tedious", + "roots", + "Condition", + "Intercept", + "017", + "itives", + "definitively", + "obliter", + "clandestine", + "stagnation", + "blindness", + "abiding", + "remix", + "feeding", + "unrecogn", + "2003", + "960", + "381", + "bulky", + "xia", + "ivered", + "inic", + "Soci", + "Yards", + "hides", + "Film", + "testim", + "blacklist", + "Deep", + "Standard", + "Clash", + "riddled", + "diseng", + "TRE", + "IDs", + "migrating", + "protect", + "graded", + "vaguely", + "Character", + "382", + "MOD", + "Eng", + "mobilized", + "sincerity", + "317", + "sighted", + "ownt", + "âĢİ", + "umpy", + "itching", + "Verd", + "cook", + "simulator", + "players", + "Early", + "infeld", + "maximizing", + "Philipp", + "Photoshop", + "destroys", + "befriend", + "filthy", + "Incident", + "gha", + "complicity", + "messing", + "YA", + "Negro", + "adows", + "374", + "pip", + "cean", + "1924", + "Sent", + "represent", + "deems", + "Rue", + "titanium", + "manners", + "âĢ¦âĢ¦", + "bare", + "usur", + "mma", + "Panda", + "ulus", + "Slav", + "324", + "Mole", + "^", + "micro", + "foreign", + "lest", + "ocular", + "Univ", + "Frag", + "shepherd", + "electron", + "FSA", + "unl", + "dose", + "immersion", + "DeL", + "biomedical", + "Anna", + "skillet", + "recre", + "trillions", + "voy", + "normalized", + "radio", + "cue", + "urbed", + "thinkers", + "328", + "327", + "Forge", + "505", + "unbearable", + "olini", + "disinfect", + "shaving", + "toxicity", + "453", + "heterosexual", + "Baltimore", + "stool", + "lr", + "Mk", + "antidote", + "Dark", + "810", + "irritated", + "SUPPORT", + "Chance", + "bent", + "Zelda", + "Penguin", + "ifled", + "arte", + "705", + "condol", + "izza", + "CK", + "projector", + "ravings", + "1919", + "burner", + "Schwarz", + "Oregon", + "ridicule", + "instructional", + "\"#", + "Dign", + "kitten", + "constit", + "iration", + "Speed", + "ecycle", + "False", + "Dealer", + "Could", + "655", + "outside", + "worldview", + "246", + "spitting", + "595", + "MN", + "Comes", + "ingu", + "enzymes", + "compass", + "exclaimed", + "Malays", + "1916", + "coloring", + "repeats", + "soils", + "trivia", + "Isles", + "Const", + "Fiction", + "665", + "criminality", + "Zi", + "384", + "Wilderness", + "Canary", + "Vs", + "и", + "APIs", + "behest", + "eb", + "Hipp", + "preempt", + "evoke", + "inept", + "tele", + "447", + "Garmin", + "pursuits", + "351", + "cliché", + "Jihad", + "308", + "Snake", + "Announce", + "Nearly", + "!'\"", + "1927", + "saw", + "abhor", + "Plan", + "rawled", + "Riy", + "ensor", + "Fal", + "quick", + "odynamic", + "substitution", + "provoking", + "Operation", + "rupulous", + "sweetness", + "folk", + "Default", + "starved", + "Printing", + "urious", + "Tracker", + "them", + "leth", + "emptied", + "footprints", + "ilian", + "battalion", + "prophet", + "railing", + "hect", + "rouch", + "lees", + "ideologies", + "254", + "Gods", + "Avalon", + "frontrunner", + "Pork", + "Pipe", + "scaven", + "ming", + "erg", + "520", + "hatched", + "asant", + "HI", + "pend", + "288", + "Prom", + "achev", + "Ecology", + "enforcement", + "467", + "dule", + "realism", + "Types", + "USB", + "utra", + "Hiroshima", + "contradicted", + "393", + "DSL", + "therein", + "Reconstruction", + "243", + "irled", + "479", + "Whats", + "Currently", + "POWER", + "Hiro", + "Breath", + "Yourself", + "lantern", + "376", + "É", + "Humans", + "Lady", + "dissemination", + "ecake", + "Chao", + "flat", + "inspecting", + "stration", + "identifiable", + "CV", + "Lobby", + "function", + "Roll", + "DIV", + "Tell", + "fasc", + "AOL", + "HM", + "Keefe", + "porous", + "smoot", + "existence", + "Deg", + "divor", + "isner", + "allas", + "Bloomberg", + "dictators", + "Geh", + "silicone", + "dab", + "mashed", + "pric", + "might", + "BLM", + "patriarch", + "Microsoft", + "Ads", + "coronary", + "Contrary", + "dra", + "Started", + "buckle", + "lear", + "accept", + "Within", + "bd", + "interested", + "bia", + "POR", + "motion", + "Founders", + "Cassandra", + "Passion", + "behavioural", + "Healing", + "markings", + "snowball", + "ridiculed", + "phase", + "unto", + "aque", + "uggets", + "frantically", + "coward", + "inconvenient", + "Taking", + "Afee", + "twisting", + "930", + "Sieg", + "Git", + "curs", + "Glas", + "Significant", + "achieves", + "preferably", + "condensed", + "fetus", + "univers", + "pse", + "Access", + "intertwined", + "been", + "quit", + "LEGO", + "imagining", + "454", + "plains", + "sequently", + "pull", + "Fast", + "Pot", + "yles", + "AIR", + "blatantly", + "eki", + "ilated", + "Membership", + "262", + "}", + "excavation", + "ethn", + "addin", + "foundational", + "ceptions", + "Viet", + "exempt", + "microphones", + "244", + "778", + "dwar", + "attery", + "502", + "Kik", + "inspir", + "Maximum", + "vengeance", + "etched", + "outine", + "552", + "unicorn", + "gged", + ".�", + "Blackwell", + "Statue", + "dissidents", + "Kaine", + "deforestation", + "Scholar", + "pleasantly", + "ÑĤ", + "398", + "RUN", + "arent", + "undeniably", + "technologically", + "consciously", + "Ether", + "proportional", + "laund", + "Rye", + "ambiguity", + "unmist", + "Terror", + "ciplinary", + "Improved", + "hesis", + "cooker", + "elsen", + "guerrilla", + "opped", + "ATURE", + "requ", + "unprepared", + "camel", + "fitt", + "Sex", + "edged", + "recurrent", + "ctuary", + "Compare", + "Serving", + "Tri", + "transient", + "Bees", + "covenant", + "fantasies", + "espresso", + "draft", + "baugh", + "democratically", + "Bans", + "Manual", + "Turtle", + "ennett", + "achy", + "Clim", + "descending", + "prow", + "inconsistencies", + "Player", + "oblivious", + "Wonderland", + "nav", + "aughter", + "lod", + "403", + "Polaris", + "Leia", + "Infantry", + "Sy", + "Meter", + "autoimmune", + "diagnoses", + "trespass", + "011", + "wrong", + "GREAT", + "telescopes", + "shows", + "Pac", + "olation", + "clerics", + "dissenting", + "406", + "etiquette", + "deterrence", + "765", + "ove", + "Has", + "Pak", + "ा", + "Nec", + "sociology", + "witz", + "kittens", + "continual", + "overlapping", + "monks", + "Mechanical", + "Captain", + "ocial", + "Falling", + "Correction", + "Trouble", + "slog", + "253", + "emanating", + "widest", + "PROV", + "Japanese", + "urat", + "boxed", + "Cases", + "jarring", + "Fix", + "'?", + "Strateg", + "Republic", + "ovy", + "362", + "Mothers", + "streaks", + "localized", + "ONLY", + "eh", + "Object", + "stub", + "Fre", + "Scarlet", + "multip", + "Maul", + "Problems", + "cest", + "mortal", + "arche", + "ulet", + "fuller", + "GER", + "Si", + "mr", + "Powerful", + "boxing", + "Peer", + "Jean", + "TF", + "plural", + "optim", + "Jimmy", + "Friendly", + "Mex", + "depri", + "PK", + "waitress", + "eph", + "arrass", + "ikawa", + "feel", + "Finally", + "fourth", + "394", + "conom", + "VT", + "eleg", + "ivot", + "harsher", + "Pepe", + "Impl", + "ankles", + "idity", + "Prepare", + "Rather", + "conservatism", + "unquestion", + "ribution", + "Patent", + "Deluxe", + "AE", + "007", + "prag", + "bg", + "palate", + "intric", + "ossom", + "spac", + "Spotlight", + "Seven", + "amacare", + "Gotham", + "encompass", + "nicer", + "Lauder", + "scaff", + "worn", + "442", + "propri", + "443", + "Compos", + "Initi", + "inth", + "rehe", + "Prov", + "gri", + "ossip", + "Modest", + "quiet", + "wealthier", + "241", + "icum", + "communism", + "helpers", + "bellig", + "405", + "uttered", + "bitterness", + "nl", + "474", + "vitality", + "blank", + "Leth", + "PAC", + "326", + "Napoleon", + "299", + "Reviews", + "Sect", + "strongh", + "Tube", + "woodland", + "humming", + "411", + "Alpha", + "undet", + "mounts", + "Officials", + "igning", + "830", + "Stamp", + "ubby", + "424", + "outlandish", + "jerk", + "radiant", + "cubes", + "Director", + "atro", + "vous", + "Sab", + "pretended", + "620", + "975", + "Sham", + "potassium", + "Attention", + "gly", + "opens", + "Worker", + "porter", + "splendid", + "embed", + "Je", + "Meal", + "surname", + "Usually", + "timer", + "weave", + "irin", + "Genetics", + "ensual", + "merry", + "apprehend", + "utsche", + "strate", + "supplementary", + "Roundup", + "upid", + "miraculous", + "HUN", + "glaciers", + "weed", + "Suggest", + "XL", + "authors", + "barking", + "UKIP", + "leased", + "RAD", + "fide", + "phen", + "scanners", + "Parents", + "Blaze", + "tweaking", + "elaborated", + "susp", + "iscovered", + "thighs", + "radicals", + "ULTS", + "aggressive", + "endants", + "Hon", + "correcting", + "391", + "pps", + "Territories", + "conferred", + "crazy", + "utor", + "Survival", + "browsers", + "Conflict", + "pn", + "deprive", + "riage", + "ilan", + "à¦", + "949", + "Congratulations", + "radical", + "Hits", + "powerful", + "crypt", + "745", + "Registrar", + "ophile", + "Element", + "cooked", + "Twilight", + "demos", + "IER", + "stricken", + "Magic", + "abby", + "Sack", + "Shrine", + "Nev", + "Probably", + "Wisdom", + "ulpt", + "opher", + "colonel", + "atl", + "Tem", + "kun", + "Indie", + "Putin", + "jection", + "areth", + "Bullet", + "smartest", + "Esper", + "proficiency", + "cessation", + "mars", + "DATA", + "sup", + "ostr", + "Jane", + "pathogens", + "hd", + "NK", + "horribly", + "regulated", + "esteemed", + "Chinatown", + "vibration", + "overboard", + "Rhod", + "feces", + "otation", + "cryptic", + "Bal", + "OPER", + "affirmation", + "menstrual", + "untold", + "anecdotes", + "HOUSE", + "cape", + "311", + "ittance", + "Remy", + "Waves", + "COVER", + "ordinate", + "restricts", + "Samsung", + "plantations", + "olver", + "Better", + "Explos", + "nasal", + "Syri", + "Perl", + "latency", + "othermal", + "Sweet", + "Ryzen", + "Yuri", + "smack", + "crow", + "aniel", + "iological", + "monk", + "tutorial", + "Aure", + "cliffs", + "ameron", + "umers", + "Mour", + "unorthodox", + "gulf", + "intrusive", + "VIII", + "FF", + "enlarged", + "spheres", + "Cheap", + "Amend", + "::", + "pacing", + "Startup", + "Dating", + "racist", + "Divine", + "pollen", + "Meaning", + "Lei", + "MOT", + "ARC", + "legate", + "brav", + "Ross", + "redit", + "414", + "ringe", + "perhaps", + "SPA", + "Southern", + "Front", + "undrum", + "assorted", + "Dawkins", + "Wrap", + "consequential", + "Fuji", + "458", + "unst", + "Bon", + "acter", + "Trade", + "ingers", + "Clin", + "stimul", + "arah", + "inois", + "urdy", + "obsessive", + "Zone", + "primitive", + "unctions", + "adapter", + "assures", + "Daddy", + "unsatisf", + "441", + "1910", + "secondly", + "truth", + "RED", + "040", + "Pope", + "venants", + "estim", + "hemorrh", + "excruciating", + "459", + "boils", + "ieved", + "Storm", + "manifestation", + "insulated", + "fb", + "classify", + "Mbps", + "inclination", + "aur", + "polarized", + "occupations", + "Secretary", + "customizable", + "scribe", + "adjunct", + "1922", + "rived", + "ocative", + "Friends", + "Oak", + "psyche", + "wrinkles", + "anthrop", + "coercion", + "enos", + "variability", + "hma", + "phot", + "Xander", + "Diss", + "tigers", + "ahoo", + "focus", + "rical", + "grow", + "seminal", + "disciples", + "Cas", + "Hundreds", + "scissors", + "correct", + "fascism", + "imoto", + "nudity", + "charg", + "rusty", + "Lyndon", + "anomalies", + "onial", + "iCloud", + "annoy", + "distortion", + "Lou", + "Giul", + "eyes", + "870", + "uum", + "Ultr", + "Action", + "cigarette", + "igators", + "kj", + "323", + "uine", + "Score", + "mans", + "Security", + "arom", + "Boards", + "wrists", + "602", + "astronomy", + "resin", + "width", + ")/", + "concurrent", + "unless", + "606", + "Magnet", + "authorizing", + "Junk", + "atical", + "authent", + "zac", + "413", + "Grape", + "circled", + "ooz", + "visceral", + "ointment", + "incendiary", + "Bourbon", + "gimmick", + "vette", + "Stan", + "detachment", + "488", + "misogyny", + "enlight", + "utic", + "inquire", + "BEL", + "ascular", + "Wasserman", + "Dallas", + "constellation", + "dystopian", + "504", + "Optical", + "silhou", + "Girl", + "Gong", + "Highest", + "????????", + "Sav", + "ocity", + "leted", + "attrition", + "Expedition", + "Killed", + "501", + "ONES", + "dat", + "glyphosate", + "plugs", + "lact", + "Fla", + "fps", + "riger", + "paragraphs", + "innate", + "Foo", + "aternity", + "Gry", + "oneself", + "642", + "Iowa", + "oodle", + "Coconut", + "Chess", + "ommel", + "magnesium", + "airliner", + "exceedingly", + "Creator", + "YouTube", + "sleeper", + "longing", + "Percy", + "matrix", + "âľ", + "barren", + "Mrs", + "invading", + "incom", + "emperor", + "ip", + "irie", + "predictably", + "Bless", + "superpower", + ":-", + "propensity", + "easy", + "educ", + "Polly", + "cumbersome", + "collide", + "016", + "transports", + "scraps", + "below", + "hairs", + "mentation", + "evolves", + "Fallen", + "unsurprisingly", + "cuff", + "249", + "mental", + "Camel", + "337", + "Clinton", + "decad", + "STEP", + "Testament", + "irresistible", + "ACE", + "hamm", + "Terr", + "caul", + "iggins", + "proficient", + "resp", + "heirs", + "321", + "dress", + "Clothing", + "560", + "264", + "Robb", + "frail", + "optimizing", + "615", + "Refuge", + "rowth", + "washing", + "genders", + "indu", + "NAT", + "leans", + "eyed", + "hilar", + "vice", + "wolf", + "fatig", + "ococ", + "Carry", + "Community", + "Clark", + "itably", + "sv", + "448", + "numer", + "1925", + "Behavioral", + "Scream", + "geek", + "rake", + "TTC", + "additives", + "Bye", + "ylon", + "foliage", + "ateral", + "rapnel", + "Science", + "recollection", + "thening", + "Ubisoft", + "Lur", + "Okinawa", + "Provision", + "ferred", + "Grounds", + "hops", + "aterial", + "acad", + "engulf", + "Apex", + "frequency", + "relations", + "Corvette", + "Repeat", + "anew", + "hes", + "Lair", + "PSP", + "foundation", + "Band", + "Publisher", + "reciprocal", + "287", + "pir", + "Adams", + "prostitute", + "Mecca", + "ectomy", + "skew", + "Lol", + "Voice", + "Calais", + "ISION", + "rue", + "gaping", + "prot", + "6000", + "tilted", + "goofy", + "Stand", + "fellows", + "curly", + "POW", + "lore", + "inhabited", + "Identification", + "Metro", + "dispel", + "invoking", + "deleting", + "stigmat", + "Dalai", + "equate", + "mascara", + "endered", + "NYT", + "Committees", + "rians", + "Olympus", + "QR", + "Drinking", + "batt", + "andr", + "computer", + "Senator", + "Twist", + "Noise", + "cheesy", + "1931", + "tyranny", + "negligible", + "Bok", + "webpage", + "HEAD", + "Novel", + "quarry", + "expressive", + "forgiving", + "Among", + "asin", + "Suc", + "Democrats", + "795", + "aback", + "¨", + "Neon", + "392", + "RNC", + "PROC", + "sein", + "Ros", + "emot", + "ASA", + "Seb", + "Extended", + "atern", + "psychedelic", + "Fil", + "Orwell", + "SOS", + "conceive", + "hobbies", + "specimens", + "TEXT", + "sometimes", + "Mario", + "orpor", + "Temporary", + "apocalypse", + "counterproductive", + "QUEST", + "Cargo", + "Amb", + "optic", + "groups", + "paranoia", + ".?", + "sounding", + "mediately", + "System", + "ubi", + "uttered", + "graphs", + "âĢĭâĢĭ", + "scientifically", + "bluntly", + "hopping", + "Fun", + "SUPER", + "robe", + "VB", + "Quote", + "incarnation", + "treadmill", + "1915", + "bart", + "669", + "hoc", + "309", + "improvis", + "hut", + "mixer", + "Ct", + "span", + "watered", + "patriot", + "dehyd", + "laughs", + "Fancy", + "Voc", + "intellect", + "Tid", + "nesting", + "Tel", + "()", + "letter", + "Seems", + "Ops", + "Contents", + "ript", + "hani", + "recru", + "pickups", + "repair", + "Throughout", + "bear", + "conquered", + "656", + "malf", + "ordained", + "755", + "Reprodu", + "brain", + "Outs", + "Wage", + "Ru", + "________", + "LAW", + "Wass", + "complication", + "Fri", + "regener", + "Wait", + "577", + "misconception", + "bombardment", + "unloaded", + "dictionary", + "IU", + "025", + "etically", + "Narr", + "repe", + "assigning", + "Rail", + "notebooks", + "ingest", + "rpm", + "alienated", + "Credits", + "indis", + "Gathering", + "aration", + "-+-+-+-+", + "ori", + "sr", + "ndra", + "libertarian", + "coerced", + "ording", + "tranqu", + "elbows", + "549", + "ping", + "RELE", + "Yanuk", + "maneuvers", + "Trojan", + "IFIED", + "Violent", + "è", + "lest", + "arrows", + "frog", + "anty", + "WB", + "Seen", + "648", + "clutter", + "Bender", + "pessim", + "Teg", + "Asian", + "IFIC", + "exponential", + "sponge", + "rite", + "DAM", + "tacit", + "Zoom", + "olds", + "onward", + "Sandwich", + "missible", + "isol", + "940", + "inciner", + "Trick", + "awakening", + "dart", + "Couch", + "respons", + "Elephant", + "Pluto", + "Tags", + "itcher", + "644", + "702", + "electrons", + "Myth", + "Aad", + "Danny", + "craw", + "Certification", + "tending", + "pellets", + "amused", + "Auschwitz", + "Appl", + "iris", + "ashion", + "walking", + "abnorm", + "Cro", + "?:", + "Icelandic", + "Availability", + "cann", + "Opt", + "buster", + "Quartz", + "Executive", + "tracks", + "igel", + "MIT", + "Tracking", + "conditioned", + "sampled", + "Genius", + "substit", + "Siberia", + "frequ", + "historic", + "okin", + "OWS", + "1500", + "warts", + "Etsy", + "licks", + "Smooth", + "unity", + "515", + "perk", + "aida", + "forts", + "UA", + "RIC", + "Spain", + "Wired", + "cuts", + "furnace", + "TOTAL", + "Tables", + "662", + "Fab", + "quaint", + "Worlds", + "Cabin", + "atche", + "List", + "VO", + "keyword", + "258", + "Farm", + "timer", + "Volt", + "Build", + "pressed", + "*,", + "324", + "aiman", + "TING", + "sneaking", + "cery", + "crib", + "Illust", + "later", + "compar", + "propulsion", + "647", + "Trails", + "periphery", + "steel", + "vividly", + "Conver", + "eatured", + "427", + "463", + "approx", + "spin", + "configured", + "inside", + "razy", + "account", + "anye", + "riend", + "bows", + "809", + "DEF", + "Rez", + "Fans", + "DF", + "stains", + "Atom", + "Conce", + "TOM", + "ELECT", + "disappro", + "019", + "afia", + "Temperature", + "extracts", + "fab", + "unsur", + "seasoning", + "Ty", + "KB", + "posit", + "locality", + "1200", + "cour", + "izons", + "hh", + "506", + "DLC", + "iago", + "corpses", + "iddling", + "Mayor", + "simplistic", + "libel", + "almonds", + "swast", + "Change", + "Joker", + "MAR", + "Scully", + "mailbox", + "VIDEO", + "Kyoto", + "esley", + "Incredible", + "youtube", + "inequalities", + "bolts", + "bothering", + "attentive", + "Sparrow", + "diaper", + "fanbase", + "uncont", + "Ap", + "Qi", + "Price", + "471", + "pearl", + "wid", + "899", + "Pony", + "casting", + "inhabit", + "unve", + "insur", + "Wee", + "658", + "effected", + "gger", + "installments", + "imilar", + "FU", + "infertility", + "climate", + "HEAD", + "fashion", + "THEY", + "jc", + "satisf", + "Guidelines", + "insure", + "RSA", + "virt", + "interpre", + "Joshua", + "Shut", + "testimonies", + "Ñģ", + "untary", + "417", + "beck", + "Milky", + "ç", + "sequels", + "281", + "Ribbon", + "roomm", + "synchron", + "452", + "1926", + "hawk", + "Disorder", + "backstory", + "Num", + "overheard", + "technical", + "Jud", + "aii", + "decon", + "Rape", + "Warrant", + "poop", + "spir", + "Country", + "weld", + "abuser", + "------", + "material", + "preserves", + "spring", + "puzzled", + "Debate", + "Joseph", + "272", + "Blood", + "antry", + "converge", + "imaginable", + "oward", + "545", + "fug", + "Vision", + "075", + "adoptive", + "unknow", + "Stream", + "affili", + "PUR", + "Wally", + "gamer", + "fart", + "stice", + "congen", + "н", + "685", + "orst", + "ATF", + "ml", + "Mozilla", + "calmed", + "bage", + "Vault", + "arkable", + "Guan", + "clueless", + "umatic", + "shameless", + "preached", + "misconceptions", + "anthology", + "biomass", + "Ps", + "tails", + "excessively", + "extr", + "Davis", + "grounding", + "shortcuts", + "Shift", + "Rew", + "Illum", + "incite", + "sense", + "Scouting", + "otos", + "respond", + "beware", + "gran", + "XV", + "JM", + "Sounders", + "276", + "shockingly", + "gastrointestinal", + "erences", + "df", + "NG", + "discredited", + "demoral", + "gladly", + "Tal", + "Predator", + "708", + "doi", + "decentral", + "illin", + "printed", + "inflicting", + "ribes", + "supper", + "abc", + "graz", + "980", + "Bull", + "millionaires", + "vanity", + "imony", + "biologists", + "alternating", + "sleeps", + "Force", + "Princ", + "Transgender", + "314", + "Provide", + "enthal", + "plum", + "resurrect", + "CW", + "injure", + "Perspective", + "Bei", + "restless", + "aciously", + "chlor", + "catch", + "Luigi", + "inconsistency", + "whiff", + "Arizona", + "ustration", + "Raid", + "Demons", + "Vita", + ":\"", + "migraine", + "Hamb", + "widget", + "451", + "randomized", + "etchup", + "Particularly", + "diced", + "perfected", + "roid", + "710", + "reflections", + "antioxidants", + "Label", + "326", + "igious", + "Eucl", + "608", + "strand", + "Dirt", + "Lift", + "suits", + "Controls", + "RAW", + "cowardly", + "Umb", + "Growing", + "mington", + "339", + "Commit", + "nonviolent", + "contaminants", + "acrylic", + "MAP", + "269", + "degrading", + "miracles", + "Establishment", + "despite", + "cry", + "pauses", + "mythical", + "twenties", + "Actually", + "phan", + "recorded", + "unwillingness", + "engineering", + "avored", + "devout", + "item", + "bunny", + "Merchants", + "consumes", + "508", + "lex", + "Clause", + "checklist", + "Sus", + "uther", + ".#", + "Bit", + "uay", + "bf", + "populace", + "316", + "combust", + "nano", + "popul", + "Indust", + "capitalists", + "Files", + "Bang", + "kosher", + "atile", + "incrim", + "OVER", + "melee", + "ymph", + "Pupp", + "evin", + "Molecular", + "misinterpret", + "vc", + "olithic", + "Simpsons", + "shrew", + "selectively", + "Drain", + "mittedly", + "conservative", + "True", + "Using", + "562", + "apon", + "apprentice", + "Mas", + "Battlefield", + "fing", + "concoct", + "VIS", + "Huss", + "detects", + "Friedrich", + "latitude", + "Custom", + "Ù", + "Bones", + "whose", + "redirected", + "aligned", + "Neighbor", + "Amen", + "Marble", + "Beyond", + "biomark", + "erroneous", + "Atlanta", + "masturb", + "Associ", + "Albert", + "cigar", + "Fraz", + "ethe", + "skinned", + "Ford", + "throp", + "Acc", + "tricked", + "overwhelm", + "implements", + "GeForce", + "bounces", + "moderator", + "910", + "Butterfly", + "Illegal", + "Subject", + "RET", + "Freeze", + "Newt", + "uterus", + "696", + "267", + "tk", + "dodged", + "liam", + "parasite", + "obal", + "Hubble", + "theology", + "âĢĶ\"", + "height", + "Ale", + "employment", + "Wallet", + "cessive", + "404", + "similarity", + "zens", + "dumps", + "depress", + "lifeless", + "535", + "oard", + "Scotland", + "believable", + "calculator", + "Naked", + "remission", + "oranges", + "Sections", + "entangled", + "uncanny", + "teaspoons", + "vr", + "Porn", + "Organ", + "bund", + "Doug", + "GHz", + "Major", + "abus", + "Bell", + "avier", + "implanted", + "RON", + "Fle", + "462", + "509", + "goggles", + "manuscript", + "NOT", + "Canaveral", + "DID", + "Season", + "HAEL", + "Edge", + "appiness", + "DIS", + "plotted", + "wrought", + "quarantine", + "rearr", + "itage", + "socket", + "brig", + "unbelievably", + "abytes", + "TG", + "444", + "Offic", + "acquaintances", + "Comparison", + "Nine", + "Feast", + "758", + "YC", + "finer", + "Strawberry", + "eternity", + "liament", + "urrency", + "Cortana", + "Sabbath", + "sprinkle", + "unker", + "UE", + "flies", + "blender", + "acutely", + "emark", + "Affect", + "Politics", + "sane", + "corrosion", + "spirituality", + "redeemed", + "ingrained", + "manager", + "joined", + "Dumb", + "Height", + "seventeen", + "640", + "reviewer", + "wallpaper", + "nurs", + "subset", + "703", + "symbolism", + "dudes", + "mismatch", + "gans", + "please", + "KE", + "atom", + "004", + "ionic", + "servings", + "proxies", + "transcription", + "yx", + "bowl", + "iscovery", + "Scotch", + "brace", + "riter", + "Desktop", + "limestone", + "æ", + "Neg", + "013", + "formulas", + "eval", + "zombies", + "GU", + "Hermes", + "brist", + "Mand", + "mastery", + "governs", + "construed", + "region", + "emitted", + "Vice", + "060", + "Jennifer", + "mol", + "jealousy", + "ingenuity", + "bug", + "olitical", + "perce", + "Sapp", + "dim", + "utral", + "interrogated", + "Gate", + "amber", + "911", + "Everyday", + "DDR", + "Blades", + "nifty", + "murderers", + "presumption", + "Pitt", + "Div", + "Destination", + "having", + "prolifer", + "breaker", + "BW", + "courier", + "Try", + "BUR", + "itized", + "compress", + "repetition", + "Tik", + "divergence", + "cube", + "everyone", + "Poles", + "418", + "Highly", + "468", + "Jeremy", + "contradictions", + "manure", + "Sad", + "pletion", + "626", + "279", + "frivolous", + "Canaan", + "olor", + "incapac", + "Gentle", + "insomnia", + "Jing", + "688", + "Views", + "syll", + "486", + "antom", + "cog", + "aintain", + "DVDs", + "318", + "archy", + "reprodu", + "concedes", + "Brook", + "interpreting", + "extracting", + "ess", + "uning", + "Mathematics", + "iably", + "multit", + "Acts", + "iliated", + "Foreign", + "flaming", + "Coup", + "glitches", + "differentiation", + "ihadi", + "Drone", + "incompatible", + "asher", + "documented", + "agons", + "wark", + "shielding", + "Correct", + "romising", + "uned", + "conduit", + "Diablo", + "beginner", + "archived", + "smanship", + "TBD", + "digy", + "322", + "268", + "Tears", + "Priority", + "Italy", + "^", + "annot", + "different", + "Joy", + "breathed", + "heon", + "racists", + "vascular", + "Between", + "etition", + "Likely", + "icans", + "529", + "Monsters", + "agy", + "Orange", + "hide", + "SIM", + "deceive", + "DAR", + "shattering", + "ow", + "peak", + "preferable", + "piping", + "LEDs", + "COMMUN", + "Construct", + "008", + "dissatisfied", + "KNOW", + "Frame", + "Toast", + "adore", + "history", + "Soviet", + "reporting", + "266", + "pract", + "Sauce", + "686", + "ievers", + "Domain", + "ousand", + "768", + "Cos", + "609", + "432", + "transl", + "oof", + "292", + "Turkish", + "POLIT", + "Harris", + "bj", + "rodents", + "556", + "intellectuals", + "interoper", + "ixt", + "unbiased", + "itia", + "504", + "buttocks", + "Flam", + "chrom", + "259", + "shock", + "RJ", + "Lich", + "422", + "condom", + "phen", + "vigilante", + "owl", + "dwellings", + "archaeologists", + "680", + "RAY", + "1921", + "625", + "PLAN", + "alde", + "030", + "abbling", + "Wave", + "Ni", + "furthe", + "JS", + "psycho", + "François", + "undergrad", + "successors", + "padded", + "introdu", + "reasoned", + "vas", + "creen", + "onsequ", + "starter", + "Court", + "HIS", + "plaster", + "ranger", + "298", + "esters", + "glare", + "ype", + "compute", + "Ali", + "mallow", + "masculine", + "Examination", + "improve", + "declass", + "decoration", + "FIG", + "abre", + "stale", + "abling", + "Rusty", + "ASAP", + "adjusts", + "bluff", + "density", + "disse", + "censor", + "ervatives", + "kettle", + "skeptics", + "fd", + "Imm", + "461", + "advantageous", + "419", + "Presents", + "482", + "Rewards", + "overshadow", + "Alabama", + "CPC", + "sock", + "Churches", + "hidden", + "cringe", + "HOR", + "PB", + "Pretty", + "Hong", + "?),", + "687", + "grocer", + "472", + "565", + "itent", + "partake", + "wait", + "usters", + "cones", + "concurrently", + "levers", + "aroma", + "Drill", + "498", + "804", + "ithering", + "355", + "legion", + "vitri", + "condu", + "Angel", + "OWER", + "{*", + "Simon", + "synthesis", + "Container", + "sheet", + "Bi", + "Raspberry", + "328", + "anders", + "Blossom", + "FINAL", + "acid", + "borderline", + "Aut", + "originate", + "transm", + "buffalo", + "atial", + "Craigslist", + "credential", + "disbanded", + "unprotected", + "Zer", + "waukee", + "diagn", + "1999", + "doc", + "ellig", + "warheads", + "ADS", + "verified", + "HAM", + "785", + "Cu", + "enorm", + "Skill", + "\\", + "bashing", + "loudspe", + "during", + "debunked", + "adequ", + "uh", + "Feed", + "ificial", + "pred", + "Passing", + "Kyle", + "enance", + "Mex", + "itect", + "cavern", + "trop", + "Eliot", + "753", + "encountering", + "sulf", + "Always", + "Gest", + "additive", + "278", + "loops", + "liberal", + "urion", + "Refresh", + "Dynasty", + "sweaty", + "sails", + "protection", + "Rooms", + "EXT", + "few", + "Paid", + "377", + "colonialism", + "chuckle", + "armour", + "softly", + "661", + "Building", + "AMER", + "babe", + "shif", + "Sem", + "disembark", + "Substance", + "Stone", + "dialect", + "Aph", + "spreadsheet", + "ierra", + "lineage", + "Cust", + "Babe", + "wra", + "Mafia", + "flakes", + "EVER", + "cong", + "Creation", + "loo", + "Ampl", + "Spectre", + "012", + "geons", + "swarm", + "Pale", + "Seek", + "itures", + "arri", + "redistribution", + "campaign", + "Ability", + "579", + "ournament", + "locks", + "nests", + "Constantine", + "whisper", + "shrouded", + "changed", + "Enhanced", + "920", + "glob", + "Tam", + "outwe", + "illiter", + "surg", + "Nap", + "Aerial", + "iferation", + "Egypt", + "ERO", + "antip", + "environment", + "machine", + "rupture", + "treatment", + "internal", + "infiltrate", + "gratification", + "Uber", + "unequal", + "flav", + "Lord", + "tein", + "LOT", + "bullshit", + "originals", + "minced", + "multiply", + "ayson", + "recomm", + "receptors", + "flashlight", + "inhuman", + "Future", + "puzzling", + "routers", + "uncontroll", + "responsible", + "cellul", + "Tablet", + "bolted", + "permissible", + "adra", + "picture", + "ODY", + "BRE", + "Iraq", + "Total", + "rising", + "273", + "nv", + "327", + "alysed", + "infect", + "1912", + "VT", + "Lazarus", + "ictive", + "Bu", + "NEVER", + "CODE", + "Modified", + "fetched", + "Trap", + "mob", + "upkeep", + "WARD", + "brewed", + "saliva", + "1923", + "steroid", + "rather", + "VER", + "contextual", + "Ont", + "LSD", + "agine", + "audible", + "Meta", + "erek", + "aults", + "Ottoman", + "Includes", + "occ", + "678", + "ipple", + "contrasted", + "014", + "Lenin", + "omega", + "885", + "civil", + "overload", + "},\"", + "programmers", + "geometry", + "?).", + "shift", + "Clancy", + "nr", + "verb", + "760", + "staggered", + "Playing", + "Smile", + "complains", + "Sloven", + "disobedience", + "creator", + "ly", + "incoln", + "emp", + "crate", + "Pledge", + "GPUs", + "protected", + "Vo", + "medium", + "acet", + "603", + "478", + "469", + "Further", + "sensed", + "Lock", + "crabs", + "Chains", + "NEO", + "experimented", + "Rhythm", + "802", + "hormonal", + "491", + "Median", + "evaluates", + "ippi", + "removable", + "vector", + "ilant", + "TERN", + "purch", + "Bind", + "athering", + "cords", + "Lib", + "damned", + "orc", + "Everywhere", + "gorilla", + "ystem", + "fail", + "ecstasy", + "allion", + "Sea", + "uploading", + "Specific", + "reinforcement", + "cerned", + "Dollars", + "Twenty", + "OX", + "ADD", + "braces", + "raven", + "1890", + "circulate", + "udden", + "Disney", + "Nope", + "Bagg", + "Buddha", + "rael", + "urus", + "Karma", + "curl", + "flips", + "bearer", + "misunderstand", + "abras", + "Assassin", + "Fact", + "interf", + "vantage", + "Genocide", + "deducted", + "Sep", + "McC", + "Jessica", + "Backup", + "Ian", + "urnal", + "laborers", + "438", + "Continuous", + "NBN", + "Cool", + "mitting", + "Normandy", + "purchaser", + "acquainted", + "blogging", + "route", + "marine", + "startled", + "6000", + "Radical", + "kiss", + "Blitz", + "express", + "601", + "hent", + "tink", + "pires", + "launch", + "sg", + "Effects", + "stiffness", + "Allies", + "thirsty", + "myst", + "logger", + "stances", + "Evaluation", + "090", + "proclaiming", + "hypocritical", + "496", + "caus", + "Kappa", + "Lann", + "Scientist", + "empath", + "etrical", + "lege", + "Hom", + "Aud", + "Colors", + "Straw", + "each", + "Patron", + "nuance", + "send", + "ourney", + "Phen", + "amino", + "Seconds", + "Sn", + "Civ", + "conglomer", + "411", + "versely", + "487", + "prises", + "277", + "necessary", + "dope", + "Late", + "rake", + "Brigham", + "ogun", + "STATES", + "Gaal", + "intellig", + "glacier", + "destruct", + "Zucker", + "484", + "332", + "Arist", + "protagonists", + "graveyard", + "names", + "Pax", + "thresholds", + "Seeing", + "munitions", + "contradicts", + "684", + "529", + "Concent", + "Blessed", + "Hz", + "inhibit", + "shenanigans", + "Spear", + "overlay", + "ritis", + "ilus", + "variance", + "overpower", + "viol", + "erning", + "polarization", + "aito", + "GV", + "493", + "Keeping", + "paternity", + "Happiness", + "oops", + "sb", + "xit", + "ophysical", + "conclusive", + "Arch", + "miser", + "suffice", + "Stout", + "hrs", + "643", + "principled", + "azine", + "atorium", + "Fairy", + "infiltrated", + "Hier", + "MIA", + "inders", + "rebutt", + "xx", + "feats", + "izzle", + "780", + "668", + "repressive", + "Yugoslavia", + "sole", + "704", + "RPG", + "Troll", + "packing", + "Database", + "Velvet", + "RELEASE", + "ablish", + "smoking", + "Bottle", + "Fully", + "Lean", + "objectively", + "Founding", + "Classics", + "mosaic", + "473", + "rooft", + "centrally", + "dismissive", + "parasites", + "009", + "cursed", + "vex", + "econom", + "Bore", + "enery", + "Fundamental", + "Omni", + "489", + "714", + "foregoing", + "fragment", + "oros", + "070", + "Faust", + "sucking", + "node", + "righteous", + "Powered", + "426", + "HQ", + "chronically", + "BAL", + "prest", + "rapists", + "Relationship", + "CHR", + "linen", + "numerical", + "oters", + "iterations", + "ttes", + "ENTER", + "rabbi", + "hoard", + "merciless", + "robes", + "Spray", + "advers", + "ilantro", + "483", + "fungus", + "alcoholism", + "anasia", + "Cruiser", + "morals", + "cision", + "measures", + "sabot", + "recol", + "Saur", + "Error", + "mysteriously", + "sle", + "feminists", + "д", + "ackle", + "Marxist", + "selves", + "doorway", + "discard", + "bandits", + "Dive", + "ameless", + "TRY", + "gull", + "republican", + "sr", + "Dynamo", + "embryo", + "MENTS", + "LOW", + "319", + "gly", + "cowork", + "Coll", + "cris", + "Banana", + "reality", + "mobilization", + "unal", + "Updated", + "Crew", + "Gideon", + "vines", + "knitting", + "dag", + "Surv", + "vacc", + "impulses", + "Northern", + "nanop", + "allows", + "UTH", + "flashbacks", + "alsa", + "282", + "transmissions", + "Almighty", + "Office", + "Bride", + "Beasts", + "othy", + "Clouds", + "Dyn", + "Jolly", + "District", + "veget", + "antit", + "Smoking", + "hess", + "compose", + "religiously", + "HY", + "fluorescent", + "rame", + "Meier", + "SQ", + "benefit", + "Thirty", + "559", + "Cance", + "586", + "grouped", + "phys", + "rebellious", + "BASE", + "chid", + "582", + "Lessons", + "Wonderful", + "ODE", + "uctions", + "barbaric", + "rahim", + "635", + "cloves", + "NIH", + "ossession", + "Employ", + "liberate", + "Gro", + "magician", + "ountain", + "FORM", + "533", + "unpredict", + "rity", + "faked", + "plets", + "ppelin", + "Living", + "nearer", + "superiors", + "Ur", + "heroism", + "bearded", + "006", + "Cole", + "1970", + "sill", + "Reduce", + "OLOG", + "onel", + "Billy", + "Painter", + "ansas", + "intermediary", + "trump", + "Mith", + "otom", + "434", + "territ", + "Wa", + "suprem", + "Rh", + "liction", + "DEAD", + "bothers", + "503", + "frogs", + "sprinkled", + "nil", + "628", + "Private", + "KGB", + "overriding", + "deceived", + "698", + "idium", + "seeker", + "Final", + "subconscious", + "wom", + "cass", + "chicks", + "verifying", + "ective", + "inia", + "Detection", + "MH", + "fortable", + "ISPs", + "crumble", + "Recap", + "598", + "ummies", + "export", + "Irish", + "lil", + "Rapt", + "RIGHT", + "anecdotal", + "piercing", + "deck", + "Liber", + "Books", + "assassin", + "Tur", + "revolution", + "Sheep", + "Publishers", + "EMS", + "iosis", + "finder", + "Curiosity", + "ARB", + "Convers", + "IVES", + "clave", + "Chaos", + "Mim", + "Costume", + "twe", + "intim", + "757", + "berto", + "261", + "VPN", + "cribed", + "Verb", + "cb", + "axle", + "sandwic", + "Ice", + "Thermal", + "654", + "709", + "Pact", + "Ensure", + "izable", + "497", + "bloodstream", + "Aw", + "leakage", + "alleg", + "Melody", + "681", + "Austin", + "428", + "summarized", + "Defendants", + "Vader", + "Ê", + "1880", + "assemb", + "YOU", + "GREEN", + "jury", + "4000", + "venerable", + "computational", + "perpetuate", + "torpedo", + "aborted", + "rhetorical", + "Overt", + "acknowledgment", + "essment", + "IGN", + "Sheen", + "571", + "contag", + "cultiv", + "spawn", + "mess", + "Dur", + "vortex", + "ixties", + "Blow", + "Sum", + "Åį", + "Rom", + "Radeon", + "Fed", + "americ", + "Anth", + "antic", + "fortress", + "Cold", + "Predict", + "Fake", + "illuminate", + "Find", + "intellectually", + "gon", + "alker", + "invoice", + "IELD", + "fools", + "Ending", + "-(", + "alk", + "Controlled", + "purposefully", + "Chronic", + "rele", + "Ops", + "Party", + "ethnic", + "Specifications", + "ffee", + "Teach", + "ulas", + "enslaved", + "onomy", + "tenets", + "ammonia", + "1913", + "dripping", + "612", + "659", + "Sagan", + "inaccur", + "abol", + "LIKE", + "visualization", + "learn", + "anon", + "cipline", + "adaptations", + "waiter", + "nergy", + "507", + "DK", + "YD", + "pedest", + "Sense", + "Obst", + "resurrection", + "SPECIAL", + "Unlike", + "lia", + "persuasive", + "iatrics", + "ONEY", + "esthetic", + "494", + "zik", + "fract", + "Output", + "Bers", + "rozen", + "Revis", + "draconian", + "Words", + "asions", + "Clintons", + "CU", + "History", + "twilight", + "iform", + "displ", + "progress", + "IO", + "cannibal", + "Michelle", + "nerv", + "contexts", + "Horses", + "anatomy", + "Legislation", + "Bloody", + "unwittingly", + "inquired", + "Zip", + "Designs", + "irritating", + "unison", + "RG", + "aviour", + "pseudo", + "Venom", + "obscured", + "ner", + "uked", + "ORGE", + "momentarily", + "olyn", + "Syrian", + "microscopic", + "mistress", + "Less", + "awoke", + "tutor", + "esome", + "ollar", + "egg", + "UTE", + "Buzz", + "attainment", + "discriminating", + "::", + "525", + "azard", + "Brist", + "oras", + "veterin", + "jing", + "idon", + "Austral", + "arious", + "Grav", + "anol", + "Quran", + "bleach", + "588", + "Osw", + "differed", + "typ", + "SIL", + "failed", + "436", + "palms", + "Fail", + "idespread", + "chap", + "IMAGES", + "ACP", + "matched", + "jaws", + "MHz", + "Nik", + "Hume", + "OSH", + "presume", + "secut", + "Died", + "Breat", + "gins", + "prison", + "UR", + "ROS", + "isitions", + "pelvic", + "exclusive", + "522", + "689", + "FN", + "ener", + "dispers", + "cohorts", + "shut", + "Load", + "needs", + "azaki", + "inoa", + "Inside", + "usra", + "ighters", + "271", + "subordinate", + "HOL", + "Glow", + "incred", + "Madame", + "oats", + "deviation", + "Approach", + "narc", + "bart", + "bole", + "SHE", + "effects", + "ADA", + "muse", + "Squ", + "neuroscience", + "Values", + "engu", + "dosage", + "whispers", + "naughty", + "Farming", + "Recently", + "relapse", + "rentice", + "UGH", + "darkened", + "appings", + "Slaughter", + "Anim", + "overtly", + "poses", + "deficient", + "necks", + "Iron", + "physiological", + "Liang", + "lear", + "celestial", + "pistols", + "eyebrow", + "915", + "ratch", + "cephal", + "PSU", + "photograp", + "Gaul", + "uncontrolled", + "Joined", + "652", + "itory", + "274", + "GAN", + "imester", + "essional", + "Ø©", + "uncons", + "THER", + "paternal", + "Zero", + "ugen", + "538", + "ende", + "505", + "movie", + "Lind", + "scorn", + "ulty", + "pesky", + "8000", + "677", + "homophobia", + "ranch", + "narciss", + "Voyager", + "HELP", + "528", + "edly", + "detract", + "Hope", + "787", + "Merlin", + "grids", + "KI", + "Mu", + "Selected", + "select", + "Moder", + "Feet", + "rename", + "intensity", + "Wilson", + "414", + "leave", + "Ready", + "intuitive", + "meager", + "Franc", + "DH", + "rhy", + "Pillar", + "DOE", + "minist", + "Grave", + "isible", + "Ess", + "empt", + "patched", + "Abortion", + "rals", + "dow", + "crawled", + "igrate", + "Virginia", + "conting", + "orphans", + "Crimean", + "dyn", + "shadowy", + "sound", + "ailable", + "293", + "vm", + "accompanies", + "Meanwhile", + "JR", + "Directions", + "adolescence", + "penetrated", + "bars", + "Rev", + "Ta", + "Skywalker", + "Fires", + "concept", + "SIG", + "554", + "currently", + "----------------", + "WHITE", + "767", + "rors", + "PDF", + "casing", + "673", + "disapprove", + "1800", + "Weed", + "inhib", + "morbid", + "433", + "awfully", + "Ts", + "Maria", + "illusions", + "totalitarian", + "ollo", + "suppl", + "sarc", + "RGB", + "launcher", + "badass", + "Syd", + "scrape", + "CLA", + "circum", + "657", + "nucleus", + "Ukip", + "modem", + "Jou", + "adders", + "wiser", + "thereal", + "democr", + "Invalid", + "Mine", + "manifested", + "meat", + "MORE", + "Larry", + "acements", + "specimen", + "results", + "swallowing", + "pigeon", + "tons", + "Lose", + "quartz", + "intraven", + "412", + "alyst", + "engraved", + "client", + "ADV", + "Shared", + "rites", + "hysterical", + "HUM", + "Cow", + "orously", + "pleasures", + "democratic", + "amph", + "nib", + "rieg", + "calculates", + "frying", + "favorite", + "antim", + "Doom", + "monitor", + "Want", + "templates", + "558", + "iever", + "Photos", + ",,", + "Sync", + "confronts", + "kept", + "dt", + "ERROR", + "ETF", + "578", + "spor", + "718", + "ivation", + "Haskell", + "Ca", + "dick", + "civilized", + "blah", + "enough", + "occup", + "334", + "antically", + "584", + "Dolphin", + "Starts", + "fanatic", + "ت", + "imag", + "microbial", + "freedom", + "cult", + "wra", + "423", + "RIPT", + "601", + "BTC", + "atmeal", + "653", + "agogue", + "derives", + "Wolf", + "466", + "Susan", + "Passage", + "ARDS", + "Guy", + "Council", + "erotic", + "pure", + "Memories", + "Wikileaks", + "elines", + "anth", + "Capital", + "807", + "Eggs", + "cv", + "ctors", + "shatter", + "esteem", + "vity", + "Vulcan", + "effic", + "BELOW", + "platoon", + "Commun", + "oustic", + "Amy", + "Freedom", + "ppo", + "Ja", + "Conan", + "insepar", + "scene", + "urinary", + "gain", + "Hillary", + "TAM", + "Hist", + "mechan", + "Robots", + "Leader", + "cartridges", + "whistleblowers", + "SPL", + "Labour", + "unction", + "faithfully", + "coarse", + "synth", + "LV", + "justifying", + "439", + "Victoria", + "Proceedings", + "alogy", + "morph", + "cove", + "laughable", + "ECA", + "670", + "aturated", + "Souls", + "Sleeping", + "Ly", + "Retro", + "astroph", + "seism", + "atherine", + "Hercules", + "fuse", + "HL", + "unintentionally", + "Ré", + "iery", + "conco", + "eras", + "recent", + "launchers", + "Volcano", + "Jace", + "terminating", + "Ide", + "zee", + "asonic", + "itone", + "nutshell", + "bip", + "dies", + "286", + "nood", + "Fathers", + "alys", + "theor", + "???", + "548", + "674", + "efined", + "806", + "âĻ", + "697", + "decap", + "FN", + "bureaucr", + "Goat", + "Shang", + "semin", + "throats", + "moth", + "herer", + "Democratic", + "ixtures", + "impl", + "Logo", + "ortunate", + "clumsy", + "innocuous", + "Blend", + "abulary", + "Faces", + "pornographic", + "px", + "Information", + "fluoride", + "atroc", + "delta", + "whatever", + "ossier", + "Noir", + "Yao", + "551", + "undred", + "millennium", + "feral", + "convinc", + "cano", + "imsy", + "angles", + "sterile", + "Menu", + "779", + "Crack", + "abundantly", + "mL", + "infiltration", + "Definition", + "733", + "oubt", + "orbital", + "piss", + "beet", + "679", + "counteract", + "ALE", + "ulative", + "crew", + "liberating", + "Dull", + "Speaking", + "Sadly", + "misfortune", + "dolphin", + "557", + "bould", + "Torah", + "Confederacy", + "421", + "orbits", + "ocused", + "beer", + "Rand", + "ORIG", + "muc", + "LER", + "Misty", + "inexpl", + "reptiles", + "aven", + "blocking", + "PASS", + "arisen", + "Mock", + "ops", + "shin", + "524", + "digestion", + "Soft", + "irect", + "POL", + "Spell", + "Level", + "hex", + "bitcoins", + "Hungry", + "VL", + "Realm", + "RELATED", + "Delta", + "Pri", + "rejoice", + "Latter", + "LG", + "stupidity", + "donkey", + "nova", + "Vill", + "decomp", + "externally", + "sequest", + "815", + "shortcut", + "riminal", + "Hun", + "EH", + "regiment", + "Case", + "definition", + "appendix", + "Played", + "associated", + "izens", + "Vag", + "flung", + "fru", + "coil", + "________________________", + "selects", + "solves", + "aea", + "985", + "Tomorrow", + "sear", + "APE", + "492", + "enlightened", + "nonexistent", + "Potato", + "Ghost", + "richness", + "Karin", + "familial", + "JA", + "Regardless", + "epis", + "GD", + "insanely", + "Phill", + "Block", + "Finding", + "omal", + "decipher", + "Swap", + "derived", + "OFFIC", + "Support", + "nylon", + "exaggeration", + "evangelicals", + "bearings", + "587", + "locale", + "powerfully", + "appropriated", + "itates", + "irlfriend", + "cule", + "Somewhere", + "747", + "Interesting", + "464", + "elong", + "degrade", + "rafted", + "tutorials", + "905", + "Intervention", + "uniqueness", + "284", + "explorers", + "nucle", + "Millenn", + "511", + "Reneg", + "execut", + "urai", + "leon", + "deserts", + "Cig", + "suggestive", + "instead", + "lousy", + "enigmatic", + "594", + "Know", + "rollment", + "ipher", + "humanities", + "modifying", + ".....", + "degraded", + "suppressing", + "eman", + "abouts", + "functional", + "OU", + "Relax", + "786", + "esses", + "Login", + "spec", + "WWF", + "364", + "Isis", + "Wisconsin", + "equival", + "Collector", + "ibilities", + "malink", + "acea", + "chained", + "arist", + "disadvantages", + "Brus", + "limits", + "Dmit", + "544", + "Recipe", + "habitual", + ".):", + "PRODUCT", + "772", + "rept", + "pathology", + "resurrected", + "uders", + "lingu", + "denomination", + "firewall", + "scient", + "valiant", + "Kansas", + "516", + "contemporaries", + "Roman", + "accompan", + "antennas", + "Xan", + "electromagnetic", + "Nek", + "alien", + "indle", + "graphene", + "graceful", + "syn", + "Bosh", + "1908", + "succumb", + "Technology", + "toxin", + "myra", + "essert", + "Hell", + "Gil", + "diarr", + "imeters", + "explo", + "geometric", + "Navigation", + "cern", + "programmer", + "oÄŁan", + "dodging", + "LU", + "573", + "inters", + "serum", + "uber", + "manga", + "762", + "Occasionally", + "437", + "Theme", + "immature", + "activating", + "Truly", + "د", + "osion", + "Age", + "TIME", + "Silver", + "sand", + "ulnerable", + "cram", + "Large", + "Anger", + "icators", + "431", + "Honest", + "zip", + "dism", + "fades", + "Pik", + "Ast", + "sequent", + "unsigned", + "xious", + "creation", + "395", + "ottenham", + "undesirable", + "ugal", + "Divide", + "lp", + "563", + "POP", + "CET", + "session", + "occurrences", + "chu", + "ACS", + "Prosecut", + "hypnot", + "rely", + "ERG", + "Ven", + "Republicans", + "inez", + "Implementation", + "sprang", + "obs", + "Defense", + "unexpl", + "PAGE", + "Tent", + "Neurolog", + "intuition", + "759", + "terrestrial", + "morphine", + ".\"", + "Hydra", + "651", + "neoliberal", + "683", + "abnormalities", + "quant", + "monastery", + "jac", + "Reaction", + "contraceptive", + "Balls", + "apost", + "676", + "HELL", + "approximately", + "vibrations", + "COR", + "CPUs", + "contin", + "semblance", + "shorth", + "tip", + "Chips", + "makes", + "prett", + "conspicuous", + "Amp", + "visualize", + "Hu", + "sorry", + "nai", + "Arcade", + "rimination", + "obin", + "vampire", + "773", + "Caucasus", + "Medic", + "GitHub", + "Wicked", + "Fet", + "Krist", + "998", + "frontal", + "283", + "ndum", + "idols", + "MSG", + "Shuttle", + "Towards", + "saturation", + "®", + "cradle", + "eteen", + "prejudices", + "separ", + "Soda", + "ynam", + "nause", + "penetrating", + "Vampire", + "mole", + "google", + "earance", + "583", + "domin", + "727", + "Kind", + "cust", + "manuel", + "Astro", + "Roger", + "JO", + "killed", + "Disapp", + "833", + "EQU", + "precedence", + "mberg", + "641", + "Roller", + "specifying", + "035", + "phil", + "powdered", + "blot", + "deline", + "Bruce", + "536", + "pim", + "leasing", + "vacc", + "RN", + "spacing", + "hangar", + "Plot", + "537", + "legraph", + "596", + "polyg", + "doi", + "Nerd", + "installed", + "Seeds", + "Plays", + "Romance", + "layer", + "unsu", + "curric", + "Mi", + "restrial", + "Niño", + "Proper", + "pores", + "Giving", + "aeus", + "Middle", + "liber", + "combatants", + "Bulk", + "502", + "stru", + "Lonely", + "Companies", + "inence", + "Autom", + "fearsome", + "summar", + "rotated", + "PLA", + "FAT", + "572", + "Skies", + "iour", + "intimately", + "amera", + "475", + "623", + "irrig", + "boosters", + "transmitting", + "DOWN", + "Able", + "furiously", + "spirit", + "grun", + "bible", + "Admir", + "§", + "Raise", + "flowering", + "uxe", + "ravis", + "urther", + "Scientology", + "pathy", + "ruth", + "tempor", + "whispered", + "ogly", + "coord", + "chlor", + "processing", + "iott", + "TY", + "wik", + "abolic", + "Unable", + "Literary", + "pH", + "Eastern", + "Craig", + "Fear", + "inventions", + "Nost", + "afflicted", + "Swamp", + "INST", + "Jerry", + "prope", + "Lancet", + "refres", + "Principles", + "Lys", + "ERAL", + "addock", + "cynicism", + "massacres", + "roo", + "collagen", + "Johnny", + "Keith", + "Italian", + "553", + "Dad", + "Neither", + "cler", + "ilers", + "assass", + "Travel", + "672", + "eaves", + "ATOR", + "oily", + "581", + "ateful", + "728", + "chiefly", + "tical", + "enes", + "Wouldn", + "Jacket", + "Suit", + "industrialized", + "Nose", + "SECTION", + "redd", + "cavity", + "conn", + "Shield", + "tongues", + "succinct", + "views", + "MUST", + "oliath", + "limitless", + "apocalyptic", + "Atlantis", + "DNA", + "ilded", + "Dresden", + "nit", + "subdiv", + "gressive", + "701", + "hops", + "alist", + "unintentional", + "psychic", + "controvers", + "foreground", + "naïve", + "folders", + "icist", + "drawbacks", + "Toxic", + "ophy", + "Masonic", + "cis", + "olated", + "depletion", + "Rap", + "692", + "inver", + "FAQ", + "meanings", + "bisc", + "Rage", + "resear", + "Ep", + "unbeat", + "Components", + "bub", + "Interface", + "Isa", + "Argon", + "denomin", + "mammal", + "519", + "sizing", + "imbabwe", + "Replacement", + "Georgia", + "Participation", + "melts", + "femin", + "514", + "seams", + "513", + "Gaw", + "brood", + "Mit", + "annoyance", + "equilibrium", + "patri", + "338", + "561", + "mentioned", + "Votes", + "intoler", + "strikingly", + "352", + "skeletal", + "616", + "isition", + "fluor", + "provided", + "517", + "climates", + "sensibilities", + "Frequ", + "onite", + "Kenn", + "magnets", + "assis", + "prerequisite", + ">>>", + "scree", + "google", + "Mirage", + "evict", + "Peace", + "missionaries", + "617", + "748", + "rient", + "STATS", + "Bird", + "Shiva", + "Blessing", + "redundancy", + "photoc", + "Ones", + "754", + "alert", + "urous", + "folklore", + "Ideal", + "sheets", + "according", + "Hor", + "Cle", + "Edit", + "671", + "olitics", + "ESC", + "paraly", + "orgasm", + "speak", + "ð", + "sneaky", + "swords", + "fandom", + "776", + "Scandinav", + "darts", + "546", + "cerpt", + "Gifts", + "magically", + "phys", + "Laughs", + "Sour", + "ources", + "789", + "Eps", + "ository", + "uality", + "literally", + "heavens", + "FUL", + "ie", + "ISP", + "wink", + "weeping", + "docking", + "ACY", + "iece", + "signifies", + "guns", + "Sac", + "Leave", + "imation", + "unex", + "uctive", + "Fees", + "Portable", + "Investigator", + "pill", + "rehensible", + "potency", + "803", + "embodiment", + "overty", + "shine", + "REL", + "MPH", + "Patriarch", + "aspirin", + "rinse", + "inher", + "ograms", + "THREE", + "qt", + "ipples", + "dehuman", + "slander", + "flora", + "brow", + "blindly", + "ectar", + "endish", + "pigment", + "cellent", + "yells", + "Lust", + "Attacks", + "Syndicate", + "otin", + "gress", + "reenshot", + "picking", + "acupuncture", + "images", + "glas", + "Policies", + "intestinal", + "1998", + "ULE", + "runs", + "Ning", + "Asuka", + "Skull", + "Motor", + "defund", + "attaching", + "BAD", + "quarrel", + "Child", + "Dog", + "issan", + "irmation", + "inline", + "Lover", + "cyan", + "entary", + "awareness", + "traveller", + "âĢIJ", + "beasts", + "boobs", + "Deadly", + "plutonium", + "Intellectual", + "Jam", + "consec", + "663", + "Vegan", + "331", + "uron", + "HEL", + "reements", + "clone", + "outputs", + "oult", + "DOM", + "NX", + "Ze", + "909", + "brate", + "arations", + "Jindal", + "booklet", + "amide", + "scraping", + "Sol", + "Date", + "796", + "fulf", + "skeletons", + "saints", + "Curious", + "Han", + "repud", + "osity", + "Gravity", + "metadata", + "Focus", + "thrott", + "Programming", + "Break", + "erver", + "knight", + "yrs", + "376", + "sat", + "auto", + "broom", + "nerd", + "Political", + "022", + "-------------", + "oulos", + "relic", + "enactment", + "rious", + "Uniform", + "Teen", + "Colorado", + "055", + "angled", + "bolt", + "Neander", + "Dism", + "thanks", + "Polit", + "ersion", + "dro", + "install", + "Jake", + "hz", + "770", + "Commodore", + "lahoma", + "shri", + "....", + "7000", + "scope", + "genesis", + "resided", + "Rivals", + "sarcastic", + "elicit", + "multiplied", + "uitous", + "oppress", + "PROT", + "perpetually", + "Adds", + "buffers", + "mush", + "354", + "presc", + "Kung", + "682", + "Education", + "pled", + "bsp", + "confessions", + "revocation", + "Micro", + "Hobby", + "Fatal", + "STAR", + "workspace", + "transformations", + "portals", + "orned", + "figured", + "linguistic", + "pperc", + "ergus", + "Fel", + "Intent", + "289", + "delinquent", + "handwriting", + "vap", + "576", + "redited", + "736", + "psychiatry", + "GMT", + "disingen", + "crou", + "801", + "malice", + "itutes", + "Tiff", + "stink", + "574", + "Story", + "Modern", + "Gly", + "Jamie", + "advertis", + "hiber", + "infiltr", + "elector", + "rovers", + "Fist", + "peed", + "Classical", + "592", + "conscientious", + "Surv", + "Text", + "Drunk", + "supplemented", + "THIS", + "timid", + "stacking", + "rites", + "rebirth", + "balcon", + "yawn", + "rosc", + "axy", + "Hart", + "OPER", + "996", + "rabid", + "Tick", + "grinning", + "elfth", + "045", + "justifies", + "Pirate", + "Salary", + "mirac", + "613", + "inately", + "LIN", + "inadequ", + "NPR", + "iddled", + "storage", + "seventy", + "onet", + "gastro", + "FIR", + "rodent", + "629", + "Include", + "Categories", + "Literally", + "pree", + "aunder", + "LOL", + "694", + "indef", + "Ped", + "menstru", + "censored", + "configure", + "overest", + "igenous", + "rectangular", + "MIS", + "Mub", + "witches", + "izards", + "obnoxious", + "Loll", + "SEM", + "spiritually", + "coer", + "modesty", + "butt", + "edits", + "Shall", + "sburgh", + "1911", + "Rex", + "manent", + "Lithuan", + "pointers", + "ativity", + "retch", + "cascade", + "Ragnarok", + "Painting", + "ATL", + "Born", + "padding", + "whel", + "grotesque", + "theorists", + "forcer", + "Jinn", + "renal", + "jamin", + "FEC", + ".\"\"", + "redict", + "oppos", + "opted", + "Sel", + "ipment", + "752", + "792", + "Pur", + "volt", + "flap", + "CASE", + "dyed", + "orers", + "becca", + ",.", + "ifice", + "ubes", + "yr", + "DW", + "alteration", + "Simpl", + "unequiv", + "756", + "Dou", + "plunder", + "commons", + "stag", + "Zeal", + "avanaugh", + "Self", + "none", + "EGIN", + "flashback", + "VAL", + "Gab", + "Capture", + "Brilliant", + "Disk", + "Mood", + "haun", + "rotting", + "Cobra", + "psychopath", + "helper", + "Starting", + "Orbit", + "caf", + "Half", + "Volume", + "aptop", + "Saga", + "azor", + "593", + "774", + "Caucasian", + "compan", + "VERY", + "GES", + "vomit", + "dispro", + "Mechanics", + "385", + "mystical", + "AFTA", + "bacter", + "availability", + "hairc", + "Vec", + "rypt", + "manipulative", + "shell", + "Weird", + "jab", + "Byr", + "Bow", + "uin", + "quot", + "MX", + "960", + "Sharia", + "Weapon", + "PowerPoint", + "stitching", + "constraint", + "âľ", + "ulic", + "597", + "omedical", + "Supplemental", + "Surve", + "Subcommittee", + "Darkness", + "python", + "LU", + "402", + "Quan", + "Moderate", + "clusively", + "extrap", + "latt", + "STUD", + "oslav", + "symb", + "battle", + "flash", + "Deploy", + "microbiome", + "ingested", + "distort", + "assimil", + "mobs", + "illet", + "Gre", + "294", + "forbids", + "Efficiency", + "Clan", + "763", + "dragons", + "States", + "MAKE", + "BOOK", + "Runs", + "UX", + "EED", + "Whoever", + "ionics", + "worldly", + "Mermaid", + "benz", + "Info", + "523", + "biod", + "Poison", + "ceivable", + "Services", + "ATIVE", + "Item", + "disav", + "heter", + "asteroids", + "Wooden", + "electroly", + "assadors", + "nance", + "reflect", + "attent", + "iphany", + "spaceship", + "begg", + "algia", + "Ax", + "idiosyncr", + "inserting", + "CSS", + "LET", + "Strikes", + "ossibly", + "Exp", + "Opp", + "dden", + "playable", + "JM", + "lawfully", + "Blink", + "413", + "overpowered", + "commenter", + "Track", + "methyl", + "fermented", + "invaders", + "Moves", + "communicates", + "rint", + "Tray", + "jug", + "superf", + "ochet", + "Jelly", + "estrogen", + "Dom", + "mix", + "Gun", + "ochemistry", + "952", + "overe", + "Plaintiff", + "Pilgrim", + "SERVICES", + "Expend", + "FRE", + "smelling", + "Spaces", + "bris", + "Mission", + "arter", + "autonom", + "Lisa", + "Percent", + "NK", + "Limits", + "356", + "Recent", + "Siberian", + "etermin", + "nets", + "Sword", + "essee", + "Ùĩ", + "icycle", + "paras", + "rud", + "scrib", + "1860", + "Shop", + "orld", + "pept", + "ENSE", + "animations", + "ership", + "Search", + "USSR", + "washed", + "promulg", + "detainee", + "underest", + "Appropri", + "Left", + "Update", + "Wallet", + "idently", + "Bicycle", + "gorge", + "abyte", + "Minecraft", + "rike", + "997", + "Tesla", + "Often", + "THESE", + "regression", + "Hen", + "snippets", + "irds", + "princes", + "wastes", + "Wond", + "itimate", + "Mongol", + "kW", + "idiots", + "foreigner", + "Upon", + "backdoor", + "umph", + "Squirrel", + "typed", + "blockers", + "Vote", + "Possibly", + "geist", + "TRANS", + "titan", + "VG", + "microbi", + "interacts", + "masc", + "finite", + "cutoff", + "ornings", + "prototyp", + "compan", + "mology", + "BOX", + "Cre", + "Bot", + "grading", + "PET", + "insidious", + "Franch", + "orians", + "AUT", + "Crush", + "589", + "question", + "anguard", + "absurdity", + "?\",", + "Hum", + "liberalism", + "postwar", + "Gener", + "Personally", + "889", + "Bul", + "lighthouse", + "291", + "VK", + "Exposure", + "subtract", + "ometime", + "arbon", + "Thieves", + "anus", + "Libertarian", + "Raw", + "solvent", + "corros", + "signific", + "scholarly", + "024", + "fetish", + "larvae", + "catast", + "traitor", + "ijing", + "Demand", + "math", + "conceivable", + "either", + "acl", + "Arrows", + "627", + "Frankenstein", + "entious", + "imitation", + "amn", + "STOP", + "cripp", + "zag", + "Zed", + "797", + "Along", + "wont", + "folds", + "Shar", + "Commentary", + "Libraries", + "Thunderbolt", + "itud", + "Toy", + "incidentally", + "Resp", + "ordinarily", + "vanish", + "acterial", + "Minnesota", + "rank", + "614", + "Exam", + "Got", + "snipers", + "ETHOD", + "dirty", + "igsaw", + "Obs", + "Authors", + "illustrating", + "782", + "864", + "blinded", + "transfer", + "spawning", + "Diary", + "DNS", + "CG", + "someone", + "cruc", + "Morgan", + "Learn", + "API", + "toc", + "STAT", + "Flame", + "aganda", + "Benef", + "stuff", + "SEA", + "incest", + "Normally", + "RU", + "arsenic", + "isine", + "TG", + "Type", + "regn", + "Cass", + "Touch", + "Site", + "pict", + "corrupted", + "729", + "nineteen", + "paraph", + "tavern", + "retard", + "Kaf", + "colleg", + "bucks", + "imum", + "Candle", + "Misc", + "Awesome", + "edited", + "DN", + "otomy", + "disclaimer", + "798", + "Goodbye", + "ucle", + "atom", + "Judge", + "cipl", + "inexplicable", + "iddler", + "781", + "empirical", + "Veter", + "ascert", + "aest", + "laz", + "binary", + "358", + "contained", + "multipl", + "ocado", + "delusional", + "aeros", + "udence", + "jargon", + "estine", + "arbitrarily", + "prick", + "BACK", + "amines", + "Mess", + "Knowing", + "ublic", + "Warfare", + "signify", + "fragmentation", + "Tex", + "nin", + "dise", + "882", + "hospital", + "volent", + "Need", + "infer", + "Sony", + "783", + "YING", + "infinity", + "Fortress", + "mustache", + "corresponds", + "DX", + "unmarried", + "Cruel", + "1901", + "appropri", + "ZI", + "phosph", + "901", + "IFE", + "347", + "convoluted", + "Apost", + "htm", + "illuminating", + "568", + "assassinate", + "param", + "impractical", + "cedes", + "Procedure", + "Mouth", + "Battle", + "451", + "Sand", + "contamin", + "Hour", + "Cell", + "BIL", + "precon", + "Scor", + "config", + "Muscle", + "hive", + "underworld", + "plement", + "postage", + "interpersonal", + "pierced", + "charms", + "oscopic", + "ASC", + "Dex", + "render", + "png", + "critiques", + "992", + "Vinyl", + "Bear", + "idia", + "Temp", + "cyn", + "BCE", + "patriarchal", + "antagonist", + "GMO", + "unnatural", + "Race", + "imeo", + "Ukrainians", + "Train", + "329", + "ritten", + "igil", + "Lin", + "alus", + "*****", + "olded", + "Pegasus", + "Bas", + "photos", + "820", + "squadron", + "ESE", + "373", + "Uk", + "Lost", + "Store", + "Scenes", + "JJ", + "lick", + "Tyler", + "cius", + "lishing", + "ocl", + "associ", + "ensitivity", + "entanyl", + "Rum", + "443", + "onding", + "pedals", + "Psychological", + "thro", + "Network", + "591", + "Pick", + "chords", + "Hound", + "entials", + "faces", + "Yin", + "ugi", + "bows", + "Forms", + "886", + "Ox", + "351", + "mating", + "chirop", + "916", + "expend", + "usefulness", + "Marvel", + "Stretch", + "omez", + "JS", + "Hal", + "fle", + "Countdown", + "LH", + "assian", + "vd", + "Transcript", + "Extrem", + "idine", + "ustainable", + "ederal", + "Owl", + "creed", + "Grateful", + "prenatal", + "________________________________", + "Elements", + "âĢ¦)", + "nesia", + "ARGET", + "boredom", + "depictions", + "verbal", + "eSports", + "Laura", + "ilage", + "Galactic", + "Investigators", + "scattering", + "instein", + "Experiment", + "Recre", + "regul", + "relent", + "STE", + "slicing", + "igans", + "raped", + "Deter", + "smoker", + "Wikimedia", + "pages", + "Ted", + "713", + "puberty", + "hars", + "Starter", + "patch", + "leeve", + "346", + "Accessories", + "ventions", + "STAND", + "Urug", + "Occupy", + "binds", + "Bubble", + "incorporation", + "stereotypical", + "gor", + "987", + "evils", + "tower", + "astronomer", + "Ble", + "Nid", + "Widow", + "paw", + "innoc", + "OWN", + "tofu", + "drops", + "Eval", + "693", + "Collins", + "penter", + "Nib", + "smokes", + "1850", + "techno", + "oooo", + "Unic", + "Kirin", + "\":[\"", + "increments", + "989", + "oodoo", + "Cyborg", + "cures", + "OW", + "Annex", + "behavior", + "/-", + "buggy", + "onent", + "Bey", + "summarize", + "putable", + "fri", + "Gi", + "urances", + "Appalach", + "hegemony", + "Origins", + "connectors", + "AST", + "object", + "Slay", + "Arm", + "oston", + "EVEN", + "prophecy", + "Bright", + "Vector", + "Marg", + "omical", + "Holy", + "RPM", + "Receiver", + "tracts", + "boss", + "blurry", + "aspx", + "DES", + "cess", + "Aster", + "anything", + "levard", + "unciation", + "jong", + "iv", + "Common", + "Distance", + "imus", + "outheast", + "cir", + "Cato", + "inscribed", + "ersed", + "anarchy", + "plagiar", + "thug", + "Actor", + "Tant", + "Researchers", + "remember", + "itch", + "refill", + "sucker", + "WANT", + "RAG", + "rencies", + "Tape", + "attaches", + "nb", + "Tan", + "append", + "alas", + "951", + "panel", + "Climate", + "icrobial", + "Brandon", + "Freud", + "fungi", + "commenters", + "Delicious", + "hitherto", + "conv", + "chemist", + "denominations", + "Behavior", + "comed", + "Lantern", + "Floating", + "magic", + "Barbar", + "bender", + "iliar", + "unny", + "retracted", + "atars", + "Lovely", + "infinitely", + "humili", + "interestingly", + "municip", + "Panic", + "comprehension", + "Massacre", + "persuasion", + "enf", + "coded", + "higher", + "chart", + "umbered", + "Indigo", + "thinker", + "goof", + "Petition", + "fascist", + "absor", + "assay", + "Classification", + "halluc", + "speech", + "issues", + "inexper", + "Libre", + "sling", + "zech", + "pouch", + "Offense", + "HF", + "Fight", + "026", + "Trident", + "fm", + "intox", + "465", + "colonial", + "ovies", + "794", + "Techn", + "undreds", + "childish", + "arenthood", + "Shade", + "Host", + "directional", + "reader", + "rimp", + "Eater", + "prep", + "meas", + "latch", + "inant", + "nels", + "finished", + "application", + "Board", + "filler", + "ivably", + "CAST", + "stereotyp", + "warranties", + "Probe", + "spontaneously", + "tropes", + "Meg", + "Handling", + "hemer", + "986", + "Sly", + "plates", + "molten", + "HIT", + "strings", + "centrif", + "ENG", + "Indeed", + "429", + "sly", + "490", + "hordes", + "boot", + "691", + "ihara", + "subversive", + "Russell", + "aceous", + "wk", + "reverence", + "ingenious", + "holiday", + "eligible", + "Tactical", + "978", + "herence", + "gimm", + "archaic", + "adam", + "297", + "Father", + "Lerner", + "hesitated", + "Safety", + "awakened", + "ueller", + "extrater", + "mummy", + "Buddhism", + "359", + "legions", + "prehistoric", + "ancouver", + "melancholy", + "Enemy", + "Syl", + "Robo", + "verting", + "Bullets", + "essler", + "marvelous", + "Bened", + "savior", + "omever", + "Bee", + "rapp", + "predomin", + "Scripture", + "snapshots", + "unrem", + "squid", + "Buddh", + "Santorum", + "Internet", + "avoid", + "unamb", + "296", + "nexus", + "interchangeable", + "ockets", + "foll", + "OPT", + "023", + "²", + "hereditary", + "vape", + "=\"", + "1996", + "س", + "Emergency", + "neb", + "isot", + "diam", + "stairs", + "Appendix", + "venient", + "invol", + "theorist", + "conqu", + "Mich", + "Sort", + "antasy", + "dating", + "771", + "ape", + "indemn", + "ween", + "Games", + "ascal", + "Muslims", + "leaflets", + "traverse", + "transgress", + "flushed", + "893", + "lasses", + "obos", + "ooming", + "tou", + "mast", + "âģ", + "751", + "Either", + "grate", + "urgy", + "endowed", + "Rasm", + "Nat", + "odka", + "olon", + "iants", + "sensations", + "situational", + "pox", + "Figure", + "slime", + "421", + "ollow", + "anesthesia", + "adult", + "Piece", + "994", + "Analog", + "Iv", + "flo", + "domest", + "cabal", + "garg", + "rabb", + "REC", + "ISTORY", + "Friend", + "ancestor", + "Lets", + "elf", + "lobb", + "Adren", + "silver", + "astical", + "stitch", + "028", + "Hug", + "moss", + "ompl", + "unob", + "883", + "cortex", + "olutely", + "052", + "Seattle", + "restling", + "endment", + "366", + "ventus", + "Rated", + "Clever", + "cloak", + "phrase", + "flake", + "philosophies", + "784", + "skulls", + "wake", + "oru", + "ACTION", + "comprom", + "Manufacturer", + "Improve", + "Ns", + "Revenge", + "lords", + "417", + "iddles", + "condesc", + "tiny", + "chloride", + "greg", + "REST", + "subject", + "undes", + "ftime", + "bottleneck", + "Zombie", + "habitable", + "cigars", + "enlarg", + "icester", + "ðĿ", + "regulation", + "arters", + "formulations", + "adhesive", + "344", + "pod", + "etitive", + "continuum", + "aghd", + "701", + "disband", + "Tu", + "civilisation", + "PCI", + "crooked", + "ammy", + "brim", + "Jr", + "Bunker", + "plot", + "wielded", + "caricature", + "Infinite", + "piracy", + "aretz", + "stares", + "incinnati", + "agents", + "ObamaCare", + "asuring", + "ansion", + "astonished", + "iovascular", + "Bio", + "advisable", + "sender", + "887", + "Led", + "DN", + "aggregation", + "Innocent", + "Transactions", + "worms", + "Worm", + "363", + "Biblical", + "rared", + "gazing", + "chant", + "subordinates", + "1600", + "actually", + "olition", + "RTX", + "Pyramid", + "alph", + "FPS", + "errone", + "LR", + "Scientists", + "incons", + "brittle", + "027", + "Bowser", + "Rub", + "links", + "Wik", + "ussion", + "Marsh", + "resents", + "Clean", + "brute", + "Inventory", + "1100", + "ATK", + "793", + "caveats", + "Knot", + "IRT", + "Canad", + "isma", + "entin", + "Own", + "455", + "lesions", + "Ares", + "Kali", + "paws", + "Auto", + "discrim", + "044", + "COUN", + "1905", + "experien", + "406", + "achelor", + "scarcely", + "synchronized", + "Rat", + "Blake", + "rewriting", + "cannons", + "stem", + "Apparently", + "leveling", + "?]", + "fins", + "Tone", + "ogether", + "Sound", + "microsc", + "Asylum", + "individuality", + "432", + "lease", + "Chuck", + "hating", + "leftists", + "Personality", + "Bundle", + "Dutch", + "transformer", + "iami", + "Tradition", + "Recipes", + "discour", + "Viol", + "Ext", + "Oliv", + "ashington", + "millennia", + "psychiatrists", + "Trilogy", + "inction", + "disliked", + "088", + "954", + "overloaded", + "opium", + "acus", + "resources", + "mud", + "ometry", + "Hit", + "guild", + "abyss", + "884", + "ensity", + "Difference", + "Electric", + "authent", + "downloadable", + "ellar", + "Savior", + "FRI", + "445", + "incidental", + "analogue", + "ounters", + "Builder", + "narration", + "ategor", + "raise", + "indoctr", + "Aren", + "baptism", + "obe", + "tubing", + "apsed", + "Fortunately", + "gered", + "Pict", + "mastering", + "HIM", + "Obesity", + "ornament", + "advant", + "Cous", + "032", + "cells", + "preclude", + "anecdote", + "patriarchy", + "Sending", + "Pie", + "depressive", + "Ends", + "712", + "zos", + "icka", + "1906", + "Anti", + "vana", + "Restrict", + "protr", + "username", + "parach", + "1997", + "imental", + "rower", + "carb", + "033", + "obligatory", + "willful", + "snail", + "json", + "izarre", + "miscar", + "dopamine", + "л", + "applic", + "nervously", + "YY", + "alez", + "Soviets", + "Mister", + "crates", + "heavenly", + "doct", + "048", + "2400", + "ivia", + "adies", + "Phone", + "asks", + "perenn", + "composing", + "raiding", + "requent", + "ibli", + "Feedback", + "cellaneous", + "Contracts", + "Casting", + "vim", + "Cut", + "abbrevi", + "intest", + "ricted", + "969", + "nostic", + "inverted", + "EG", + "aiden", + "Claud", + "iP", + "urized", + "Emily", + "353", + "((", + "ammad", + "Reb", + "plom", + "YES", + "connection", + "Wra", + "Merch", + "ether", + "Elizabeth", + "Chip", + "relevant", + "URA", + "antioxidant", + "Chron", + "theological", + "HCR", + "ruits", + "Body", + "enezuel", + "Few", + "adder", + "inducing", + "Darth", + "implicitly", + "overfl", + "relics", + "Must", + "Answers", + "retina", + "Slowly", + "Shib", + "software", + "\"\"", + "hack", + "Apart", + "told", + "Ger", + "Civil", + "problem", + "slang", + "tactile", + "tabl", + "Ascension", + "humankind", + "Howard", + "rescent", + "Releases", + "arijuana", + "Christopher", + "Warden", + "blogspot", + "Vari", + "idency", + "Handler", + "Round", + "MJ", + "rhyth", + "Tai", + "terson", + ",\"", + "portation", + "Orbital", + "fantas", + "attribut", + "diagram", + "atech", + "1992", + "ibl", + "Woman", + "ternally", + "Days", + "debunk", + "Phant", + "Oath", + "sharp", + "claws", + "Lots", + "Incre", + "Aff", + "hooting", + "rect", + "altru", + "wors", + "tho", + "349", + "clusions", + "pseudonym", + "Bec", + "phosphorus", + "ivic", + "348", + "otent", + "ub", + "coales", + "regate", + "1870", + "glide", + "treated", + "Symb", + "enchant", + "Besides", + "stocks", + "388", + "--------------", + "interpret", + "ouple", + "drawback", + "Revised", + "anat", + "psychosis", + "ب", + "diffuse", + "affidav", + "elve", + "amination", + "Tackle", + "hunter", + "env", + "chests", + "subter", + "conquest", + "fidelity", + "infringing", + "opathic", + "Grip", + "Keyboard", + "objectionable", + "metabol", + "Gö", + "Room", + "...)", + "KEN", + "assic", + "geop", + "Tro", + "cursing", + "dile", + "ultraviolet", + "inarily", + "distilled", + "sect", + "Shooter", + "uckles", + "distortions", + "Map", + "Doctor", + "installs", + "oire", + "starch", + "ociation", + "Lev", + "scripture", + "salient", + "ilitating", + "wb", + "Sov", + "Damn", + "Grey", + "980", + "jung", + "licking", + "029", + "Dian", + "Babylon", + "к", + "Romantic", + "guesses", + "Fren", + "Generally", + "ultural", + "istence", + "initi", + "341", + "Slave", + "ultan", + "Trash", + "Empty", + "Hundred", + "Directive", + "Anderson", + "Advertisement", + "RH", + "Oo", + "Hik", + "peg", + "Sup", + "XT", + "encrypt", + "selage", + "Throne", + "consecut", + "Li", + "Virus", + "Cookies", + "SHIP", + "flavorful", + "odynamics", + "animal", + "spread", + "IPCC", + "jobs", + "ernand", + "Haunted", + "intolerable", + "LAR", + "ixtape", + "neur", + "causal", + "Psychiatry", + "Vim", + "genomic", + "duration", + "Username", + "ategy", + "unic", + "KILL", + "blooded", + "caucuses", + "POLITICO", + "Spanish", + "obedience", + "inconven", + "MAT", + "bends", + "Improvements", + "relig", + "Forth", + "Lumia", + "uces", + "unim", + "Statistical", + "kb", + "auntlet", + "Disco", + "Instruction", + "ooo", + "Dictionary", + "culated", + "Adv", + "Avatar", + "ictional", + "centr", + "ifles", + "orks", + "skill", + "latex", + "Pagan", + "devast", + "prol", + "896", + "Product", + "968", + "french", + "083", + "Cluster", + "cloth", + "Filter", + "Disorders", + "etimes", + "instinctively", + "Britann", + "aft", + "Vict", + "âĺħ", + "perverse", + "contraceptives", + "Hannibal", + "escap", + "Apostle", + "Xiao", + "Magnum", + "phosphate", + "399", + "utable", + "sten", + "wearer", + "smug", + "Influence", + "384", + "Truth", + "struction", + "maniac", + "Magnetic", + "ousands", + "semen", + "dir", + "Tornado", + "explos", + "1995", + "Xi", + "Steel", + "057", + "Barn", + "Fan", + "Chatt", + "Chem", + "Fold", + "bees", + "1080", + "Maze", + "ierre", + "oeuv", + "Cand", + "odium", + "mmm", + "ereo", + "reactionary", + "acidic", + "Removal", + "nont", + "031", + "Terminator", + "Vendor", + "enemy", + "reconstructed", + "Galileo", + "testers", + "albeit", + "uminium", + "rite", + "Input", + "committee", + "jour", + "gements", + "germ", + "Dick", + "Requirements", + "omsday", + "Î", + "ISSION", + "molded", + "rye", + "Attorney", + "population", + "repet", + "Sync", + "breaks", + "banished", + "raspberry", + "ammo", + "orthodox", + "webcam", + "Asc", + "vl", + "1989", + "discipl", + "moreover", + "explodes", + "1960", + "propositions", + "Protect", + "sexes", + "physical", + "Athena", + "ocent", + "Gothic", + "Racial", + "istani", + "helium", + "Presumably", + "perman", + "becue", + "HW", + "rued", + "CNS", + "DEP", + "Manifest", + "2500", + "Myst", + "Economic", + "Prot", + "ledge", + "imitate", + "Totally", + "Beaut", + "OIL", + "1440", + "Moscow", + "Sets", + "merga", + "lesbians", + "Walker", + "Move", + "SOM", + "Psy", + "strument", + "iter", + "Tosh", + "oola", + "Antiqu", + "Shining", + "observational", + "VW", + "rophe", + "034", + "contiguous", + "starve", + "sure", + "negate", + "mindless", + "tf", + "downwards", + "046", + "riors", + "reverted", + "Athe", + "Bra", + "eah", + "Rachel", + "Hung", + "Join", + "Races", + "mutant", + "uncond", + "usability", + "NESS", + "haust", + "036", + "obscurity", + "imperialism", + "emitting", + "ideologically", + "Iro", + "erva", + "Izzy", + "Levels", + "onym", + "Conspiracy", + "Sapphire", + "Ul", + "huh", + "ochem", + "behaves", + "Mesh", + "Ark", + "vec", + "Actions", + "distinguishing", + "Tsarnaev", + "Endurance", + "ederation", + "itant", + "streetcar", + "041", + "Aval", + "Companion", + "Cartoon", + "calculus", + "993", + "eq", + "Vanilla", + "MAC", + "wolves", + "fg", + "fermentation", + "informants", + "sudo", + "peripher", + "indign", + "parts", + "detail", + "femin", + "blade", + "inserts", + "offsets", + "antidepressants", + "phr", + "resultant", + "biology", + "acquies", + "UFF", + "****************", + "Penalty", + "rever", + "heric", + "Shadows", + "command", + "reprint", + "089", + "empty", + "TAG", + "stim", + "FK", + "kins", + "uggle", + "imura", + "wit", + "Kill", + "Beck", + "Ocean", + "labyrinth", + "Norse", + "IENCE", + "+++", + "DoS", + "gm", + "barbar", + "Ceres", + "hashing", + "eworthy", + "recite", + "electrodes", + "conformity", + "response", + "olate", + "357", + "Snap", + "Crime", + "pointer", + "TIT", + "distinctions", + "427", + "ÙĪ", + "abases", + "Mars", + "Spiritual", + "impuls", + "Philadelphia", + "1994", + "cunning", + "fram", + "inco", + "omnip", + "imize", + "ervative", + "Gy", + "Drug", + "carniv", + "Sailor", + "download", + "Beetle", + "Earthqu", + "izontal", + "Alan", + "Nice", + "Prior", + "MAG", + "autobi", + "Brill", + "predominant", + "Messiah", + "REM", + "Slip", + "Webs", + "ademic", + "<", + "Vessel", + "vari", + "Code", + "beetle", + "projects", + "BAT", + "psychotic", + "underside", + "refute", + "Considering", + "kees", + "wd", + "priority", + "twentieth", + "atheist", + "amina", + "euphem", + "tripod", + "Trayvon", + "NON", + "2200", + "NPC", + "ependence", + "MHz", + "Bung", + "pane", + "aboriginal", + "PLUS", + "igers", + "Sexy", + "MF", + "Chall", + "Ay", + "ilingual", + "adj", + "frown", + "successful", + "stack", + "ic", + "Seah", + "consequ", + "bugs", + "Scand", + "Curve", + "Nob", + "Hoo", + "Kissinger", + "Timeline", + "mt", + "Description", + "YP", + "Installation", + "levision", + "anthropology", + "itzerland", + "iaries", + "kward", + "robat", + "carbohydrate", + "Phot", + "оÐ", + "SQL", + "Disc", + "dataset", + "ynski", + "fiat", + "Dres", + "Favor", + "Halls", + "Alt", + "PART", + "Spider", + "disabling", + "RG", + "Ward", + "aturation", + "willfully", + "lockout", + "Shutdown", + "956", + "communists", + "Against", + "Ore", + "Rik", + "ASD", + "Onion", + "particulars", + "Analy", + "checked", + "selected", + "romy", + "Akira", + "congr", + "Choice", + "bos", + "organisms", + "frowned", + "Tok", + "Bir", + "Scrib", + "realms", + "coercive", + "1993", + "021", + "âĢĵâĢĵ", + "athetic", + "rior", + "folly", + "AMERICA", + "cassette", + "953", + "absorbs", + "043", + "quad", + "''.", + "Extract", + "424", + "Whit", + "Dun", + "exerted", + "brethren", + "Chronicles", + "eric", + "Mot", + "endings", + "piration", + "predetermined", + "Airl", + "gasp", + "367", + "exclaim", + "cation", + "sort", + "idden", + "missive", + "ع", + "oice", + "same", + "Ott", + "scatter", + "Flight", + "TOD", + "Stra", + "amia", + "IZE", + "compressor", + "ixels", + "lethal", + "Experimental", + "Ing", + "knife", + "vanishing", + "Required", + "Stat", + "Plex", + "spection", + "Bakr", + "Amazing", + "breaths", + "rots", + "OSP", + "840", + "Wars", + "OGR", + "372", + "Khe", + "inous", + "lightly", + "Rounds", + "refinement", + "property", + "metaph", + "oultry", + "istor", + "intestine", + "eus", + "Wilhelm", + "Bane", + "emption", + "oubtedly", + "Virtue", + "'),", + "Ħ¢", + "appar", + "Translation", + "Quite", + "physicists", + "priesthood", + "allowable", + "Saint", + "OSED", + "bind", + "torches", + "osexual", + "Cruz", + "ertility", + "AES", + "ascended", + "muzzle", + "electors", + "Krug", + "cc", + "classic", + "Mace", + "Å«", + "âĢ¦\"", + "TEST", + "gomery", + "Person", + "translations", + "Dys", + "Consent", + "361", + "alos", + "allerg", + "Wast", + "Checks", + "cerning", + "lizard", + "revolutions", + "tether", + "minimized", + "Reverse", + "itely", + "iguous", + "athing", + "Flow", + "Moving", + "409", + "047", + "snug", + "Nich", + "cartridge", + "YL", + "forwarding", + "umerous", + "Abedin", + "iolet", + "tick", + "Transform", + "Grant", + "subtitles", + "Emin", + "ghost", + "Kurd", + "fireball", + "compatible", + "projectiles", + "amorph", + "Satisf", + "quirks", + "recept", + "spective", + "graphical", + "Picard", + "Authent", + "Sponge", + "Army", + "Lumin", + "SOME", + "solitude", + "SHOULD", + "Fasc", + "opez", + "types", + "gallery", + "OLOGY", + "shake", + "369", + "reused", + "378", + "exorc", + "docs", + "Yu", + "GOD", + "ocrine", + "location", + "fif", + "Grid", + "powd", + "'[", + "posterior", + "Thompson", + "Table", + "oslov", + "Goddess", + "odon", + "STD", + "responsiveness", + "stab", + "absolute", + "Enough", + "Essence", + "Upgrade", + "hematically", + "Subscribe", + "alsh", + "repl", + "selector", + "Length", + "temporal", + "Tele", + "ocalyptic", + "Deaths", + "rl", + "Target", + "Orn", + "ongh", + "1909", + "Quest", + "Place", + "Disabled", + "ascending", + "giene", + "MSI", + "ivil", + "caval", + "intermitt", + "salts", + "Apr", + "059", + "Keeper", + "emis", + "Eternal", + "SER", + "estones", + "rudimentary", + "pooled", + "Alright", + "diagrams", + "ydia", + "Jacob", + "architectures", + "USPS", + "footnote", + "Brav", + "Leopard", + "virtuous", + "ploma", + "HIP", + "horizontally", + "olith", + "Prop", + "Apocalypse", + "Syria", + "Showdown", + "constitutional", + "Independent", + "Miliband", + "Tracks", + "adle", + "ESL", + "FIGHT", + "john", + "é", + "benef", + "eware", + "TABLE", + "Veg", + "ainers", + "resolves", + "Warren", + "Ranked", + "possibly", + "bian", + "simple", + "uniformly", + "Slash", + "otton", + "Absent", + "agically", + "Pieces", + "Station", + "Beware", + "Discrimination", + "ponies", + "Import", + "utory", + "Paras", + "Phoenix", + "Lat", + "UTC", + "push", + "astically", + "urrent", + "untarily", + "paranormal", + "glanced", + "manifestations", + "Neuroscience", + "irgin", + "ROM", + "($)", + "379", + "missing", + "mercenaries", + "enumer", + "Shant", + "Ws", + "wered", + "buffs", + "ultane", + "Rohing", + "igger", + "Ring", + "manifests", + "Fat", + "Reduced", + "Minerva", + "uart", + "Armory", + "orange", + "igible", + "physiology", + "Ut", + "parchment", + "Fired", + "trap", + "oggle", + "mson", + "Poster", + "bount", + "import", + "maximum", + "422", + "Femin", + "nodding", + "inscription", + "Results", + "GRE", + "icative", + "cognition", + "ions", + "Bite", + "neutron", + "duplication", + "ZIP", + "Quit", + "grasping", + "Daylight", + "layouts", + "CLA", + "reason", + "Huh", + "pige", + "Bomber", + "Produ", + "gland", + "Absolute", + "writ", + "massac", + "fixation", + "device", + "yz", + "GOT", + "Dying", + "adjust", + "grain", + "deform", + "typew", + "dagger", + "Turing", + "Bucc", + "Heavy", + "commod", + "files", + "ogeneous", + "roth", + "Buff", + "bookmark", + "porary", + "Medical", + "Um", + "translucent", + "Anxiety", + "Corinthians", + "optional", + "PUT", + "crucifix", + "alloween", + "VK", + "blu", + "Corinth", + "Mount", + "membranes", + "particip", + "extraord", + "stimulated", + "leneck", + "specifies", + "Sin", + "lash", + "Edited", + "fused", + "Nin", + "Bungie", + "Tooth", + "WATCH", + "Nav", + "Initially", + "+)", + "Ancest", + "transmitter", + "Volks", + "ezvous", + "Nirvana", + "Cald", + "font", + "Und", + "remlin", + "ichever", + "Heal", + "shall", + "attribution", + "authorized", + "INTO", + "acteria", + "Tsu", + "Plane", + "iphate", + "igraph", + "chev", + "inverse", + "ifest", + "Players", + "!!\"", + "Contrast", + "1984", + "sevent", + "colour", + "Rational", + "virtual", + "fec", + "ETH", + "Pru", + "Õ", + "asma", + "Cur", + "assigns", + "ridic", + "Todd", + "ulton", + "Defendant", + "opsis", + "percentile", + "shr", + "wagen", + "368", + "SIGN", + "Screen", + "reprene", + "erection", + "Freak", + "Stard", + "stained", + "cla", + "fet", + "ramids", + "QL", + "avorable", + "TCP", + "nown", + "ulence", + "similar", + "linkage", + "ercise", + "Path", + "LECT", + "Collections", + "Module", + "cs", + "Current", + "mono", + "Alv", + "Dude", + "hypers", + "2600", + "surface", + "predictor", + "Colomb", + "Prof", + "anqu", + "natal", + "adultery", + "Generations", + "clerosis", + "371", + "enlightenment", + "onomic", + "satir", + "Basics", + "Graham", + "Rove", + "adul", + "Shut", + "ocious", + "handc", + "BW", + "Cognitive", + "visible", + "inev", + "978", + "Supported", + "arrays", + "alienation", + "Weight", + "kWh", + "warped", + "386", + "lance", + "herpes", + "PHP", + "claimant", + "uitive", + "pussy", + "corpus", + "Ao", + "Qual", + "XVI", + "requ", + "sympt", + "mination", + "hairy", + "Battles", + "owntown", + "Roberts", + "nec", + "ablo", + "AMD", + "internet", + "Tar", + "direction", + "ouston", + "Glock", + "Yanukovych", + "ogens", + "rogram", + "otype", + "Pt", + "tenance", + "aromatic", + "oxin", + "Vert", + "sociop", + "cible", + "Db", + "________________", + "Third", + "Ships", + "!.", + "expensive", + "WOR", + "primary", + "666", + "decaying", + "clustered", + "beetles", + "Hogwarts", + "headers", + "Judah", + "scen", + "cosmos", + "Genetic", + "blems", + "feeble", + "NOW", + "NSA", + "administ", + "Docker", + "portion", + "gression", + "1904", + "heard", + "inhab", + "Leaves", + "cortisol", + "atinum", + "unknown", + "Observ", + "Philosophy", + "Ide", + "copyrighted", + "surv", + "Locations", + "glands", + "Knife", + "Ember", + "Unicorn", + "haste", + "kinderg", + "Territ", + "Koran", + "aval", + "addon", + "Nero", + "\"]", + "392", + "comfort", + "clothed", + "ashtra", + "mode", + "??", + "!\",", + "knob", + "EMP", + "norm", + "Ago", + "RECT", + "Denver", + "1907", + "Bombs", + "Sche", + "triangular", + "perv", + "rises", + "Jes", + "calibration", + "ts", + "Same", + "Axe", + "Mei", + "multi", + "exerc", + "orney", + "Ware", + "abul", + "Fior", + "Eventually", + "Grizz", + "Past", + "married", + "scram", + "Cache", + "posure", + "heav", + "Shirt", + "powder", + "complex", + "Doc", + "arus", + "Pi", + "curv", + "Topic", + ".)", + "wills", + "philis", + "gui", + "leground", + "Eth", + "Strike", + "Kid", + "delegated", + "Soon", + "wast", + "gage", + "prosecut", + "374", + "opolis", + "chest", + "ensation", + "redes", + "presum", + "Portland", + "annihil", + "yssey", + "forks", + "vitro", + "walker", + "Psal", + "Stealth", + "Quick", + "Baghd", + "Drift", + "//", + "invincible", + "GAM", + "castles", + "bondage", + "Balloon", + "Amid", + "individual", + "tis", + "Guides", + "xe", + "Cong", + "URI", + "HH", + "PHOTOS", + "ASIC", + "burst", + "ahon", + "FIX", + "ilib", + "457", + "Logged", + "à¹", + "Creat", + "inatory", + "column", + "Augustus", + "suggest", + "pret", + "Paran", + "subsistence", + "wx", + "×", + "aleigh", + "dash", + "Mana", + "Ko", + "opausal", + "bene", + "Sabb", + "Ghosts", + "1830", + "Hats", + "Hive", + "Perfect", + "socialists", + "tumult", + "EGA", + "NAME", + "Android", + "assembled", + "phis", + "Stage", + "Char", + "Double", + "insign", + "IED", + "perial", + "EMP", + "mx", + "skept", + "wifi", + "parad", + "Frequency", + "Dist", + "nil", + "iots", + "å", + "Message", + "Furthermore", + "hideous", + "LDL", + "Fault", + "Dimensions", + "Implement", + "fram", + "amaz", + "Indones", + "Tile", + "lar", + "gc", + "correlate", + "ensl", + "mite", + "homosexuals", + "agric", + "8000", + "curing", + "rament", + "recons", + "ocene", + "ENTION", + "communion", + "Function", + "iple", + "redund", + "calibrated", + "contribut", + "Huck", + "limit", + "Fedora", + "Tsuk", + "brates", + "1903", + "ozo", + "visual", + "Discipline", + "chains", + "OCD", + "expended", + "0002", + "sty", + "Nightmare", + "Replace", + "ounty", + "fn", + "1900", + "Epidem", + "FW", + "gul", + "Tomato", + "Perse", + "wl", + "Formation", + "Scan", + "cosystem", + "Brand", + "398", + "captives", + "×", + "ESCO", + "Ender", + "lesh", + "Ascend", + "poly", + "eous", + "hyster", + "Murray", + "phe", + "radiator", + "esthes", + "opin", + "conspic", + "intosh", + "witchcraft", + "CFR", + "ussian", + "escent", + "locking", + "nonsensical", + "uala", + "Serial", + "1991", + "Calm", + "containing", + "stimulates", + "448", + "Pir", + "âĨĴ", + "Diver", + "manuscripts", + "Gaia", + "Ñĥ", + "Learning", + "nipple", + "reads", + "android", + "Meditation", + "incomprehensible", + "edded", + "descendant", + "Morty", + "Luckily", + "ARCH", + "ausible", + "Dig", + "shared", + "Clip", + "trope", + "narcissistic", + "ventures", + "curiously", + "Cosmos", + "Aust", + "Lay", + "Shard", + "Recorded", + "458", + "........", + "perish", + "Example", + "luent", + "apes", + "Hitch", + "holiest", + "amplifier", + "minent", + "xxxxxxxx", + "inite", + "genomes", + "Guilty", + "mult", + "orc", + "nipples", + "Side", + "logically", + "datasets", + "Titanium", + "rotor", + "undle", + "handled", + "nexpected", + "dw", + "diagonal", + "Animated", + "numbering", + "Forest", + "âĨ", + "Prin", + "chemically", + "Github", + "aph", + "Faster", + "Tinker", + "ikini", + "Dest", + "dri", + "Manufact", + "isance", + "Return", + "Alert", + "elcome", + "MMR", + "resid", + "LIC", + "specificity", + "zanne", + "anyways", + "426", + "Scot", + "astery", + "Via", + "Blocks", + "activates", + "abstinence", + "chronological", + "Soul", + "Schne", + "watts", + "AUT", + "calcul", + "Simply", + "Emb", + "ceptive", + "Catholicism", + "obook", + "Bits", + "Mbps", + "indignation", + "shorthand", + "Active", + "Limbaugh", + "Capcom", + "adesh", + "clipping", + "Instructor", + "Secret", + "___", + "Fer", + "rawling", + "Reward", + "weep", + "motherboard", + "Above", + "metry", + "PTS", + "bombard", + "abetes", + ".--", + "Lens", + "Comb", + "basic", + "REALLY", + "Later", + "383", + "positional", + "olesc", + "crotch", + "MDMA", + "requently", + "Pants", + "433", + "uctor", + "illumination", + "Ùħ", + "ocrin", + "pamph", + "atio", + "etc", + "restores", + "Protector", + "Develop", + "Mew", + "trop", + "Slayer", + "Ti", + "Notwithstanding", + "Match", + "LIST", + "IDES", + "Thick", + "disks", + "Kin", + "ghetto", + "Objects", + "prism", + "Nether", + "vul", + "iky", + "]:", + "Detail", + "fucked", + "!?", + "anium", + "lords", + "ilities", + "Ethnic", + "static", + "$$", + "evidence", + "mainline", + "peasant", + "Enhance", + "Forced", + "virt", + "ii", + "symm", + "converter", + "ularity", + "repent", + "num", + "Screw", + "FTA", + "marines", + "hetto", + "blow", + "ado", + "Typical", + "overw", + "Berm", + "keley", + "Song", + "hao", + "valid", + "EXT", + "Provides", + "âĺħâĺħ", + "Odin", + "Shot", + "gamma", + "Princ", + "asonry", + "Accuracy", + "criterion", + "descriptive", + "Gall", + "gray", + "Calcul", + "axes", + "Communists", + "Rebellion", + "Success", + "tg", + "âĺ", + "multiplier", + "ravity", + "Thus", + "URL", + "alternatively", + "duction", + "sarcast", + "Carth", + "USL", + "Invisible", + "larg", + "pleted", + "pathic", + "Additionally", + "Cao", + "latent", + "Surge", + "MEN", + "communications", + "Array", + "Pink", + "commit", + "isodes", + "earcher", + "Ukraine", + "Anthrop", + "incial", + "quotations", + "adena", + "whining", + "retri", + "Assass", + "elligent", + "PERSON", + "Py", + "Send", + "âĪĴ", + "DON", + "watt", + "description", + "POS", + "repro", + "destroy", + "icidal", + "midrange", + "infographic", + "interesting", + "category", + "Flash", + "Invasion", + "Exodus", + "restricted", + "inference", + "dding", + "mingham", + "circumst", + "Wi", + "Hast", + "subjug", + "whispering", + "-.", + "adren", + "Pattern", + "BOX", + "Enhancement", + "Exc", + "Bucket", + "GUN", + "deen", + "Homo", + "1985", + "clo", + "snippet", + "1896", + "TPP", + "Seg", + "success", + ";\"", + "MUCH", + "Author", + "replication", + "hallucinations", + "Inv", + "Aware", + "Viper", + "kai", + "frames", + "THANK", + "SHA", + "wordpress", + "bc", + "CIA", + "arrison", + "alloc", + "Alz", + "letcher", + "Daredevil", + "iversary", + "manuals", + "Catholic", + "feat", + "kinetic", + "JB", + "yeah", + "LDS", + "ppm", + "ADC", + "pring", + "cence", + "clasp", + "setups", + "deity", + "Indra", + "Wander", + "antib", + "Otherwise", + "ombie", + "Bitcoin", + "ipop", + "expression", + "Animal", + "Resurrection", + "Moral", + "SDK", + "wretched", + "ogenous", + "species", + "chuckled", + "Thor", + "428", + "avery", + "Pry", + "asures", + "Ern", + "apor", + "innumerable", + "baptized", + "Explosive", + "elves", + "idges", + "Paradox", + "Close", + "aldehyde", + "construct", + "virginity", + "Poll", + "assin", + "Doctors", + "Pos", + "NECT", + "Moreover", + "Commercial", + "cknowled", + "1988", + "quotation", + "marriage", + "Bapt", + "Sina", + "Gloves", + "gian", + "confounding", + "URRENT", + "Dean", + "Brew", + "thur", + "pty", + "immune", + "SQU", + "counterfe", + "rider", + "inferred", + "Dimension", + "Toad", + "afterlife", + "HERO", + "Indiana", + "seek", + "distinguishes", + "Qur", + "Methods", + "combat", + "categ", + "Struggle", + "teness", + "liquid", + "blinking", + "CONTIN", + "iae", + "aerobic", + "strugg", + "egalitarian", + "hello", + "orrect", + "Abandon", + "ferment", + "Area", + "idem", + "Mania", + "js", + "BALL", + "Running", + "regenerate", + "iquid", + "Uh", + "Crystal", + "Ital", + "Heavenly", + "в", + "CRIPTION", + "Consumer", + "dust", + "amiliar", + "Rhino", + "Rocket", + "reversible", + "kok", + "Sketch", + "shotguns", + "apses", + "detach", + "Cells", + "artist", + "rily", + "Restore", + "Scar", + "evid", + "spaced", + "Contributions", + "418", + "Mystic", + "obfusc", + "Russ", + "wings", + "Pear", + "osite", + "Nusra", + "urations", + "ovie", + "icago", + "Concepts", + "stimuli", + "aroused", + "aughty", + "Talking", + "Prompt", + "Across", + "Plaint", + "branching", + "Thankfully", + "Original", + "Esc", + "Technician", + "fleet", + "usher", + "Mos", + "livion", + "oenix", + "hr", + "ibble", + "indent", + "Finished", + "Department", + "INFO", + "Movie", + "++", + "THING", + "timers", + "rocket", + "Natural", + "lime", + "angular", + "osure", + "dynamically", + "pacif", + "Processor", + "disgu", + "moderators", + "ceases", + "inertia", + "paperback", + "yton", + "Huma", + "prohibitions", + "gestation", + "Bomb", + "termin", + "caric", + "oS", + "tc", + "Cop", + "raved", + "eighty", + "Enable", + "implementations", + "conquering", + "Finder", + "window", + "Gra", + "fonts", + "laughter", + "colonization", + "DOD", + ")!", + ",)", + "Geral", + "Spoiler", + "Component", + "gist", + "hiro", + "licens", + "nesses", + "karma", + "?\".", + "OPA", + "squats", + "RAND", + "orally", + "document", + "olars", + "presumptive", + "Pers", + "OAD", + "ufficient", + "LESS", + "Hidden", + "ORK", + "xs", + "mathematician", + "Gloss", + "annihilation", + "manifold", + "Ry", + "Thunder", + "Yan", + "Activ", + "worldly", + "TED", + "marg", + "Stun", + "ryce", + "VG", + "Isn", + "Cyn", + "Expl", + "IRED", + "compr", + "indisc", + "Boss", + "()", + "berman", + "Begins", + "ujah", + "ornia", + "hetical", + "civilizations", + "fundamentalist", + "strap", + "Forward", + "ettlement", + "prophetic", + "glers", + "bending", + "Terry", + "idi", + "trunc", + "creeps", + "intel", + "switch", + "ailand", + "installer", + "GOP", + "499", + "Parallel", + "Cru", + "\"@", + "396", + "Unlock", + "Raven", + "Corn", + "circadian", + "********************************", + "iliate", + "Functional", + "pronouns", + "Satoshi", + "stim", + "Gay", + "Iss", + "Thief", + "atellite", + "shards", + "phil", + "protein", + "alters", + "Poor", + "Typically", + "KER", + "ociate", + "emits", + "recy", + "mechanically", + "...\"", + "nature", + "sys", + "ysc", + "wavelengths", + "pattern", + "insured", + "parasitic", + "LCS", + "PACs", + "heals", + "CCP", + "Hacker", + "psy", + "Beans", + "demonic", + "JV", + "atmosp", + "equality", + "airst", + "incarn", + "ynthesis", + "equations", + "tch", + "HUGE", + "Changed", + "itatively", + "Job", + "gaming", + "1899", + "Morsi", + "conjecture", + "riad", + "primates", + "Artemis", + "Thro", + "biologically", + "Church", + "topia", + "recomm", + "gradient", + "ful", + "bastard", + "CHO", + "IUM", + "sleep", + "Construction", + "raints", + "vable", + "ionage", + "comrade", + "populate", + "nerds", + "Xie", + "result", + "Imper", + "pamphlet", + "Ku", + "backend", + "ificent", + "etus", + "disson", + "config", + "suc", + "wavelength", + "external", + "owder", + "predis", + "eenth", + "Det", + "andem", + "1865", + "Defeat", + "Individual", + "retrieving", + "stories", + "desolate", + "lett", + "unpublished", + "passively", + "dissertation", + "raits", + "abee", + "Resist", + "Robin", + "benevolent", + "blast", + "Offic", + "snap", + "vernment", + "extermin", + "wt", + "bitious", + "hibited", + "Insp", + "posted", + "Yugoslav", + "rational", + "adapt", + "Atari", + "plugin", + "oglobin", + "efeated", + "HRC", + "cko", + "ilver", + "Destruction", + "gewater", + "Radiation", + "imprison", + "origin", + "antine", + "Publication", + "healer", + "istered", + "THEIR", + "hazard", + "Contract", + "mediated", + "indexed", + "SYSTEM", + "Labor", + "Blade", + "yog", + "Champ", + "Gordon", + "IAS", + "nineteenth", + "animous", + "begin", + "Holo", + "Planet", + "udding", + "default", + "OMG", + "wond", + "wm", + "pend", + "Extreme", + "interstellar", + "ASED", + "Berks", + "primal", + "Foot", + "inadvert", + "amboo", + "Leica", + "Events", + "Pigs", + "RAFT", + "ï", + "Gentleman", + "Multiple", + "Psychiatric", + "despise", + "Zionism", + "SSL", + "shit", + "threaded", + "artifact", + "mitochondrial", + "Layer", + "inus", + "podcast", + "awaken", + "Management", + "delusions", + "grey", + "pseud", + "agonal", + "Hirosh", + "Georg", + "Dragon", + "Stack", + "ohm", + "vener", + "Row", + "sandbox", + "blinding", + "razen", + "389", + "crappy", + "lith", + "antha", + "plurality", + "DAC", + "inently", + "intage", + "1902", + "Depend", + "elapsed", + "==", + "Genie", + "Bush", + "Planetary", + "Bah", + "Kira", + "emn", + "Month", + "allic", + "coded", + "VOL", + "[...]", + "Rampage", + "(*", + "Production", + "licts", + "inoc", + "Cour", + "spurious", + "ultras", + "ggles", + "delusion", + "Racer", + "Prism", + "FH", + "uppet", + "cultured", + "436", + "aneously", + "اÙĦ", + "Missions", + "monton", + "criptions", + "ificate", + "Cause", + "1898", + "ocaust", + "bri", + "Shoals", + "ommod", + "alted", + "ogenesis", + "warn", + "illus", + "vv", + "contam", + "Lesbian", + "cavalry", + "Presence", + "rehens", + "tool", + "accessible", + "(~", + "Licensed", + "prophets", + "boulder", + "mean", + "akura", + "unres", + "Cinnamon", + "Leaks", + "........................", + "Contact", + "assassins", + "Greenwald", + "dk", + "amazon", + "agreeable", + "ernandez", + "Easy", + "PLA", + "Bigfoot", + "convent", + "empires", + "387", + "grasped", + "ruby", + "reconc", + "Warning", + "atem", + "retrieval", + "FDR", + "Reaper", + "orem", + "Luo", + "hig", + "Armor", + "tp", + "Interpret", + "Conservative", + "Sodium", + "bead", + "propagate", + "claw", + "href", + "Paste", + "omit", + "Boost", + "Diamond", + "goo", + "anomal", + "DISTRICT", + "Greek", + "warning", + "despised", + "Karl", + "AGES", + "serotonin", + "ESSION", + "_______", + "Collider", + "auldron", + "squee", + "Control", + "ffield", + "cycles", + "Legal", + "xa", + "minimum", + "Generic", + "Circ", + "·", + "Behind", + "guide", + "Ground", + "roying", + "Grail", + "thee", + "9000", + "Batman", + "Brother", + "nons", + "RW", + "saf", + "Croat", + "tainment", + "sci", + "Ye", + "Range", + "Ey", + "perature", + "Dracula", + "oreal", + "Fighting", + "releg", + "coupling", + "Tracker", + "tyard", + "Mut", + "Military", + "lamm", + "ittens", + "CRC", + "Xiang", + "orthodoxy", + "Goth", + "algorith", + "Athen", + "tyrann", + "Torrent", + "IDs", + "GENERAL", + "ASUS", + "rastructure", + "Faith", + "models", + "rentices", + "Curse", + "calibr", + "attled", + "monary", + "penet", + "aclysm", + "album", + "remnant", + "fung", + "itiveness", + "thodox", + "unlocks", + "probabilities", + "ster", + "scrim", + "analytic", + "Urban", + "âĢĶâĢĶâĢĶâĢĶ", + "Craft", + "brut", + "1986", + "Section", + "raged", + "arij", + "Hero", + "Hebdo", + "Empress", + "vivo", + "Publications", + "cannabinoids", + "arrett", + "bounded", + "quests", + "omin", + "Ruler", + "Yue", + "ridges", + "peasants", + "Alloy", + "Desk", + "ULAR", + "thor", + "Overs", + "Tome", + "mk", + "1050", + "shroud", + "distribut", + "weapons", + "Authorization", + "Poke", + "Alternate", + "scan", + "artisan", + "Gems", + "Forums", + "atonin", + "viron", + "Rog", + "duct", + "tabletop", + "crow", + "/)", + "Stainless", + "ottest", + "reborn", + "anchez", + "cium", + "Nicarag", + "elfare", + "upd", + "ritic", + "bm", + "608", + "Slightly", + "Drops", + "ISO", + "iT", + "xiety", + "Gawker", + "omination", + "Reached", + "Student", + "Drop", + "MET", + "Kubrick", + "1950", + "Tuls", + "computed", + "depending", + "Cosmetic", + "udget", + "Lex", + "icut", + "Depth", + "1893", + "ahah", + "ath", + "fights", + "thia", + "occult", + "Wheel", + "Sega", + "theolog", + "reement", + ")--", + "unus", + "Gamma", + "Looks", + "ellipt", + "airflow", + "Himself", + "pagan", + "Rei", + "pilgr", + "Submission", + "Region", + "insertion", + "sket", + "satisfies", + "Pixie", + "contempl", + "abbit", + "Replay", + "Galile", + "Godzilla", + "arithmetic", + "iasm", + "1987", + "Feminist", + "Liter", + "Disable", + "ouble", + "essors", + "fors", + "ensu", + "Putting", + "MSM", + "Cond", + "emade", + "indistinguishable", + "Magn", + "ms", + "MAL", + "BF", + "dm", + "iltration", + "irection", + "Spir", + "Gb", + "Ibn", + "Abs", + "imens", + "RNA", + "============", + "655", + "Conversion", + "imilation", + "igion", + "Somew", + "mL", + "Border", + "Ë", + "Factor", + "Number", + "ejac", + "Cho", + "righteousness", + "PATH", + "Elys", + "ouched", + "multic", + "faculties", + "Earthquake", + "References", + "ensitive", + "impat", + "................", + "buff", + "1895", + "colo", + "Vi", + "ubiqu", + "Chev", + "Fish", + "Blueprint", + "CHQ", + "linem", + "Flavor", + "crimson", + "Abstract", + "arette", + "plete", + "ranean", + "Dash", + "dimensional", + "Cub", + "ttle", + "DSM", + "instantaneous", + "esy", + "epoch", + "Brit", + "Î", + "ECD", + "warp", + "obyl", + "ubric", + "utilitarian", + "summarizes", + "letal", + "Ord", + "opath", + "tained", + "ghai", + "whis", + "insert", + "phon", + "rils", + "earthly", + "Alic", + "PCIe", + "furthermore", + "ocard", + "uter", + "Admin", + "ographics", + "Constantin", + "gravity", + "iPhone", + "wasteland", + "fps", + "Tip", + "murm", + "paces", + "Samurai", + "FOIA", + "Radiant", + "Unreal", + "microw", + "usterity", + "zyme", + "itbart", + "metadata", + "Dat", + "Moons", + "Protestants", + "ungle", + "videog", + "pid", + "disple", + "aucus", + "coils", + "Dwar", + "fixed", + "Alice", + "garrison", + "Velocity", + "Jehovah", + "fascists", + "CHO", + "jl", + "metaphors", + "Siege", + "scientific", + "Ä«", + "Slow", + "hex", + "Blaz", + "mediated", + "esthesia", + "Avg", + "belie", + "Carter", + "exposition", + "azeera", + "dial", + "bask", + "Scale", + "disob", + "gore", + "hypocr", + "phantom", + "Synd", + "BLIC", + "pter", + "Scorpion", + "eor", + "Recover", + "summoning", + "orb", + "jump", + "768", + "Enix", + "Spons", + ",...", + "Wide", + "parse", + "debtor", + "pathological", + "serpent", + "Franç", + "reetings", + "deletion", + "volunt", + "Notification", + "liga", + "Disk", + "Account", + "1979", + "symmetry", + "Bearing", + "ABV", + "ORDER", + "rpm", + "Fuck", + "?!\"", + "mask", + "Grade", + "neath", + "ocom", + "Detect", + "ryption", + "Aura", + "inert", + "PLAY", + "gres", + "INTON", + "Deal", + "fficient", + "Void", + "gement", + "scorp", + "reincarn", + "Vapor", + "1840", + "Yellow", + "......", + "parameter", + "DISTR", + "Forgotten", + "Eat", + "izational", + "Witness", + "Dupl", + "dogma", + "zipper", + "Zeus", + "mage", + "ormal", + "\".", + "ecc", + "Slot", + "Regist", + "Others", + "VID", + "Windows", + "shitty", + "Lethal", + "Monster", + "Expression", + "tx", + "ythm", + "Were", + "ivalry", + "atcher", + "Format", + "Plasma", + "Phys", + "laugh", + "Fu", + "java", + "roma", + "Increases", + "licensee", + "mystic", + "proto", + "Loki", + "forcing", + "hots", + "->", + "Outside", + "Endless", + "achie", + "Turtles", + "convin", + "JUST", + "immobil", + "Causes", + "clich", + "xes", + "ffiti", + "hypot", + "Bat", + "bigot", + "Personal", + "Pharmac", + "Lot", + "VERT", + "bapt", + "idelines", + "prox", + "MAP", + "Spirit", + "Slug", + "ebook", + "eches", + "Andromeda", + "ceremon", + "1975", + "PRE", + "asshole", + "linear", + "Nevertheless", + "willpower", + "azel", + "Fif", + "andise", + "extravag", + "Buffy", + "correlations", + "ptr", + "Progress", + "shape", + "Symbol", + "arag", + "Context", + "ucer", + "1983", + "Myster", + "Pain", + "Login", + "mbol", + "codes", + "RANT", + "overse", + "opot", + "STEM", + "enser", + "Cosmic", + "Spl", + "ritional", + "Pharaoh", + "Remix", + "xon", + "XII", + "unman", + "immedi", + "monog", + "LX", + "abstraction", + "ocolate", + "Donkey", + "!!", + "LIA", + "shed", + "rules", + "calc", + "Autob", + "anmar", + "eworks", + "notations", + "tenancy", + "Petraeus", + "dp", + "amphetamine", + "Cortex", + "rw", + "projectile", + "intrinsically", + "Route", + "negoti", + "anuts", + "Analysis", + "redits", + "GG", + "thread", + "Chosen", + "Years", + "otyp", + "NCT", + "udic", + "ochemical", + "Neigh", + "fishes", + "Float", + "Print", + "okia", + "barb", + "quote", + "Lew", + "announ", + "istors", + "Reading", + "ACTION", + "intakes", + "Beet", + "matter", + "Swe", + "Ther", + "tyrant", + "Psycho", + "Destroy", + "esoteric", + "biom", + "idious", + "Merc", + "hran", + "Baal", + "seconds", + "superhuman", + "ancel", + "worshipped", + "webs", + "violet", + "Metallic", + "eday", + "ordering", + "Nut", + "constructs", + "olescent", + "Unit", + "otypes", + "embryonic", + "perm", + "Nature", + "Decre", + "levant", + "ss", + "+(", + "Doctrine", + "puters", + "saline", + "orsche", + "1111", + "values", + "utopian", + "Booster", + "Technical", + "ì", + "LIMITED", + "nir", + "clones", + "Performance", + "aple", + "shudder", + "contempor", + "lator", + "Oops", + "ammon", + "david", + "bom", + "bish", + "detectable", + "multiplying", + "reddit", + "Prim", + "medial", + "substrate", + "Sanskrit", + "Spect", + "Magical", + "arcane", + "align", + "1861", + "neocons", + "Ì", + "Bounty", + "Continent", + "hurd", + "alions", + "generalized", + "Insect", + "simul", + "actual", + "advert", + "ukong", + "Resp", + "Warcraft", + "Hunter", + "hyper", + "Breach", + "ught", + "computation", + "react", + "Feel", + "Cheong", + "slut", + "galactic", + "taunt", + "Enjoy", + "reprinted", + "Word", + "Handbook", + "amins", + "exit", + "Wo", + "adherents", + "Counter", + "Node", + "Twisted", + "grinned", + "universal", + "Amon", + "aster", + "Equip", + "!\".", + "analogous", + "rients", + "alky", + "Qian", + "spont", + "docs", + "contemplation", + "revolutionaries", + "preset", + "Amendments", + "executes", + "Duration", + "compulsion", + "stagger", + "ynamic", + "blem", + "];", + "Higher", + "Balt", + "heast", + "corp", + "awei", + "Motion", + "Mis", + "adventurer", + "eger", + "arsen", + "Voltage", + "EVENTS", + "Salt", + "issance", + "DK", + "Ship", + "unwitting", + "Ton", + "PROGRAM", + "tentacles", + "erness", + "thirst", + "Fig", + "fty", + "Tolkien", + "Sleep", + "Explain", + "Pub", + "Bounce", + "Demo", + "1897", + "SPI", + "intern", + "********", + "Kills", + "Zombies", + "Single", + "ratom", + "Claw", + "hid", + "asel", + "Shock", + "erential", + "upgr", + "holy", + "\\", + "aghetti", + "thence", + "genic", + "papers", + "1982", + "ravel", + "UNIVERS", + "Charge", + "Delay", + "ibrary", + "HDD", + "olson", + "enchanted", + "Wr", + "graph", + "corro", + "ept", + "etsu", + "Qin", + "Û", + "antidepressant", + "Cerberus", + "appe", + "DEFENSE", + "dysph", + "split", + "zilla", + "attr", + "Clar", + "Äĵ", + "hov", + "IRC", + "hibition", + "'/", + "URLs", + "Draft", + "Prep", + "Languages", + "Travels", + "ceiver", + "aturally", + "pair", + "ALWAYS", + "aaaa", + "Tenth", + "NAD", + "Serv", + "UID", + "cens", + "Learned", + "traject", + "moaning", + "Nare", + "ingen", + "surn", + "floppy", + "breeding", + "uph", + "rossover", + "Understanding", + "Glass", + "runtime", + "gp", + "âľĵ", + "cyt", + "bley", + "agall", + "unworthy", + "otine", + "chromosome", + "utters", + "µ", + "expans", + "dement", + "insurrection", + "surviv", + "genre", + "ospital", + "Plato", + "Trigger", + "selection", + "ilege", + "segreg", + "itizens", + "RAID", + "Pure", + "hetti", + "Failed", + "Characters", + "Creep", + "akra", + "Ec", + "Aristotle", + "Lim", + "error", + "yrus", + "umably", + ">>", + "tsun", + "knowledge", + "Cert", + "bable", + "hesion", + "Procedures", + "markup", + "ideo", + "rhet", + "Chapters", + "Checking", + "mega", + "photons", + "required", + "Unknown", + "Drawn", + "vari", + "EEK", + "compuls", + "cloning", + "ccoli", + "1070", + "kindred", + "discl", + "Cind", + "Collect", + "chromosomes", + "phant", + "Kafka", + "everlasting", + "mercenary", + "Hmm", + "----", + "riber", + "doubtless", + "susceptibility", + "beta", + "notice", + "crochet", + "respir", + "philosophers", + "Extras", + "separat", + "shown", + "iblings", + "Hispanic", + "copy", + "Tang", + "Knight", + "pursu", + "Anime", + "lipid", + "ggies", + "levels", + "phalt", + "Completed", + "bral", + "cerv", + "Afric", + "Phar", + "Color", + "ogene", + "Compan", + "memory", + "Dust", + "XIV", + "Console", + "').", + "1888", + "byn", + "polygamy", + "Auth", + "BUT", + "istine", + "sacr", + "absor", + "ijah", + "Neural", + "olester", + "ql", + "Already", + "Creating", + "Starg", + "Philos", + "Consider", + "repositories", + "cludes", + "Buffer", + "Perspect", + "comput", + "Stew", + "iamond", + "Judgment", + "OVA", + "angible", + "oxid", + "epigen", + "sidel", + "Eag", + "devices", + "icone", + "1920", + "atism", + "beard", + "Gujar", + "Playstation", + "glances", + "COMPLE", + "VERTIS", + "ukemia", + "Edit", + "Tickets", + "Square", + "Serpent", + "transporter", + "MQ", + "Mongo", + "1967", + "ibaba", + "timet", + "sylvania", + "Latin", + "osaurs", + "humanoid", + "cannabinoid", + "disciple", + "Psych", + "impro", + "mc", + "Raid", + "Letter", + "ificant", + "Portug", + "Freem", + "appell", + "Mushroom", + "clans", + "sinful", + "ingestion", + "Directory", + "abetic", + "antigen", + "imagin", + "mitter", + "!!!!!", + "DPR", + "leness", + "\":\"\",\"", + "AUTHOR", + "grunt", + "flickering", + "Cath", + "asury", + "nozzle", + "Secure", + "Stre", + "BIT", + "deviations", + "Professor", + "bilt", + "Conscious", + "interrupts", + "Mormons", + "Cutter", + "Bed", + "ipient", + "Ghostbusters", + "Cart", + "endas", + "Execution", + "ycle", + "wedd", + "Sold", + "vanquished", + "Regarding", + "Depending", + "']", + "atron", + "oidal", + "Cube", + "Studio", + ":/", + "Explosion", + "activate", + "pport", + "fuck", + "Whe", + "smir", + "widgets", + "urses", + "izard", + ")*", + "icho", + "Versus", + "Introduced", + "osaurus", + "1977", + "forum", + "Gray", + "Program", + "righteous", + "endum", + "Scare", + "resists", + "*)", + "Combo", + "sockets", + "aston", + "LAB", + "mutated", + "eworld", + "DEF", + "Trend", + "âĢĶ-", + "propagation", + "emancipation", + "collection", + "Differences", + "Tweet", + "majesty", + ")...", + "sylv", + "adapters", + "milliseconds", + "Jews", + "Patreon", + "phasis", + "HTTP", + "onnaissance", + "ENDED", + "Intro", + "qs", + "superflu", + "*.", + "minions", + "Stupid", + "specialization", + "Pikachu", + "appellant", + "Training", + "circle", + "Interest", + "fallacy", + "Dinosaur", + "THEM", + "directories", + "masturbation", + "Stain", + "1978", + "odied", + "exqu", + "Rats", + "swick", + "emptiness", + "Xeon", + "thereto", + "Engels", + "Supplement", + "Chan", + "undead", + "Noct", + "erest", + "Query", + "SOLD", + "thritis", + "Encounter", + "vectors", + "Econom", + "Rogue", + "gelatin", + "Rot", + "Flickr", + "caching", + "loader", + "ELE", + "camoufl", + "Commission", + "1886", + "combos", + "Awakening", + "feudal", + "asses", + "ASY", + "atalie", + "panties", + "Mono", + "selves", + "Download", + "vampires", + "------", + "ishop", + "User", + "imperialist", + "GOODMAN", + "1973", + "Vel", + "Struct", + "UFOs", + "drivers", + "Optional", + "uably", + "Principle", + "verett", + "taining", + "1889", + "Communism", + "auder", + "Keys", + "lore", + "Medieval", + "Hyd", + "weapon", + "Register", + "Highlander", + "RFC", + "Demon", + "ardless", + "Orche", + "Kick", + "pixel", + "address", + "OUP", + "Brain", + "Morph", + "bash", + "ANG", + "Idle", + "Lucifer", + "correlates", + "gazed", + "colm", + "Kard", + "Solar", + "Variable", + "PACK", + "fuzz", + "anonym", + "ECO", + "feature", + "Esports", + "Anthropology", + "cise", + "manac", + "Supports", + "rists", + "Quant", + "istical", + "çļĦ", + "dexterity", + "monster", + "ordial", + "Mob", + "DEC", + "Conj", + "entric", + "1981", + "ECTION", + "ietal", + "Uses", + "Armageddon", + "Capitalism", + "Ub", + "iazep", + "helps", + "ouls", + "grim", + "Ethiop", + "tesy", + "clipboard", + "chimpanzees", + "PLIC", + "Sexual", + "wallet", + "Rect", + "ocytes", + "Hels", + "lace", + "Damn", + "blasp", + "ildo", + "Rober", + "APD", + "WCS", + "ippery", + "ellectual", + "$(", + "universes", + "holster", + "shading", + "inflic", + "else", + "Shiny", + "AVG", + "Lower", + "Mayhem", + "Originally", + "Crypt", + "SHARE", + "Beir", + "!:", + "repentance", + "WHAT", + ".......", + "auditory", + "aaa", + "Loot", + "ciples", + "contem", + "photon", + "æľ", + "omach", + "Whedon", + "Valid", + "asonable", + "pha", + "assad", + "Pse", + "Heat", + "plugins", + "clenched", + "Americ", + "transform", + "Enh", + "agnetic", + "usalem", + "sych", + "Wed", + "replace", + "Kinect", + "shield", + "Sax", + "ividually", + "functionally", + ":)", + "typically", + "Opening", + "Fa", + "SELECT", + "samurai", + "horde", + "entle", + "sth", + "Changes", + "Pin", + "ithing", + "illance", + "Emblem", + "Micha", + "crypt", + "Objective", + "ophys", + "avg", + "poon", + "readable", + "Rx", + "allel", + "Sit", + "gom", + "ureau", + "Doodle", + "dungeon", + "($", + "Nintendo", + "\"],\"", + "Notes", + "Grab", + "Prosecutors", + "Advanced", + "1862", + "Veter", + "jurisd", + "Launcher", + "Catal", + "udder", + "residues", + "regress", + "Conquer", + "osal", + "Dice", + "************", + "braska", + "ipolar", + "athe", + "bringing", + "Suddenly", + "IEEE", + "verbs", + "delet", + "ipeg", + "Previous", + "]\"", + "sidebar", + "illac", + "Property", + "α", + "REP", + "authenticated", + "gypt", + "uilding", + "Ging", + "wart", + "Birth", + "obedient", + "Xuan", + "TYPE", + "inhibits", + "1972", + "humans", + "IENT", + "youtube", + "Shortly", + "ophen", + "Winc", + "Writ", + "AUD", + "Hobbit", + "emphasis", + "Wonders", + "twitch", + "Prophe", + "Berry", + "Ginny", + "Burst", + "Generator", + "epile", + "Balanced", + "GPU", + "maps", + "neurotrans", + "IRC", + "\"$", + "Create", + "Particip", + "Marxism", + "thou", + "Mortal", + "�", + "ninja", + "inburgh", + "appro", + "Pistol", + "Jar", + "prophes", + "classes", + "anarchist", + "extant", + "message", + "itaire", + "1863", + "Prol", + "propell", + "impossibility", + "propos", + "itamin", + "Rating", + "olphin", + "mitochond", + "versions", + "Liberal", + "ishy", + "spherical", + "Survive", + "FREE", + "rawler", + "Metal", + "Starship", + "=================================================================", + "Dharma", + "Seller", + "wrapper", + "Experience", + "Integ", + "Customer", + "hammad", + "unanim", + "Jenn", + "schizophren", + "agree", + "EVENT", + "Shell", + "fractions", + "1968", + "extermination", + "Sniper", + "pronoun", + "Hitman", + "xp", + "resource", + "WIND", + "hierarchical", + "ted", + "Changing", + "plaus", + "Transform", + "bicy", + "imentary", + "Fuck", + "Mini", + "overc", + "Optimus", + "outer", + "helial", + "akening", + "fx", + "nig", + "+/-", + "VICE", + "nm", + "1976", + "Ritual", + "Tyrann", + "scriptures", + "inical", + "Null", + "ourgeois", + "dra", + "pious", + "neuron", + "colonists", + "Nebula", + "apply", + "Sah", + "Marx", + "hypotheses", + "notation", + "acists", + "Math", + "Manager", + "Library", + "audi", + "mp", + "ergic", + "wizards", + "fw", + "DVD", + "Scala", + "Different", + "ampoo", + "Dread", + "abbage", + "Rus", + "Dumbledore", + "keleton", + "elsh", + "esian", + "Corsair", + "Tier", + "Celest", + "noun", + "lucid", + "requisites", + "genus", + "Event", + "1974", + "Satanic", + "iox", + "Handle", + "Destroyer", + "invocation", + "XD", + "modified", + "Gam", + "RPC", + "subsystem", + "Compared", + "odan", + "Passive", + "Helmet", + "nutrition", + "riction", + "HOW", + "Jess", + "piston", + "imately", + "hypoc", + "Celestial", + "MRI", + "compiler", + "Badge", + "Revelation", + "intrig", + "Grad", + "SPACE", + "Poly", + "Vul", + "trembling", + "independ", + "doctor", + "Certain", + "emet", + "Password", + "gasped", + "pronunciation", + "Fuel", + "SPEC", + "assets", + "Extra", + "formatting", + "mods", + "\"!", + "akedown", + "circuitry", + "TRUE", + "Veil", + "sighed", + "Charg", + "eals", + "workaround", + "ank", + "Scrolls", + "diffusion", + "amps", + "Tempest", + "adata", + "phenomen", + "???", + "popup", + "inhibition", + "aliases", + "erity", + "agraph", + "Jew", + "bec", + "Classic", + "comment", + "usable", + "rodu", + "Enlightenment", + "invis", + "biochemical", + "latest", + "GMOs", + "Socialism", + "pollut", + "eluc", + "Js", + "orthern", + "PDATED", + "alyses", + "Experts", + "Blog", + "Democr", + "etooth", + "pause", + "âĢ¢âĢ¢", + "Shinji", + "dystop", + "Sources", + "Brach", + "np", + "XY", + "neurot", + "assembly", + "bourgeois", + "Reson", + "IDE", + "recoil", + "raq", + "Avenger", + "Paper", + "UTF", + "Wrest", + "Simulation", + "elaide", + "DMCA", + "utm", + "1963", + "arcs", + "maximal", + "cyl", + "philosoph", + "enium", + "relativity", + "Macintosh", + "pneum", + "LOC", + "goddamn", + "SHA", + "localization", + "PHI", + "hierarch", + "atheists", + "±", + "Luck", + "Jugg", + "options", + "alore", + "Edward", + "Monitor", + "neoc", + "numbered", + "Arc", + "Codes", + "Hallow", + "olitan", + "sections", + "Ezek", + "amy", + "task", + "CLS", + "Valkyrie", + "circumference", + "amac", + "Notting", + "proverb", + "Spec", + "elemental", + "Bitcoins", + "Except", + "Release", + "ADVERTISEMENT", + "Complete", + "phrine", + "spores", + "random", + "neum", + "trigger", + "ocide", + "longitudinal", + "isec", + "peat", + "precept", + "Wing", + "âĹ", + "otropic", + "mouse", + "Witcher", + "Appearance", + "ROR", + "||", + "aird", + "Blu", + "incomp", + "Firefly", + "update", + "Loc", + "nihil", + "hesive", + "Quality", + "youtu", + "Seriously", + "annot", + "Coins", + "Visit", + "lc", + "----------", + "diction", + "afore", + "immortality", + "Forbidden", + "Allah", + "Partial", + "Gears", + "trance", + "Hat", + "irez", + "SATA", + "electrode", + "Linear", + "rikes", + "deriv", + "Xue", + "Fine", + "Ignore", + "desc", + "DOM", + "Simple", + "orescence", + "Previously", + "circumcision", + "Sphere", + "renown", + "SET", + "ilight", + "Byzantine", + "EXP", + "whine", + "Missing", + "Lt", + "Guide", + "hippocampus", + "wip", + "yrights", + "submer", + "Maker", + "Switch", + "spectral", + "nect", + "Ãį", + "reven", + "WER", + "Adding", + "CONTROL", + "asper", + "0000000", + "ynt", + "annabin", + "Aliens", + "PCR", + "asketball", + "ricia", + "Unch", + "Tap", + "practicable", + "Usage", + "soluble", + "Scroll", + "Random", + "moan", + "Puppet", + "Dim", + "Attack", + "spears", + "rectangle", + "amuse", + "Doct", + "reon", + "Reset", + "vag", + "unin", + "Bris", + "Swarm", + "Model", + "Standing", + "denotes", + "{", + "Lizard", + "nesty", + "wor", + "amplification", + "Inferno", + "Cover", + "SAM", + "respective", + "Shift", + "libertarians", + "Runner", + "Revelations", + "Spr", + "Crusader", + "caffe", + "Patch", + "stros", + "Immortal", + "insofar", + "itance", + "Valhalla", + "radial", + "Beast", + "sync", + "--------", + "Pathfinder", + "iless", + "operator", + "Choose", + "decode", + "vou", + "Mutant", + "CVE", + "Female", + "oxidation", + "inational", + "dB", + "Scope", + "Wan", + "Bought", + "Dietary", + "rotein", + "Present", + "aukee", + "totem", + "satur", + "wagon", + "Builder", + "Bulg", + "sects", + "Flo", + "ombat", + "Hermione", + "aughs", + "hydra", + "paren", + "ë", + "Whereas", + "tsky", + "chall", + "WORK", + "opian", + "rican", + "vati", + "HTTPS", + "wrink", + "throb", + "habi", + "iodine", + "omorph", + "Scion", + "Hunt", + "Written", + "iosity", + "Browser", + "sinners", + "culosis", + "unconsciously", + "0100", + "anarchists", + "Pull", + "FFER", + "pandemonium", + "matically", + "Rush", + "purified", + "Cyan", + "Difficulty", + "«", + "Aside", + "oggles", + "untu", + "iege", + "iberal", + "COUR", + "eteenth", + "weeney", + "biased", + "Decay", + "quart", + "alysis", + "stere", + "ellect", + "kernels", + "juven", + "JPEG", + "indal", + "topic", + "identifier", + "åı", + "epid", + "1969", + "poisons", + "sym", + "mop", + "LOCK", + "axe", + "cohol", + "ctory", + "adject", + "Skin", + "Fract", + "SHAR", + "echo", + "thood", + "encoding", + "relational", + "Len", + "Bone", + "agara", + "uggish", + "Tanks", + "Stats", + "lihood", + "Mult", + "Graph", + "Cannot", + "Spac", + "handler", + "Shit", + "morp", + "controller", + "udeau", + "Screenshot", + "Development", + "Gear", + "tong", + "Colossus", + "rylic", + "STRUCT", + "capitalist", + "supplementation", + "Parts", + "pb", + "oppy", + "pite", + "processor", + "explanatory", + "Environmental", + "Compl", + "Gaming", + "arently", + "concess", + "athlet", + "forestation", + "orsi", + "igmat", + "encoded", + "misc", + "proofs", + "Revision", + "mathematic", + "constitu", + "fficiency", + "lightsaber", + "gz", + "erate", + "ournals", + "Comment", + "percept", + ".\"[", + "Techniques", + "coins", + "Shape", + "venant", + "Printed", + "Native", + "Gors", + "pecting", + "Duel", + "admins", + "Flor", + "Deus", + "cham", + "Rails", + "ceptor", + "naire", + "Squid", + "Warranty", + "SPEC", + "ensis", + "FUN", + "stellar", + "Select", + "llular", + "arget", + "Uncharted", + "Details", + "rison", + "syntax", + "chanted", + "-----", + "thats", + "Registration", + "Saber", + "ethical", + "cryptography", + "atown", + "dependencies", + "nw", + "vehement", + "rationality", + "Thou", + "----", + "rador", + "enh", + "Crate", + "STATE", + "/(", + "delim", + "CEPT", + "monkey", + "pai", + "uracy", + "mortals", + "Sanders", + "Seraph", + "-\"", + "1945", + "endix", + ":'", + "Legs", + "Exper", + "Krypt", + "clinton", + "uphe", + "Vers", + "Similarly", + "ressor", + "leans", + "LOG", + "cific", + "].", + "-)", + "resist", + "Pred", + "Latest", + "ilyn", + "blob", + "devils", + "Illusion", + "erella", + "yak", + "method", + "698", + "Shadow", + "velt", + "somet", + "xc", + "triangles", + "netic", + "Calling", + "DRM", + "triglycer", + "inhibited", + "nep", + "algebra", + "ascar", + "laim", + "appl", + "1971", + "Bernie", + "Eh", + "undefined", + "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ", + "Sys", + "ournaments", + "Solid", + "hep", + "Males", + "Agent", + "psychedel", + "Wik", + "doctrines", + "rection", + "Compare", + "âĺ", + "certific", + "substr", + "Citation", + "AFB", + "Became", + "aristocracy", + "aryl", + "anatomical", + "ocumented", + "Assy", + "FORM", + "Traditional", + "azines", + "Content", + "furt", + "scripting", + "cloaked", + "unint", + "Civilization", + "Desktop", + "Ragnar", + "curses", + "observable", + "Spock", + "Pyr", + "electrom", + "Lump", + "oresc", + "Attribution", + "egal", + "achusetts", + "marqu", + "âĻ¦", + "cursor", + "ascist", + "1966", + "edit", + "lisher", + "ocyte", + "Writer", + "BILITIES", + "Upload", + "treacher", + "recomb", + "knights", + "immutable", + "Ply", + "atten", + "Passed", + "Flying", + "icipated", + "querade", + "Zot", + "CRE", + "Cursed", + "ickr", + "Droid", + "thereum", + "adjective", + "DIT", + "tob", + "init", + "Penet", + "ignor", + "exalted", + "Dwell", + "assemb", + "sentient", + "``", + "Goo", + "Professional", + "othing", + "rupted", + "olics", + "Setup", + "Thu", + "Campaign", + "Secondly", + "clipse", + "hibit", + "amate", + "SUP", + "Suppose", + "submit", + "Debian", + "antid", + "entert", + "ysical", + "Gladiator", + "STL", + "Bugs", + "Mech", + "Coffin", + "itored", + "ICLE", + "Mist", + "infall", + "votes", + "actly", + "Occ", + "Conquest", + "alach", + "intertw", + "reverse", + "amiya", + "icularly", + "edom", + "Luxem", + "Fra", + "urrencies", + "nobility", + "Tab", + "Beer", + "10000", + "incor", + "melanch", + "Depth", + "Firstly", + "usr", + "Wiki", + "hhhh", + "Proxy", + "antagonists", + "transistor", + "Relic", + "Prometheus", + "1280", + "Coun", + "Medals", + "stats", + "Assembly", + "inished", + "cemic", + "adventurers", + "cd", + "Supporters", + "Ys", + "])", + "neglig", + "Request", + "whore", + "overcl", + "_-", + "partial", + "amd", + "fructose", + "divid", + "Administ", + "amples", + "Boo", + "akery", + "owered", + "hester", + "Links", + "GROUND", + "ethy", + "incarcer", + "incap", + "Drag", + "Elastic", + "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ", + "Ultra", + "AAAA", + "Order", + "Mysteries", + "canonical", + "Ign", + "animate", + "wegian", + "ggle", + "Hash", + "Arg", + "verty", + "analges", + "ouver", + "ittees", + "Asgard", + "______", + "Mix", + "1964", + "Rate", + "arousal", + "pheus", + "undai", + "hetamine", + "Mysterious", + "Alright", + "Herod", + "riott", + "Anarchy", + "Arche", + "Question", + "Chapter", + "Token", + "Sphere", + "induces", + "Audio", + "Normal", + "prophe", + "Valiant", + "Tag", + "Relations", + "blinked", + "onyms", + "Vortex", + "db", + "emonic", + "Phase", + "kingdoms", + "Twe", + "LORD", + "plementation", + "Constantinople", + "helm", + "Flesh", + "thumbnail", + "ledged", + "PROG", + "disbel", + "Likes", + "Gamer", + "renches", + "hattan", + "Index", + "pecially", + "Jiu", + "whats", + "erion", + "xf", + "Perception", + "Alien", + "Capt", + "ãĢĤ", + "joining", + "nesium", + "Socrates", + "Icon", + "animate", + "ocalypse", + "Tactics", + "assador", + "Veh", + "src", + ",-", + "visc", + "Discord", + "initial", + "atana", + "Size", + "Claim", + "ffect", + "iciary", + "turret", + "reset", + "Ï", + "wrap", + "ulnerability", + "Insert", + "irrad", + "ognitive", + "clips", + "uncle", + "chemy", + "ottesville", + "Write", + "earances", + "1965", + "MIC", + "manag", + "telesc", + "Termin", + "Guest", + "denote", + "Failure", + "ograp", + "âĢķ", + "scrolls", + "Armored", + "recomp", + "placeholder", + "ISBN", + "Belief", + "emporary", + "Asset", + "arcer", + "haar", + "assium", + "%:", + "ernal", + "Lv", + "atible", + "Pand", + "oubted", + "Lie", + "bial", + "STEP", + "presets", + "statist", + "Sund", + "reshold", + "endium", + "\");", + "Software", + "basal", + "Yose", + "mortg", + "ocry", + "subreddit", + "omorphic", + "Loaded", + "berra", + "vg", + "orkshire", + "Chrys", + "Repeat", + "Simulator", + "rx", + "gex", + "Linux", + "Instruct", + "irable", + "mosquit", + "Manga", + "iOS", + "synt", + "clitor", + "lobe", + "Delete", + "CVE", + "fortunately", + "Enc", + "vertising", + "anten", + "fif", + "Study", + "prev", + "ossus", + "Nar", + "Decl", + "erala", + "Prototype", + "UGE", + "1001", + "---------", + "deals", + "odcast", + "TPS", + "codec", + "ittee", + "isexual", + "Breaker", + "menu", + "URI", + "('", + "Fiorina", + "Apostles", + "Witches", + "raint", + "addafi", + "ersive", + "yrim", + "mosa", + "rog", + "Ear", + "âĺħ", + "caloric", + "matical", + "yrics", + "Krugman", + "axter", + "1016", + "sep", + "Extend", + "ropolitan", + "thren", + "ologne", + "atomic", + "Naturally", + "Pros", + "gencies", + "akens", + "Male", + "causation", + "omnia", + "Comments", + "eeee", + "iquette", + "cytok", + "ename", + "details", + "destruct", + "leep", + "Cavern", + "Invention", + "ueless", + "subsection", + "outhern", + "metic", + "blogs", + "Packs", + "Arduino", + "hhh", + "elligence", + "imity", + "Ultron", + "astrous", + "biome", + "Hover", + "privile", + "igham", + "apest", + "Yoshi", + "Artist", + ".\",", + "gamer", + "Virgin", + "Tea", + "Doomsday", + "ðŁĻĤ", + "terday", + "Commando", + "Achieve", + "chrom", + "cryptographic", + "rebell", + "Specifically", + "âĢ¦âĢ¦âĢ¦âĢ¦", + "Eternity", + "emulation", + "SERV", + "Miscellaneous", + "Participant", + "duc", + "vp", + "Sparkle", + "ategories", + "decrypt", + "GNOME", + "activation", + "anarch", + "owler", + "adiator", + "itars", + "THEN", + ")\",", + "åħ", + "embod", + "vae", + "âĺĨ", + "Member", + "rm", + "nyder", + "Leviathan", + "Gaza", + "erenn", + "Chicken", + "Definitive", + "Bolshe", + "Jagu", + "gorith", + "loader", + "exe", + ".........", + "Received", + "Proto", + "Locked", + "Posts", + "ankind", + "Clock", + "CLI", + "Throw", + "dL", + "epad", + "Atmosp", + "mk", + "Steal", + "uple", + "reference", + "GNU", + "adelphia", + "scripts", + "ilaterally", + "Mods", + "odus", + "ignty", + "REF", + "hypothesized", + "issors", + "anus", + "HUD", + "rices", + "Draw", + "Computer", + "Below", + "uthor", + "Tact", + "=$", + "00000000", + "caut", + "Sharp", + "depend", + "tatt", + "Goal", + "Sounds", + "zona", + "anyon", + "ricanes", + "USAF", + "Jump", + "Bottom", + "etermination", + "Ples", + "hypothes", + "Reference", + "swall", + "maneu", + "rifice", + "Veh", + "tex", + "geoning", + "âľĶ", + "Mach", + "eanor", + "%);", + "archives", + "encyclopedia", + "Preferences", + "damage", + "Done", + "coefficient", + "Creatures", + "ital", + "ivari", + "Revolution", + "nob", + "Diff", + "abbre", + "Writ", + "DOS", + "redd", + "splend", + "orest", + "flame", + "devs", + "==", + "Puzzle", + "git", + "MOD", + "Argument", + "Abyss", + "Studies", + "ophob", + "uild", + "scill", + "fp", + "plur", + "Delete", + "FALSE", + "FIL", + "microbiota", + "IPv", + "Stud", + "ortal", + "Divinity", + "ounter", + "ä¸", + "Naz", + "stals", + "ihilation", + "persecut", + "Planes", + "viation", + "Driver", + "EEG", + "Unity", + "Premium", + "Siren", + "Paleo", + "earchers", + "Pract", + "Ö", + "VII", + "mosp", + "identifiers", + "Near", + "achu", + "Apps", + "tackle", + "COLOR", + "perpendicular", + "viks", + "ecided", + "Dota", + "icons", + "psi", + "Brave", + "unimagin", + "ATI", + "OOL", + "Gender", + "Swords", + "oples", + "Rank", + "olphins", + "deities", + "XIII", + "м", + "Kraken", + "LEVEL", + "stasy", + "Babel", + "Hours", + "Avoid", + "Mech", + "Multi", + "ect", + "Occup", + "panic", + "mutants", + "Evidence", + "Tips", + "volts", + "Exit", + "xb", + "planet", + "avez", + "features", + ")]", + "lol", + "Neph", + "Sanct", + "impover", + "................................", + "Sty", + "Email", + "Torrent", + "gluc", + "Sins", + "Incarn", + "WITHOUT", + "Panzer", + "Assignment", + "versible", + "Strange", + "ITNESS", + "incible", + "ZX", + "MySQL", + "conson", + "oxidative", + "Machine", + "Impro", + "Parent", + "Metroid", + "Educ", + "dismant", + "dx", + "Persona", + "HDL", + "Americ", + "Users", + "eighteenth", + "WARNING", + "Lists", + "Canter", + "Trotsky", + "haha", + "]'", + "Encyclopedia", + "admin", + "ACTIONS", + "idav", + "ο", + "FTP", + "quar", + "ongyang", + "âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦", + "synchronization", + "DEM", + "riched", + "negro", + "Bench", + "filament", + "decoding", + "obj", + "joystick", + "Decre", + "Bolshevik", + "Virtual", + "Sacrament", + "xd", + "BILL", + "-+-+", + "¶", + "anchester", + "Pokemon", + "slic", + "iameter", + "errilla", + "Exactly", + "\"'", + "getic", + "3333", + "solete", + "incorpor", + "io", + "------------", + "antiquity", + "ATURES", + "Policy", + "oppable", + "=>", + "ODUCT", + "otide", + "Ú", + "normative", + "Fac", + "shaman", + "element", + "Plex", + "INTER", + "etsk", + "Gauntlet", + "BIOS", + "×ķ", + "riet", + "Rew", + "uristic", + "urches", + "Chomsky", + "ixir", + "package", + "Owner", + "schematic", + "Assistant", + "emanc", + "archetype", + "Initial", + "intent", + "filib", + "ispers", + "Flag", + "Tank", + "insurg", + "approximation", + "semantic", + "subtitle", + "Font", + "intimid", + "hath", + "tools", + "gob", + "Process", + "slave", + "JUSTICE", + "âĻ¥", + "Hardcore", + "Discover", + "exch", + "ptive", + "units", + "Django", + "itudinal", + "pc", + "akespeare", + "ospace", + "horny", + "auth", + "Skyrim", + "ENGTH", + "perors", + "Vulkan", + "chimpan", + "remem", + "opacity", + ":(", + "ushima", + "awoken", + "sacrament", + "Beginning", + "escape", + "Anim", + "advant", + "Requires", + "output", + "droid", + "Yep", + "rieving", + "pt", + "Shotgun", + "Osiris", + "disabled", + "Radius", + "Medium", + "Scient", + "Rept", + "ymm", + "cp", + "Labyrinth", + "poral", + "'(", + "Hack", + "Technique", + "/,", + "ambig", + "Basic", + "retrie", + "VICE", + "BIP", + "ragon", + "phies", + "uminum", + "Fei", + "lesi", + "semantics", + "Hz", + "Underworld", + "endot", + "olesterol", + "ourning", + "caches", + "Yug", + "Legendary", + "Documentation", + "Spiral", + "Clone", + "bnb", + "âĶ", + "ustom", + "Mp", + "gettable", + "agonist", + "neuronal", + "culus", + "enum", + "cules", + "muttered", + "ctica", + "necess", + "Subtle", + "solder", + "Environment", + "oneliness", + "orage", + "âĢ¦.\"", + "nesota", + "agements", + "Ùİ", + "WHERE", + "GDDR", + "Scient", + "Mulcair", + "Rena", + "________________________________________________________________", + "antics", + "torped", + "Brow", + "ossal", + "Category", + "Regular", + "remote", + "ãģ", + "Coil", + "ritch", + "specified", + "Average", + "fingert", + "entity", + "atibility", + "ampunk", + "Scriptures", + "unequ", + "arettes", + "arching", + "astron", + "numeric", + "eBook", + "remove", + "onday", + "metaphysical", + "Goku", + "Element", + "Ruin", + "Norm", + "tox", + "puff", + "harmonic", + "Agility", + "Hearthstone", + "mana", + "Points", + "conduc", + "Persia", + "-----", + "license", + "Application", + "assert", + "Reader", + "Sacrifice", + "float", + "inctions", + "byter", + "fundament", + "\"âĢ¦", + "Fourth", + "Effective", + "Meow", + "Errors", + "Icar", + "MMO", + "apostles", + "faintly", + "component", + "bably", + "uggage", + "MPG", + "krit", + "container", + "ixture", + "POV", + "izabeth", + "onut", + "isdom", + "trace", + "SDL", + "Interestingly", + "Explan", + "lesiastical", + "ternal", + "Bug", + "metabolites", + "geries", + "supra", + "Makoto", + "orget", + "racuse", + "][", + "Prelude", + "peria", + "tube", + "Catalog", + "Goblin", + "QUEST", + "INCLUD", + "VERS", + "erguson", + "commandments", + "UDP", + "itle", + "ι", + "domain", + "roximately", + "TLS", + "ongevity", + "modulation", + "didnt", + "Calories", + "Applications", + "ormon", + "sd", + "dullah", + "cous", + "DARK", + "clip", + "Psychiat", + "Tanz", + "Charisma", + "Merge", + "KDE", + "requires", + "urdue", + "decimal", + "âī¥", + "Auth", + "ebted", + "Templ", + "âĢº", + "Ultimate", + "mammalian", + "advertising", + "dominion", + "acron", + "Wem", + "Heist", + "oiler", + "FLAG", + "ovember", + "Syn", + "godd", + "Pyth", + "glyc", + "Helpful", + "gad", + "chedel", + "Similar", + "¶", + "np", + "REPL", + "Fill", + "Sunder", + "etsy", + "PAX", + "Females", + "Kingdoms", + "whistlebl", + "Hide", + "serial", + "Enemies", + "Peb", + "piety", + "ifact", + "esity", + "bsite", + "esides", + "ported", + "amygdala", + "Gerr", + "afety", + "adip", + "(\"", + "cf", + "url", + "unia", + "icro", + "Austral", + "Config", + "accompanied", + "isite", + "textual", + "\">", + "anecd", + "\",", + "angular", + "Unicode", + "Proof", + "multiplication", + "Address", + "bytes", + "lems", + "uterte", + "Episode", + "oshop", + "ritical", + "Adjust", + "argument", + "\\'", + "Rober", + "pection", + "Agg", + "äº", + "interrupted", + "Debor", + "lair", + "Various", + "isively", + "Static", + "ohyd", + "Echoes", + "UID", + "raught", + "Bott", + "apostle", + "Centauri", + "oxicity", + "ibling", + "paralle", + "inav", + "Crit", + "Typh", + "hig", + "EDITION", + "coord", + "uish", + "sectional", + "inki", + "Title", + "anyahu", + "osterone", + "desper", + "ribly", + "Legend", + "afort", + "Org", + "empir", + "Quake", + "SSL", + "ioxide", + "åľ", + "enz", + "urtle", + "BSD", + "Rust", + "ospels", + "Rare", + "partitions", + "heresy", + "overy", + "monop", + "Pixel", + "odder", + "Option", + "withstanding", + "Transfer", + "arrog", + "skip", + "SSH", + "Sph", + "callback", + "PIN", + "pdf", + "plaint", + "cipled", + "reenshots", + "parsing", + "::::::::", + "ioxid", + "hereafter", + "Functions", + "Bulgar", + "intu", + "DOC", + "Location", + "Hyper", + "ageddon", + "Evil", + "illions", + "Introduction", + "Physical", + "Layout", + "âķ", + "------------------------", + "Rodham", + "Patterns", + "Delivery", + "distur", + "Volunte", + "GUI", + "clen", + "inacc", + "Ballistic", + "Sprite", + "Privacy", + "theme", + "dump", + "Byte", + "Incre", + "apult", + "Wrath", + "ensibly", + "NOTE", + "ounge", + "ustomed", + "ochond", + "Qt", + "Primary", + "sidew", + "Root", + "gregation", + "SQL", + "SOFTWARE", + "Gallery", + "Dungeon", + "Vengeance", + "->", + "steam", + "frivol", + "pid", + "filter", + "facult", + "doms", + "Tool", + "1959", + "prefix", + "comma", + "relative", + "formatted", + "appropriately", + "md", + "xxx", + "Authentication", + "WTC", + "vulner", + "reditary", + "Steam", + "Tx", + "GHC", + "Increased", + "forcement", + "Guant", + "bernatorial", + "Entry", + "Warp", + "Creature", + "Ammunition", + "clust", + "Inher", + "unbel", + "RGB", + "Mankind", + "Plague", + "=================================", + "psc", + "Intern", + "tml", + "Crusade", + "inflamm", + "Storage", + "token", + "inse", + "False", + "Adult", + "Pokémon", + "PLIED", + "glac", + "Dwarf", + "sequence", + "magnification", + "Illuminati", + "hedral", + "param", + "regon", + ".\",\"", + "Eva", + "igree", + "Object", + "optimizations", + "uador", + "mmmm", + "ullivan", + "[\"", + "Dusk", + "trig", + "iss", + "hypert", + "perspect", + "assum", + ":,", + "interpol", + "Asked", + "Boot", + "LIB", + "Loading", + "Ident", + "upuncture", + "ioch", + "prefrontal", + "delay", + "Poké", + "bestos", + "overe", + "Elf", + "eteria", + "Sneak", + "bians", + "ARTICLE", + "Xbox", + "encrypted", + "ync", + "Nietzsche", + "Nonetheless", + "±", + "Primal", + "Flare", + "conflic", + "Rune", + "Tes", + "cellence", + "Mega", + "Entity", + "chrome", + "iatures", + "uninstall", + "Winner", + "aimon", + "homebrew", + "Ruby", + "araoh", + "itime", + "potion", + "Allows", + "ogyn", + "osuke", + "Limited", + "macros", + "ERROR", + "gling", + "todd", + "repre", + "Sakura", + "erker", + "items", + "FIG", + "Unle", + "hardness", + "Split", + "arous", + "ocally", + "ì", + "EVE", + "pleasant", + "ihil", + "Router", + "Lucius", + "readable", + "tremb", + "Dro", + "blaster", + "bourgeoisie", + "NUM", + "Alternative", + "flags", + "GAME", + "ebook", + "IPM", + "correl", + "Setting", + "Frame", + "atheism", + "Interested", + "Liquid", + "stanbul", + "Lv", + "tits", + "dc", + "×Ļ×", + "doctr", + "background", + "tsy", + "Ctrl", + "Compatibility", + "idae", + "example", + "perture", + "guid", + "Winged", + "Command", + "ridor", + "bool", + "comments", + "Immunity", + "Nit", + "Statement", + "manif", + "Intake", + "Bloom", + "txt", + "context", + "input", + "achus", + "proc", + "Ñĭ", + "disemb", + "ospons", + "utical", + "Render", + "Ironically", + "ursday", + "Exile", + "lishes", + "iets", + "orescent", + "cair", + "Subjects", + "Dungeons", + "iii", + "neapolis", + "Blaster", + "php", + "ORED", + "SLI", + "elig", + "Identified", + "Brawl", + "bytes", + "CTR", + "sched", + "Assuming", + "Bound", + "Mathemat", + "razil", + "Astral", + "mble", + "untled", + "mech", + "Dagger", + "Useful", + "nesday", + "tarians", + "AMY", + "Camera", + "node", + "pict", + "ginx", + "yea", + ">>>>>>>>", + "paragraph", + "Supplementary", + "9999", + "Alchemist", + "uzzle", + "igun", + "Calculator", + "Applicant", + "hift", + "GPL", + "encode", + "Crash", + "Nutr", + "kHz", + "TABLE", + "intestinal", + "andom", + "archive", + "Ëľ", + "Registered", + "Questions", + "Remote", + "ethyst", + "gren", + "Texture", + "seiz", + "Anyway", + "Variant", + "ê", + "Adapt", + "ittered", + "meta", + "ambers", + "Ruins", + "Chimera", + "password", + "Reboot", + "caster", + "amplitude", + "Position", + "notation", + "secretion", + "Excellent", + "delete", + "aminer", + "ä»", + "Exec", + "Kenobi", + "Interview", + "ontent", + "ospel", + "tuber", + "CONT", + "roups", + "emulator", + "java", + "0200", + "nested", + "fert", + ")).", + "Dex", + "Sora", + "potions", + "Anon", + "aah", + "dunno", + "μ", + "methodological", + "itles", + "phia", + "Beg", + "Rules", + "XML", + "flask", + "Shogun", + "2048", + "atchewan", + "fuckin", + "Built", + "bour", + "disag", + "yss", + "Ï", + "Spoiler", + "Wiki", + "morphology", + "endors", + "dungeons", + "dragon", + ")),", + "hous", + "overwhel", + "SAY", + "abwe", + "--------------------------------", + "epist", + "palp", + "Extensions", + "Mistress", + "Ukrain", + "================", + "edience", + "abama", + "Lua", + "Offline", + "Konami", + "unicip", + "Machina", + "Specific", + "presupp", + "GEAR", + "rition", + "rences", + "successfully", + "1024", + "Platform", + "}}", + "clude", + "roxy", + "promot", + "Adapter", + "rocal", + "Masquerade", + "Panel", + "Language", + "elsius", + "Push", + "abase", + "dB", + "argon", + "Removed", + "amph", + "Wyr", + "indisp", + "Okin", + "aepernick", + "moil", + "Continue", + "00007", + "Journals", + "TAG", + "Remastered", + "symp", + "methyl", + "Overview", + "umeric", + "Codex", + ".$", + "ranged", + "Sym", + "Verse", + "Enabled", + "FUCK", + "Hearth", + "brill", + "Chaser", + "Beh", + "Alchemy", + "Oracle", + "roleum", + "Voldemort", + "();", + "collaps", + "Visual", + "Angular", + "Osc", + "ichita", + "cig", + "toolbar", + "Enlight", + "ÑĮ", + "ε", + "aliation", + "Lovecraft", + "jri", + "Interstellar", + "debugging", + "parentheses", + "Init", + "Located", + "Weak", + "PvP", + "Cloak", + "uture", + "iths", + "asionally", + "FACE", + "Introdu", + "');", + "slot", + "aturday", + "Niet", + "puzz", + "!!!!!!!!", + "folios", + "Ç", + "verbs", + "Frames", + "Ambro", + "millisec", + "Rebell", + "ylum", + "PASS", + "Configuration", + "μ", + "brids", + "vantage", + "['", + "Scy", + "Benef", + "gradation", + "Orc", + "Resources", + "Awesome", + "Militia", + "POST", + "binaries", + "Mode", + "kb", + "WARRANT", + "hemy", + "Desc", + "alion", + "wiki", + "commer", + "Serial", + "Uncommon", + "ignore", + "constructor", + "ctl", + "):", + "Verify", + "Notice", + "RPGs", + "uckland", + "incre", + "Pinterest", + "Definitions", + "iband", + "td", + "subscrib", + "Shin", + "Gadget", + "Document", + "å®", + "Requ", + "QUIRE", + "Quadro", + "Unix", + "Enlarge", + "thens", + "\"...", + "gebra", + "pload", + "alogue", + "vironments", + "Strength", + "PID", + "Invaders", + "HOME", + "Atl", + "Blizz", + "Width", + "OpenGL", + "zx", + "$,", + "å", + "cig", + "lectic", + "relation", + "feas", + "undown", + "Said", + "ν", + "��", + "english", + "Tokens", + "ALEC", + "OOOO", + "isconsin", + "constants", + "Templar", + "Accept", + "mascul", + "enegger", + "ampires", + "Rated", + "lua", + "ucl", + "Sequence", + "NRS", + "STD", + "Cra", + "autions", + "Kernel", + "oleon", + "htaking", + "ancial", + "Pages", + "orthodox", + "ropy", + "EEE", + "transsexual", + "?????", + "surpr", + "arthy", + "Psychic", + "dorsal", + "cember", + "joice", + "/+", + "verend", + "uint", + "derog", + "Subject", + "hemat", + "!]", + ");", + "meshes", + "reperc", + "Terran", + "åĪ", + "Load", + "å¹", + "ikarp", + "rompt", + "goblins", + "Shattered", + "tests", + "Spread", + "Naruto", + "predic", + "Hyp", + "Arkham", + "NASL", + "Material", + "Rule", + "raviolet", + "Klingon", + "Memory", + "acers", + "Known", + "Important", + "α", + "traged", + "shalt", + "iso", + "JSON", + "Instant", + "pg", + "exponent", + "formance", + "bitcoin", + "DOS", + "cheat", + "rook", + "Biol", + "noticed", + "twent", + "Redux", + "Borderlands", + "Supported", + "TRUMP", + "turrets", + "include", + "Effect", + "disg", + "ophical", + "Faction", + "wiki", + "src", + "Laun", + "TIT", + "orbs", + "incompet", + "descriptor", + "Trog", + "Contribut", + "Godd", + "inances", + "Ult", + "lyak", + "âĢ¢âĢ¢âĢ¢âĢ¢", + "stitial", + "essim", + "Graphics", + "ubis", + "egreg", + "DEV", + "annotations", + "Yang", + "Druid", + "Inquisition", + "ohydrate", + "Critical", + "æĸ", + "Sample", + "Pref", + "Unleashed", + "Accessed", + "conceptions", + "Minor", + "pard", + "prus", + "Factory", + "thinkable", + "executable", + "chapter", + "inyl", + "Display", + "ilater", + "Released", + "DirectX", + "aneers", + "______", + "Hilbert", + "Options", + "sorcery", + "esm", + "ÏĦ", + "descript", + "Tycoon", + "psons", + "cov", + "Launch", + "ogeneity", + "sacrific", + "ADRA", + "netflix", + "flix", + "usage", + "properties", + "attach", + "req", + "Resource", + "requisite", + "1007", + "MIDI", + "Zoro", + "Tue", + "hower", + "dds", + "ynasty", + "headers", + "disproportion", + "omaly", + "vim", + "inces", + "edient", + "Wraith", + "ilibrium", + "Hig", + "Frie", + "Meat", + "ldom", + "KNOWN", + "orgetown", + "Improve", + "10000", + "retarded", + "Disclaimer", + "unfocused", + "Unsure", + "Elixir", + "idth", + "atural", + "Err", + "Critics", + "Bows", + "ifferent", + "proxy", + "Lic", + "aucas", + "rolet", + "CoC", + "doesnt", + "phabet", + "Version", + "hepat", + "gif", + "izophren", + "ãĥ»", + "Gutenberg", + "β", + "phans", + "Scene", + "accomp", + "ilings", + "rypted", + "aceae", + "arantine", + "heses", + "iasco", + "lopp", + "GSL", + "disk", + "ãĢģ", + "0010", + "Outbreak", + "Column", + "odox", + "atform", + "Thrust", + "SVG", + "Enhanced", + "¯", + "Tools", + "rogens", + "xus", + "Available", + "zbollah", + "è¡", + "osate", + "usb", + "ordes", + "Matrix", + "Blazing", + "ascus", + "Sovere", + "hement", + "*:", + "amaru", + "parsed", + "Bonus", + "otrop", + "spell", + "ancock", + "Enchant", + "vP", + "Referred", + "alot", + "Runtime", + "Fn", + "CPU", + "Nicotine", + "External", + "Nightmares", + "entropy", + "kB", + "Realms", + "##", + "submar", + "Slime", + "itual", + "Bastard", + "acknowled", + "Magazine", + "rendered", + "ircraft", + "CSS", + "Numbers", + "Pg", + "utenant", + "Palest", + "Roose", + "udicrous", + "anooga", + "Unt", + "capacitor", + "schema", + "hematic", + "Pinball", + "endars", + "===", + "nsic", + "ipedia", + "chromos", + "mRNA", + "Ct", + "Paladin", + "sonian", + "æ", + "ajor", + "repeat", + "ortex", + "Heroic", + "Hera", + "ociated", + "debug", + "osher", + "upiter", + "_.", + "sys", + "Downloads", + "','", + "Adventure", + "FORE", + "ocument", + "arning", + "miscon", + "vidia", + "Cod", + "ibraries", + "buffer", + "cdn", + "Modes", + "tarian", + "Pyro", + "Fixes", + "âĪ", + "Cf", + "Testing", + "Byte", + "nants", + "oufl", + "Cipher", + "Aim", + "Afgh", + "StarCraft", + "intendent", + "akespe", + "Apply", + ">>>", + "Lenin", + "Shaman", + "%\"", + "Frenzy", + "illusion", + "===", + "Website", + "Allow", + "Binary", + "ensable", + "Empires", + "promul", + "ormonal", + "ileaks", + "Ammo", + "assies", + "atican", + "avior", + "Iter", + "1024", + "uesday", + "Appears", + "achine", + "Problem", + "ousy", + "ramid", + "nox", + "··", + "omething", + "Purg", + "artney", + "0000", + "psey", + "glutamate", + "Activate", + "Repl", + "Priv", + "cyclop", + "Hispan", + "atsuki", + "Likewise", + "JOHN", + "POSE", + "pherd", + "schild", + "suffix", + "åIJ", + "optionally", + "Recomm", + "Spawn", + "ARDIS", + "inconsist", + "english", + "Beta", + "Contains", + "uddenly", + "ls", + "Dynamic", + "åĽ", + "{{", + "dq", + "Hmm", + "oliberal", + "Carnage", + "Rebirth", + "incerity", + "proletariat", + "Crafting", + "Explore", + "eld", + "Anarch", + "(>", + "Clockwork", + "Proced", + "APTER", + "Sorcerer", + "âĶ", + "Snape", + "elist", + "Balance", + "Tube", + "--------------------", + "nostalg", + "ACTED", + "VID", + "soever", + "ignt", + "hypothal", + "Obj", + "igure", + "Elves", + "gorithm", + "Romney", + "idable", + "renheit", + "aptic", + "nonex", + "Profile", + "scient", + "Achievements", + "Reload", + "Products", + "ampire", + "pread", + "Yamato", + "Thread", + "FML", + "Forsaken", + "Statistics", + "([", + "utsu", + "nces", + "...?", + "upload", + "Typ", + "Reflex", + "Dial", + "spawns", + "Server", + "acquaint", + "iterranean", + "='", + "Device", + "ר", + "ocaly", + "Remove", + "=====", + "abdom", + "ideos", + "Dual", + "Fax", + "besie", + "Adin", + "describ", + "iod", + "Limit", + "aunders", + "Assassins", + "xxxx", + "ulner", + "Shipping", + "Item", + "fortune", + "cipher", + "mA", + "acerb", + "ebus", + "modifiers", + "Added", + "prisingly", + "Dir", + "Archangel", + "umbnails", + "Huh", + "WARN", + "Role", + "usional", + "cortical", + "SCP", + "Exception", + "Warhammer", + ")))", + "](", + "synaptic", + "cached", + "archment", + "targ", + "Filter", + "Hades", + "princ", + "halla", + "ptoms", + "Ïģ", + "ructose", + "termination", + "compe", + "define", + "prosec", + "require", + "Corpse", + "Abstract", + "********************************", + "Used", + "Ibid", + "trak", + "ä¸Ń", + "GABA", + "åĬ", + "Hegel", + "Jere", + "odore", + "í", + "namese", + "Origin", + "Mastery", + "gerald", + "Charges", + "--------------------", + "Forge", + "comings", + "åį", + "(&", + "grap", + "Mask", + "Gundam", + "generic", + "Malf", + "raphics", + "Internal", + "ourge", + "irresist", + "sterdam", + "endogenous", + "Export", + "ë", + "poons", + "abund", + "Quantity", + "Issue", + "âĪĴ", + "cknow", + "Anonymous", + "DRAG", + "Wikipedia", + "subdu", + "iverpool", + "apesh", + "Ability", + "CentOS", + "iseum", + "lycer", + "Untitled", + "lineback", + "tomat", + "byte", + "tile", + "linux", + "Palest", + "canon", + "FAULT", + "kHz", + "helic", + "IGF", + "WARE", + "Feature", + "Graveyard", + "Nemesis", + "akuya", + "inement", + "whence", + "ractical", + "Ping", + "tesque", + "scroll", + "espie", + "asynchronous", + "ocre", + "Measure", + "morph", + "std", + "Settings", + "Course", + "],", + "Ïĥ", + "Documents", + "estern", + "tf", + "circumcised", + "geant", + "conject", + "Folder", + "outube", + "Medline", + "Status", + "ctr", + "anoia", + "PowerShell", + "Chel", + "Loop", + "resize", + "aphael", + "workshop", + "velength", + "hover", + "flush", + "β", + "Task", + "pedia", + "ptin", + "bidden", + "windows", + "Caucas", + "aml", + "isoft", + "rs", + "cgi", + "urrection", + "miah", + "ÏĤ", + "playthrough", + "Reddit", + "׾", + "annotation", + "nobles", + "seq", + "mares", + "wik", + "foreseen", + "RPG", + "reper", + "aredevil", + "arcity", + "/\"", + "});", + "discont", + "Binding", + "answered", + "Mesh", + "MPEG", + "perceptual", + "OTAL", + "ursive", + "ãģĦ", + "plun", + "onential", + "ãĤ", + "Reloaded", + "iscopal", + "Despair", + "FIX", + "heterogeneity", + ",[", + "ichick", + "DCS", + "cooldown", + "................", + "somew", + "Battery", + "stract", + "Attempt", + "allery", + "Nept", + "tac", + "Elemental", + "Function", + "bindings", + "versive", + "Warlock", + "Response", + "NPCs", + "ollower", + "Reborn", + "phenotype", + "uscript", + "pecul", + "!/", + "Unique", + "FreeBSD", + "Chero", + "colle", + "gently", + "Empty", + "rss", + "dd", + "forge", + "Traps", + "×Ķ", + "iblical", + "---------", + "uminati", + "login", + "asus", + "xual", + "Miko", + "Drac", + "ssh", + "Submit", + "Multiplayer", + "leanor", + "Orig", + "anism", + "peror", + "ESV", + "encour", + "å°", + "PLoS", + "Crusher", + "ocrates", + "ynchronous", + "§", + "Luffy", + "Lastly", + "differe", + "okane", + "Enh", + "ursor", + "apopt", + "Totem", + "ä½", + "Honest", + "xml", + "Created", + "teleport", + "NRS", + "ccess", + "ilitary", + "ackets", + "enchantment", + "Cunning", + "ortmund", + "Altern", + "Alternatively", + "Luthor", + "Publisher", + "GBT", + "çĶ", + "Activity", + "leptin", + "æĪ", + "Starfleet", + "å¸", + "oooooooo", + "lawy", + "Frag", + "ת", + "yright", + "cookie", + "Finish", + "wikipedia", + "Abilities", + "interface", + "glared", + "Engineers", + "Atk", + "oteric", + "byte", + "ossibility", + "Label", + "CSV", + "è", + "Oblivion", + "android", + "rehensive", + "Commands", + "clud", + "Tutorial", + "retched", + "irlwind", + "conserv", + "ministic", + "void", + "ernels", + "alias", + "Draco", + "desktop", + "Mormonism", + "oÄŁ", + "kef", + "timestamp", + "WAYS", + "ãģĹ", + "\"(", + "eneg", + "CHAT", + "npm", + "Grenade", + "rongh", + "dinand", + "Definition", + "Integer", + "modifier", + "dex", + "Parameters", + "andestine", + "SHALL", + "Purchase", + "enaries", + "starship", + "Armor", + "Skill", + "lookup", + "verages", + "Minimum", + "Bleach", + "df", + "inosaur", + "ixel", + "Zip", + "temp", + "ruby", + "Fram", + "sword", + "Minecraft", + "strous", + "Client", + "Barbarian", + "æĹ", + "USER", + "Mehran", + "axies", + "ermanent", + "Header", + "ablishment", + "hyde", + "Snake", + "Telesc", + "Pocket", + "........", + "Destroy", + "Method", + "Zup", + "olulu", + "unemploy", + "Temp", + "Explicit", + "人", + "cache", + "innamon", + "unavoid", + "Summary", + "appre", + "taxp", + "XXX", + "ieval", + "Summon", + "å¤", + "Lear", + "ibliography", + "CLASS", + "dimension", + "Horde", + "filesystem", + "Qiao", + "obbies", + "DIR", + "impedance", + "éĩ", + "Names", + "Drupal", + "Applic", + "imei", + "ynchron", + "Ire", + "Minion", + "Haste", + "ä¿", + "(=", + "LinkedIn", + "Maps", + "ifacts", + "Damage", + "odynam", + "Shroud", + "Ancient", + "enhagen", + "Tact", + "anship", + "aturdays", + "ãģ«", + "ikhail", + "ãģ®", + "framework", + "lication", + "âĢ¦]", + "Plug", + "Lilith", + "browser", + "offset", + "Juda", + "ciating", + "console", + "=================", + "._", + "Puzz", + "OPLE", + "erial", + "OHN", + "Golem", + "ierrez", + "},", + "inition", + "insula", + "Entered", + "greSQL", + "Flask", + "XCOM", + "fixes", + "Weasley", + "arser", + "rc", + "microsoft", + "HHHH", + "INFO", + "rehend", + "polymorph", + "Button", + "âī", + "QUI", + "twitch", + "jriwal", + "Saiyan", + "adherent", + "acters", + "arthed", + "âĢł", + "foss", + "ã", + "Quote", + "ependent", + "horr", + "UGC", + "Weiss", + "styles", + "advertisement", + "Credits", + "Lua", + "UCH", + "horrend", + "minion", + ">,", + "ãĥ³", + "includ", + "Compar", + "[]", + "(<", + "Phones", + "paralleled", + "HTML", + "(%", + "raltar", + "amd", + "Maximum", + "Solitaire", + "SCP", + "Vaugh", + "CLR", + "database", + "module", + "̶", + "Capture", + "Window", + "ubuntu", + "Includes", + "Uriel", + "ORPG", + "κ", + "âĪ", + "ä¸Ģ", + "dexter", + "Glac", + "slice", + "HAHAHAHA", + "\\\"", + "lations", + "ÙIJ", + "AUTH", + "earch", + "Socket", + "Character", + "Sort", + "indist", + "/_", + "Antar", + "ifix", + "lich", + "variable", + "_(", + "gui", + "Herm", + "elvet", + "è¯", + "Developer", + "kcal", + "ciation", + "Transaction", + "docker", + "###", + "Vegeta", + "Result", + "ocamp", + "aughtered", + "Increase", + "aples", + "iannopoulos", + "zbek", + "estyles", + "emonium", + "è¿", + "FANT", + "Reason", + "Elsewhere", + "\"\"", + "Artifact", + "Authent", + "herical", + "membr", + "socket", + "Elsa", + "Condition", + "lapt", + "sorcerer", + "Layer", + "apters", + "veter", + "Myth", + "ensical", + "ÏĢ", + "noxious", + "unpre", + "Flags", + "OOOOOOOO", + "incent", + "Combat", + "Session", + "teleportation", + "éĢ", + "ortment", + "Admin", + "Fixed", + "×Ļ", + "confir", + "ãģŁ", + "morrow", + "osponsors", + "\\/", + "ictionary", + "Num", + "quir", + "åº", + "à¨", + "<<", + "Attempts", + "ãģ§", + "λ", + "Features", + "XXXX", + "inflamm", + "VERSION", + "ortality", + "spawn", + "ratulations", + "charism", + "&&", + "Dialogue", + "luster", + "<<", + "args", + "redients", + "predicate", + "qqa", + "etheus", + "(!", + "showc", + "cmd", + "bringer", + "coh", + "Input", + "FANTASY", + "fict", + "Blocks", + "Install", + "vector", + "umblr", + "agnar", + "Array", + "embry", + "theoret", + "href", + "irrel", + "irements", + "iations", + "(/", + "Thumbnail", + "hashes", + "^^", + "Copy", + "eq", + "translation", + "Favorite", + "Fail", + "ogre", + "isites", + "Merit", + "ãģ¦", + "DATA", + "rarily", + "igmatic", + "Sequ", + "Els", + "ãģª", + "lehem", + "requency", + "aughed", + "distingu", + "artific", + "dwarves", + "Í", + "resy", + "~~", + "sofar", + "ideon", + "ozyg", + "EEEE", + "Melee", + "大", + "tumblr", + "ssl", + "Wra", + "ONSORED", + "vowel", + "},", + "Vari", + "cientious", + "Node", + "sorce", + "========", + "perse", + "Detailed", + "isphere", + "Background", + "ĺħ", + "Redd", + "ìĿ", + "ãģ¨", + "CTRL", + "ç", + "iculty", + "ername", + "ns", + "Deploy", + "happ", + "///", + "Begin", + "gp", + "$.", + "Output", + "Suggest", + "×IJ", + "Toggle", + "nutrit", + "\\\"", + "preval", + "subreddits", + "Menu", + "Amount", + "Wasteland", + "sprites", + "shader", + ";)", + "NAME", + "CLUD", + "goblin", + "Refer", + "ÙĴ", + "á¹", + "Improved", + "endiary", + "assail", + "chieve", + "reply", + "contrad", + "cients", + "GROUP", + "Controller", + "omsky", + "chemist", + "packages", + "ombies", + "scl", + "ibn", + "çĽ", + ":(", + "Minotaur", + "niper", + "====", + "subsc", + "è¦", + "integer", + "\"-", + "theorem", + "utenberg", + "Trigger", + "github", + "ä¼", + "##", + "xtap", + "oké", + "ilial", + "idepress", + ":\\", + "Param", + "Correction", + "ïve", + "Chest", + "ש", + "ÏĦ", + "respawn", + "rall", + "creatine", + "umsy", + "Template", + "foo", + "query", + "manufact", + "Hardware", + "iframe", + "-------", + "recip", + "Attributes", + "foreskin", + "ãĤĭ", + "ãĥĦ", + "uania", + "................................................................", + "phylogen", + "eaturing", + "sprite", + "invari", + "DonaldTrump", + "({", + "Malfoy", + "Gamer", + "Plugin", + "γ", + "Query", + "Puzzles", + "inventory", + "trl", + "Insert", + "awa", + "Werewolf", + "horizont", + "×ŀ", + "cunt", + "]]", + "Byz", + "Mouse", + "[[", + "Cthulhu", + "DRAGON", + "Default", + "Presbyter", + "ff", + "orcs", + "Construct", + "Debug", + "*/", + "×ij", + "embr", + "License", + "css", + "incinn", + "Prosecut", + "sugg", + "å¾", + "Undead", + "æĿ", + "fs", + "thw", + "Vector", + "åĮ", + "settings", + "å¯", + "ssh", + "Converted", + "ãĤĴ", + "risome", + "agre", + "Collection", + "cmp", + "puter", + "alloc", + "é", + "ascade", + "Spells", + ":-)", + "Haunted", + "adolesc", + "FORMATION", + "Imperium", + "ãĥ¼", + "Supplement", + "Render", + "Theme", + "Torment", + "([", + "ëĭ", + "html", + "juven", + "Siber", + "daemon", + "ivariate", + "objects", + "negie", + "indu", + "landish", + "Meta", + "Impl", + "glyph", + "-->", + "streng", + "agascar", + "guyen", + "((", + ")[", + "Norn", + "hippocamp", + "¯", + "îĢ", + "Connection", + "PATH", + "mbuds", + "Shards", + "advoc", + "simulac", + "âĸij", + "!?\"", + "Potion", + "amulet", + "Fnatic", + "cryptoc", + "wav", + "radius", + "pkg", + "MFT", + "æĢ", + "toile", + "Items", + "ifference", + "errors", + "Celt", + "unpop", + "ilogy", + "6666", + "hesda", + "Instruct", + "å·", + "Materials", + "ettings", + "Percent", + "resistor", + "tymology", + "deprecated", + "grep", + "WRITE", + "triv", + "scrut", + "[/", + "anyl", + "skirts", + "MSN", + "Codec", + "ecd", + "Anth", + "){", + "%]", + "veyard", + "aspberry", + "ãĢ", + "Reward", + "rha", + "Stretch", + "]-", + "Prev", + "Context", + "linux", + "HAHA", + "perties", + "VIDE", + "Domain", + "murd", + "Legions", + "apache", + "æŃ", + "Pause", + "Temperature", + "ufact", + "igslist", + "Retrieved", + "èª", + "ãģĮ", + "Ingredients", + "ruary", + "dyl", + "Alias", + "ÎĶ", + "inval", + "amsung", + "!--", + "olean", + "æī", + "ãģ¯", + "coefficients", + "DHCP", + "âĨĴ", + "utonium", + ":[", + "âĹ", + "cli", + "Container", + "å¼", + "nexus", + "SOURCE", + "Ò", + "=/", + "mysql", + "Gained", + "/*", + "uncture", + "statically", + "âĸł", + "æĺ¯", + "æ°", + "estamp", + "Cache", + "ulkan", + "staking", + "apter", + "ãģ¾", + "μg", + "tremend", + "Piercing", + "naissance", + "Healer", + "Enabled", + "éģ", + "âĸ", + "Thumbnails", + "hither", + "Format", + "utherland", + "íķ", + "destro", + "fff", + "execute", + "msg", + "romancer", + "Canaver", + "Vaults", + "oided", + "iage", + "img", + "summary", + "]);", + "ABE", + "Gamergate", + "utherford", + "overwrite", + "enment", + "æķ", + "systemd", + "tif", + "]).", + "ãĤ¤", + "Widget", + "======", + "(-", + "\"+", + "Incarnation", + "æĥ", + "���", + "GUI", + "èĥ", + "forums", + "runes", + "âī¤", + "defic", + "Distance", + "directory", + "Horus", + "iltr", + "ortium", + "./", + "bda", + "owship", + "âĨij", + "}.", + "åĩ", + "1027", + "Weapons", + "lucent", + "auth", + ";;", + "Recommended", + "surv", + "vm", + "Stronghold", + "paran", + "Trance", + "æĺ", + "sovere", + "corrid", + "Pwr", + "[/", + "seq", + "Population", + "[];", + "referen", + "Instr", + "Stamina", + "kernel", + "Python", + "-+", + "allele", + "éĽ", + "isode", + "ä¸į", + "otonin", + "modules", + "Notable", + "Spell", + "\\\\", + "Pref", + "datas", + "setup", + "hapl", + "Height", + "åĭ", + "ãģ£", + "]),", + "Handle", + "umenthal", + "Package", + "enthus", + "unsus", + "Narr", + "Examples", + "FAQ", + "REDACTED", + "notor", + "Enable", + "Pattern", + "aeda", + ">.", + "CHECK", + "����", + "'.", + "ãĥ", + "append", + "����", + "gemony", + "terness", + "Haku", + "NVIDIA", + "queue", + "Bind", + "neigh", + "armor", + "retty", + "LOD", + "plugins", + "/>", + "TYPE", + "4096", + "-------", + "Preview", + "FML", + "proletarian", + "zees", + "enfranch", + "ãģĨ", + "Ctrl", + "Module", + "Surviv", + "Starcraft", + "rored", + "reddit", + "rul", + "tx", + "mage", + "Sword", + "~/", + "Effects", + "éļ", + "ä¹", + "Sensor", + "Solution", + "ãģĻ", + "Arcade", + "predec", + "Values", + "Length", + "fortun", + "ttp", + "\"[", + "tmp", + "Berserker", + "åĨ", + "ositories", + "councill", + "ffff", + "));", + "Recipe", + "ASCII", + "âĦ¢:", + "ä", + "horm", + "=>", + "sers", + "ãģĭ", + "Recommend", + "['", + "agame", + "Animation", + "aucuses", + "Discussion", + "helicop", + "å¿", + "Float", + "Component", + "instance", + "foo", + "localhost", + "=-", + "Offset", + "Psy", + "Gohan", + "buquerque", + "defe", + "chwitz", + "parse", + "dors", + "spons", + "async", + "agonists", + "indo", + ".>>", + "Disciple", + "filename", + "rency", + "Dise", + "\"/", + "template", + "ãĤ¹", + "swers", + "++", + "[(", + "thora", + "Depths", + "livious", + "disadvant", + "foundland", + "Upload", + "§§", + "sophistic", + ";}", + "izont", + "\"}", + "estial", + "Ranked", + "Occupations", + "LEASE", + "Ogre", + "folder", + "Plot", + "farious", + "suscept", + "Types", + "Discuss", + "'/", + "æµ", + "earable", + "æ³", + "Tile", + "iatus", + "åŃ", + "reperto", + "Helper", + "Returns", + "ä¸Ĭ", + "imaru", + "req", + "dissatisf", + "multipl", + "}{", + "-[", + "itial", + "*/", + "Config", + "Example", + "jQuery", + "Mods", + "GPIO", + "laun", + "layout", + "cised", + "......", + "+++", + "prototype", + "Exception", + "subsections", + "resemb", + "âĩ", + "PubMed", + "username", + "aggro", + "éĥ", + "};", + "Mages", + "ryu", + "apons", + "Optional", + "Ancients", + "ãĤĬ", + "Quotes", + "oaded", + "suspic", + "inline", + "omial", + "Mahjong", + "auntlets", + "anarchism", + "subclass", + "MLG", + "...]", + "Dialog", + "uphem", + "recursive", + "7601", + "frac", + "Else", + "Severus", + "},{\"", + "CLIENT", + "javascript", + "sama", + "Learns", + "ãĤĤ", + "Upgrade", + "Listener", + "snipp", + "rune", + "TTL", + "ertation", + "olicy", + "=\"\"", + "«ĺ", + "expr", + "ovych", + "ãģ", + "_-_", + "munition", + "////", + "func", + ">>>>", + "Provider", + "Ïī", + "BUG", + "[-", + "arrang", + "merce", + "ãĥ", + "incarn", + "Valid", + "Aether", + "ãĤĵ", + "UTF", + "Monstrous", + "ãĤĮ", + "hedon", + "áµ", + ":#", + "Frieza", + "padding", + "Reviewer", + "psychiat", + "yrinth", + "âĶĤ", + "hillary", + "Static", + "Newsletter", + "Avg", + "fn", + "Topic", + "choes", + "newsp", + "á¸", + "[+", + "~~~~~~~~~~~~~~~~", + ":]", + "apego", + "buf", + "Translation", + "ById", + "mmol", + "ãĥ¼ãĥ", + "å½", + "ãĤī", + "parser", + "ãĥª", + "`,", + "Lair", + ")}", + "ypes", + "adobe", + "ancest", + "ernel", + "NULL", + "ç«", + "anguages", + "Increases", + "æĦ", + "utorial", + "ithmetic", + "dll", + "Arcane", + "çī", + "tc", + "urtles", + "èĪ", + "Bytes", + "Slot", + "Bahá", + "Weapon", + "widget", + "querque", + "embodiments", + "å¥", + "WARN", + "swer", + "thumbnails", + "FFFF", + "inguishable", + "âī", + "${", + "AAAAAAAA", + "Conclusion", + "ĻĤ", + "disable", + "Rect", + "subp", + "().", + "Detected", + "èĢ", + "[]", + "coerc", + "mM", + "recated", + "fusc", + "Sorce", + "çĶŁ", + ").[", + "})", + "mobi", + "yip", + "Acknowled", + "ternity", + "iqueness", + "ython", + "><", + "std", + "Url", + "namespace", + "tion", + "oother", + "Ó", + "hemor", + "rg", + "ventory", + "ãĤ¢", + "anamo", + "Socket", + "Topics", + "apeshifter", + "gnu", + "detrim", + "`.", + "romeda", + "çIJ", + "lambda", + "Compan", + "Variable", + "usb", + "Adamant", + "ournal", + "covari", + "ãĥ©", + "éĸ", + "åİ", + "otaur", + "(),", + "Marginal", + "ãģı", + "physic", + "adeon", + "RESULTS", + "200000", + "ãģį", + "udeb", + "ãģĵ", + "COMPLE", + "msg", + "ghazi", + "/*", + "Deity", + "disapp", + "Availability", + "illum", + "à©", + "ptives", + ",âĢĶ", + "chnology", + "accur", + "api", + "Obj", + "ãĤ«", + "ãĤ¸", + "ä¹ĭ", + "ËĪ", + "tcp", + "Required", + ".<", + "\".[", + "~/.", + "obser", + "RFC", + "integers", + "åī", + "Installation", + "Ô", + "ó", + "csv", + "ãĥ«", + "Noticed", + "âĸĵ", + "Tumblr", + "Reply", + "||", + "conclud", + "))", + "ebin", + "sql", + "Closure", + "++++", + "],[", + "âĹı", + "prolet", + ">=", + "estinal", + "[*", + "Inquisitor", + "cmd", + "FINE", + "CRIP", + "vertex", + "TeX", + "///", + "Ö¼", + "iscons", + "myster", + "Changed", + "timeout", + "irtual", + "Methods", + "certs", + "texture", + "Roaming", + "Proxy", + "Override", + "éĹ", + "utf", + "python", + "Rarity", + "ilitarian", + "çľ", + "().", + "æł", + "buf", + "åij", + "çķ", + "*.", + "umerable", + "~~~~", + "å¦", + "simultane", + "json", + "Requires", + "perl", + "Interface", + "rupal", + ":", + "itialized", + "HTTP", + "Trivia", + "Sov", + "wrapper", + "={", + "Azerb", + "aeper", + "neighb", + "initions", + "sts", + "Sasuke", + "#$", + "uliffe", + "æĸ¹", + "++++++++++++++++", + "Elven", + "ãģĤ", + "artif", + "Folder", + "à¨", + "åĤ", + "phyl", + "uggest", + "blance", + "ãģł", + "Requirements", + "Usage", + "initialized", + "ãģ®æ", + "conservancy", + "Reincarn", + ")|", + "antioxid", + "Clicker", + "unlaw", + "\\(", + "ãĥĪ", + "[*]", + "Characters", + "////////", + "ãĢIJ", + "ãĤ·", + "webkit", + "ãĢij", + "xp", + "alkyrie", + "Console", + "());", + "Korra", + "\"))", + "oooooooooooooooo", + "Timer", + "////////////////", + "yout", + "engeance", + "emetery", + "mages", + "mods", + "Null", + "philos", + "ascript", + "addon", + "âĸĪ", + "emale", + "----------------------------------------------------------------", + "\\\\", + "=[", + "Parables", + "ãĥĨ", + "VALUE", + "@@", + "uint", + "${", + "cpp", + "%%", + "(âĪĴ", + "utils", + "prefix", + "å°Ĩ", + "ãĥŃ", + "Completed", + "goto", + "ãĤ¯", + "Winged", + "perty", + "[\"", + "ãĥİ", + "Scythe", + "æľ", + "!=", + "Buffer", + "docker", + "WATCHED", + "èĢħ", + "())", + "dst", + "SIZE", + "Demonic", + "resil", + "ãĤ¿", + "pione", + "cpu", + "++)", + "TEXT", + "discrep", + "debian", + "quished", + "acknow", + "trave", + "gcc", + "Catalog", + "ctrl", + "Moroc", + "cpu", + "];", + "Sorceress", + "Introduced", + "Frames", + "condem", + "¶æ", + "~~~~~~~~", + "Emacs", + "][/", + "glim", + "Init", + "Primordial", + "ãĥĥ", + "+=", + "blat", + "à¼", + "------------------------------------------------", + "gpu", + "ãĥĥãĥĪ", + "xml", + "boolean", + "References", + "?)", + "satell", + "Queue", + "pestic", + "}}", + "Attribute", + "dx", + "Defin", + "Synopsis", + "..................", + "ãĥ¬", + "plugin", + "Disable", + "0000000000000000", + ")\\", + "Ichigo", + "println", + "rontal", + "Setup", + "��������", + "å§", + "âĸº", + "Pengu", + "ailability", + "Duration", + "Timeout", + "ãĢĮ", + "behav", + "Reviewed", + "toget", + "\\.", + "lished", + "thous", + "perpend", + "ecause", + "Layout", + "è»", + "Dexterity", + "unsigned", + "+=", + "[[", + "Runes", + "ãĤ¦", + "};", + "})", + "FTWARE", + "ength", + "milo", + "duino", + "天", + "Clojure", + "ļé", + "ãĥ¥", + "gradient", + "\"\"\"", + "âĨij", + "@#", + "JSON", + "proport", + "addr", + "});", + "ãĥIJ", + "ä¸ī", + "tmp", + "å£", + "../", + "zsche", + "âĪ¼", + "Entity", + "æ©Ł", + "âĶľâĶĢâĶĢ", + "filename", + "{{", + "@@", + "Seym", + "/**", + "Summoner", + "Quantity", + "ç·", + "Attach", + "bool", + "Texture", + "opio", + ".}", + "ãĥĭ", + "integer", + "regex", + "nomine", + "ription", + "ãģ®ç", + "ãĥķ", + "subparagraph", + "GGGG", + "explan", + "Header", + "Spawn", + "toggle", + "²¾", + "Abyss", + "expr", + "Zerg", + "Grimoire", + "Contents", + "Instance", + "cyclopedia", + "ãĥĹ", + "Takeru", + "=(", + "代", + "\\)", + "rgb", + "htt", + "bryce", + "livest", + "Annotations", + "âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ", + "berus", + "ntil", + "skelet", + "callback", + "åħī", + "Joined", + "ãĤª", + "args", + "artifacts", + "å¤", + "ÃĽ", + "ãĥŀ", + "Streamer", + "}\"", + "unden", + "ãĥģ", + "Īè", + "ãĥ£", + "0004", + "\\'", + "ãĤ°", + "CONFIG", + "#####", + "``", + "anguage", + "*)", + "Template", + "MODE", + "00000000", + "'';", + ">", + "lvl", + "Footnote", + "Iter", + "####", + "ãĥij", + "Carbuncle", + "[+]", + "mathemat", + "Allows", + "4090", + "Async", + "ģ«", + "Ļ½", + "))))", + "á½", + "cx", + "answ", + "{\"", + "ãĥŁ", + "addons", + "Filename", + "Appearances", + "ãĢĮ", + "addr", + "charact", + "glomer", + "Advertisements", + "dracon", + "Fenrir", + "();", + "Citiz", + "acebook", + "params", + "]=", + "subscript", + "entreprene", + "tnc", + "iversal", + "millenn", + "ithub", + "/>", + "\"{", + "Frameworks", + "avorite", + "])", + "Constructed", + "fml", + "ãĥį", + "################################", + "-|", + "¥ŀ", + "withd", + "Cth", + "AppData", + "Msg", + ":{", + "ãĤ¨", + "tuple", + "ç¥ŀ", + "intrins", + "Cooldown", + "ategory", + "^{", + "ãĥĬ", + "''''", + "çĶ°", + "DEBUG", + "cannabin", + "ocobo", + "Invalid", + "ãĥĢ", + "Compat", + "({", + "Removed", + "convol", + "}:", + "interstitial", + "\"", + "initialized", + "exting", + "Poké", + "Parameters", + "¶ħ", + "########", + "NULL", + "ãĥĩ", + "groupon", + "\\-", + "ãĥı", + "ãĤ±", + "subsequ", + "ccording", + "MODULE", + "Protoss", + "\"},{\"", + "..............", + "Integer", + "endif", + "ãĥĻ", + "parser", + "lambda", + "carbohyd", + "Unloaded", + "_{", + "âĸ¬âĸ¬", + "debian", + "]}", + "ãĤ¶", + "Parameter", + "ãĤ£", + "ãĤ»", + "$_", + "İĭ", + "iterator", + "ãĤ¬", + "WINDOWS", + "CONCLUS", + "\"\\", + "umbn", + "(&", + "ãĥ©ãĥ³", + "usercontent", + "ometimes", + "METHOD", + "ãĥ¢", + "potion", + "ãĥ¯", + "everal", + "weap", + "minecraft", + "================================", + "printf", + "Shinra", + "reluct", + "\\\",", + "Runtime", + "xff", + "Abyssal", + "akeru", + "\\(\\", + "\"/>", + "efficients", + "Ü", + "avascript", + "behavi", + "++;", + "=#", + "Attributes", + "âĵĺ", + "lvl", + "¬¼", + "/**", + "Gameplay", + "Leilan", + ">)", + "=\"/", + "));", + "ãĥĨãĤ£", + "ġ", + ".", + "DEBUG", + "âĶģ", + "ãĢı", + "WithNo", + "Redditor", + "âĶľ", + "fmt", + "ãĢİ", + "msec", + "ĪĴ", + "eatures", + "itially", + "\"\"\"", + "ãĥ¼ãĤ¯", + "Textures", + "\"},", + "\"><", + "||||", + "ß", + "iterator", + "è£ħ", + "Ĥª", + "ojure", + "ãħĭãħĭ", + "ãĥ¼ãĥ³", + "println", + "][", + "âĸĪâĸĪ", + "âķIJ", + "\\\":", + "senal", + "é¾į", + "é¾", + "cryst", + "ãĥķãĤ¡", + "Cosponsors", + "ãĤ·ãĥ£", + "Magikarp", + "Magicka", + "âĸĪâĸĪâĸĪâĸĪ", + ",,,,,,,,", + "vertisement", + "âĶĢâĶĢâĶĢâĶĢ", + "ãĥķãĤ©", + "luaj", + "CLASSIFIED", + ".''.", + "byss", + "{:", + "Nanto", + "ptr", + "%%", + "teasp", + "[_", + "ãĥ¤", + "ħĭ", + "ŃĶ", + "pci", + "\"<", + "GGGGGGGG", + "æĪ¦", + "--+", + "ãĤ®", + "())", + "âĸ¬", + "sizeof", + "}}}", + ";;;;;;;;", + ">]", + "âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ", + "Vaults", + "istg", + "newcom", + "=]", + "¿½", + "ĵĺ", + "{\\", + "Args", + "exha", + "(\\", + "unnecess", + "\"}],\"", + "UNCLASSIFIED", + ">(", + "ãĤ¢ãĥ«", + "æ©", + "70710", + "Ń·", + "ãĥ¼ãĥĨãĤ£", + "Sakuya", + "ãĥĥãĥī", + "Pyrrha", + "escription", + "VIDIA", + "================================================================", + "looph", + "=~", + "cumbers", + ")]", + "govtrack", + "ãĤµ", + "subur", + "Þ", + "âī¡", + "Interstitial", + "ãĥ¼ãĥĨ", + "gobl", + "ãĥīãĥ©", + "oldown", + "ģĸ", + "Depths", + "());", + "._", + "20439", + "ç¥ŀ", + "ãģ®å®", + "ãĤ¼", + "$\\", + "âĹ¼", + "encount", + "| + virtual std::tuple forward(int *ids, int64_t *dims, int step, bool logits_all = false) = 0; + + // Reorder cached keys and values, size=batchSize*beamSize + virtual void reorderCache(int *idx, int size) = 0; + + virtual DecoderContext *getContext() = 0; + + virtual Messenger &getMessenger() = 0; + + virtual int getRank() = 0; + + virtual int getEndId() = 0; +}; diff --git a/include/abstract_searcher.h b/include/abstract_searcher.h new file mode 100644 index 0000000..8f36549 --- /dev/null +++ b/include/abstract_searcher.h @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include + +class AbstractSearcher { +public: + // First call to get NextToken, return {batchSize, numBeams}. For + // greadySearch, numBeams = 1. + virtual std::vector getNextToken(int *ids, int batchSize, int seqLen) = 0; + + // Subsequent calls to get next Token + virtual std::vector getNextToken() = 0; + + virtual bool isDone() = 0; + + virtual std::vector finalize() = 0; +}; + +struct SearcherConfig { + bool doEarlyStopping = false; + int maxLen = -1; + int numBeams = 1; + int numBeamHypsToKeep = 1; + int eosTokenId = -1; + int padTokenId = -1; + float lenPenalty = 1.0; + + SearcherConfig(int maxLen_ = -1, int numBeams_ = 1, int numBeamHypsToKeep_ = 1, float lenPenalty_ = 1.0, + bool doEarlyStopping_ = false, int eosTokenId_ = -1, int padTokenId_ = -1) + : maxLen(maxLen_) + , numBeams(numBeams_) + , numBeamHypsToKeep(numBeamHypsToKeep_) + , lenPenalty(lenPenalty_) + , doEarlyStopping(doEarlyStopping_) + , eosTokenId(eosTokenId_) + , padTokenId(padTokenId_) {} +}; diff --git a/include/dtype.h b/include/dtype.h new file mode 100644 index 0000000..69d93de --- /dev/null +++ b/include/dtype.h @@ -0,0 +1,11 @@ +#pragma once + +namespace xft { +enum class DataType { + bf16, + fp16, + int8, + bf16_fp16, + bf16_int8, +}; +} // namespace xft \ No newline at end of file diff --git a/include/layers_norm.h b/include/layers_norm.h new file mode 100644 index 0000000..75d1778 --- /dev/null +++ b/include/layers_norm.h @@ -0,0 +1,64 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once + +#include "dtype.h" + +namespace xft { + +void invokeLayerNorm(DataType dt, void *output, const void *input, const void *gamma, const void *beta, const int rows, + const int size, int iStride = -1, int oStride = -1, const float epsilon = 1e-5); + +void invokeRmsNorm(DataType dt, void *output, const void *input, const void *weight, int rows, int cols, + int iStride = -1, int oStride = -1, float epsilon = 1e-6); + +// Layer normalization: only support the norm along last dimension +class LayerNorm { +public: + LayerNorm(); + ~LayerNorm(); + + void setWeight(const float *gamma, const float *beta, int size); + + // input and output are in shape of (rows, normSize) + // TODO: column-wise parallel + void forward(const float *input, float *output, int rows, int iStride = -1, int oStride = -1); + +private: + int normSize; + + // the weights contains gamma and beta concated together + float *weights; +}; + +// Layer normalization: only support the norm along last dimension +class RmsNorm { +public: + RmsNorm(); + ~RmsNorm(); + + void setWeight(const float *w, const float *, int size); + + // input and output are in shape of (rows, normSize) + void forward(const float *input, float *output, int rows, int iStride = -1, int oStride = -1, float epsilon = 1e-6); + +private: + int normSize; + + // the scale weight + float *weight; +}; + +} // namespace xft \ No newline at end of file diff --git a/include/models.h b/include/models.h new file mode 100644 index 0000000..f1ab865 --- /dev/null +++ b/include/models.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +#include "abstract_decoder.h" +#include "abstract_searcher.h" +#include "dtype.h" + +namespace xft { +class Model { +public: + Model() : decoder(nullptr), searcher(nullptr), isNewInput(true) {} + ~Model(); + + void input(std::vector &inputIds_, int batchSize_); + + void config(int maxLen_ = -1, int numBeams_ = 1, int numBeamHypsToKeep_ = 1, float lenPenalty_ = 1.0, + bool doEarlyStopping_ = false, int eosTokenId_ = -1, int padTokenId_ = -1); + + bool isDone(); + + std::vector generate(); + + void createSearcher(SearcherConfig &config_); + + int getRank(); + + int getBatchSize() { return batchSize; } + + int getSeqLen() { return seqLen; } + + SearcherConfig getConfig() { return configuration; } + + void setDecoder(AbstractDecoder *dec); + + std::vector finalize() { return searcher->finalize(); } + +private: + AbstractDecoder *decoder; + AbstractSearcher *searcher; + std::vector inputIds; + int batchSize; + int seqLen; + SearcherConfig configuration; + bool isNewInput; +}; + +class AutoModel : public Model { +public: + AutoModel(std::string modelPath, xft::DataType datatype); +}; +} // namespace xft \ No newline at end of file diff --git a/include/xfastertransformer.h b/include/xfastertransformer.h new file mode 100644 index 0000000..6134881 --- /dev/null +++ b/include/xfastertransformer.h @@ -0,0 +1,4 @@ +#pragma once + +#include "dtype.h" +#include "models.h" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ccf31b7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 120 +target-version = ["py38", "py39", "py310", "py311"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6295974 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +cmake==3.26.1 +sentencepiece==0.1.99 +tokenizers==0.13.3 +torch==2.0.1 +transformers==4.30.0 diff --git a/run_dev_docker.sh b/run_dev_docker.sh new file mode 100644 index 0000000..d2b55de --- /dev/null +++ b/run_dev_docker.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +cd "$( dirname "${BASH_SOURCE[0]}" )" || exit + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE="${SCRIPT_DIR}" + +USER_NAME=$(id -u -n) +DOCKER_IMG_NAME="intel/xfastertransformer:dev-ubuntu22.04" + +TAG="xfastertransformer" +CONTAINER_NAME=${TAG}_$((`date '+%s'`*1000+`date '+%N'`/1000000)) + +# Check if the current user is root +if [ "$(id -u)" -ne 0 ]; then + # Docker image name + DOCKER_IMG_NAME="${TAG}:${USER_NAME}" + # Handling illegal characters in the docker image names. + DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g') + # Convert to all lower-case, as per requirement of Docker image names + DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]') + + # Check if the Docker image exists + if [ -n "$(docker images -q "${DOCKER_IMG_NAME}")" ]; then + echo "Docker image ${DOCKER_IMG_NAME} exists." + else + echo "Docker image ${DOCKER_IMG_NAME} does not exist." + echo "Build dev image with current USER ID..." + + # Dockerfile to be used in docker build + DOCKERFILE_PATH="${SCRIPT_DIR}/dockerfiles/Dockerfile.dev" + DOCKER_CONTEXT_PATH="${SCRIPT_DIR}" + + DOCKER_USER="--build-arg USER_NAME=$(id -u -n) --build-arg USER_UID=$(id -u) \ + --build-arg USER_GROUP=$(id -g -n) --build-arg USER_GID=$(id -g)" + + if [[ ! -f "${DOCKERFILE_PATH}" ]]; then + die "Invalid Dockerfile path: \"${DOCKERFILE_PATH}\"" + fi + + # Build the docker container. + echo "Building container (${DOCKER_IMG_NAME})..." + docker build ${DOCKER_USER} \ + -t "${DOCKER_IMG_NAME}" \ + -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}" + + # Check docker build status + if [[ $? != "0" ]]; then + echo "ERROR: docker build failed. Dockerfile is at ${DOCKERFILE_PATH}" + fi + fi +fi + +# Print arguments. +echo "WORKSPACE: ${WORKSPACE}" +echo " (docker container name will be ${CONTAINER_NAME})" +echo "" + +# By default the container will be removed once it finish running (--rm) +# and share the PID namespace (--pid=host) so the process inside does not have +# pid 1 and SIGKILL is propagated to the process inside. +docker run -it \ + --rm \ + --privileged=true \ + --pid=host \ + -P \ + --shm-size=16g \ + --name "${CONTAINER_NAME}" \ + -v /data/:/data/ \ + -v "${WORKSPACE}":"${WORKSPACE}" \ + -w "${WORKSPACE}" \ + -e "http_proxy=$http_proxy" \ + -e "https_proxy=$https_proxy" \ + "${DOCKER_IMG_NAME}" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9f2bfc6 --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +import os +from setuptools import setup, Extension, find_packages +from setuptools.command.build_ext import build_ext + + +class CMakeExtension(Extension): + """ + Overrides the base extension class so that setuptools + doesn't try to build your sources for you + """ + + def __init__(self, name, sources=[]): + super().__init__(name=name, sources=sources) + + +class BuildCMakeExt(build_ext): + def run(self): + for extension in self.extensions: + if extension.name == "xft": + self.build_cmake() + else: + super().run() + + def build_cmake(self): + self.announce("Preparing the build environment", level=3) + build_dir = os.path.abspath(os.path.dirname(self.build_temp)) + os.makedirs(build_dir, exist_ok=True) + + self.announce("Building xft binaries", level=3) + self.spawn(["cmake", f"-B {build_dir}", "."]) + self.spawn(["make", "-C", build_dir, "-j"]) + + +setup( + name="xfastertransformer", + version="1.0.0", + keywords="LLM", + description="Boost large language model inference performance on CPU platform.", + long_description="Boost large language model inference performance on CPU platform.", + license="Apache 2.0", + url="https://github.com/intel/xFasterTransformer", + author="xFasterTransformer", + python_requires=">=3.8", + package_dir={"": "src"}, + packages=find_packages(where="src", include=["xfastertransformer"]), + package_data={"xfastertransformer": ["*.so"]}, + platforms="x86_64", + ext_modules=[CMakeExtension(name="xft")], + cmdclass={"build_ext": BuildCMakeExt}, + install_requires=[ + 'torch>=2.0.0' + ] +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..bb63ad0 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (C) 2023-2024 Intel Corporation +cmake_minimum_required(VERSION 3.15.1) + +add_subdirectory(utils) +add_subdirectory(kernels) +add_subdirectory(layers) +add_subdirectory(models) +add_subdirectory(searchers) + +add_library(xfastertransformer_static STATIC) + +set(SRC_LIB_LIST "utils" "layers" "kernels" "models" "searchers") + +target_link_libraries(xfastertransformer_static + ${SRC_LIB_LIST} + ${3RDPART_LIB_LIST} + libiomp5.so + libmklml_intel.so) + +add_library(xfastertransformer SHARED) + +target_link_libraries(xfastertransformer + ${SRC_LIB_LIST} + ${3RDPART_LIB_LIST} + libiomp5.so + libmklml_intel.so) + +add_dependencies(xfastertransformer_static ${DEPEND_LIST}) +add_dependencies(xfastertransformer ${DEPEND_LIST}) + +execute_process(COMMAND python -m pip show torch + RESULT_VARIABLE EXIT_CODE + OUTPUT_QUIET) +if(${EXIT_CODE} EQUAL 0) + message(STATUS "PyTorch found. Compiling torch extension...") + add_subdirectory(pytorch) +else() + message(STATUS "PyTorch not found. Skipping torch extension compliling...") +endif() diff --git a/src/common/bfloat16.h b/src/common/bfloat16.h new file mode 100644 index 0000000..ac1191d --- /dev/null +++ b/src/common/bfloat16.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "bit_cast.h" + +class bfloat16_t { +public: + bfloat16_t() = default; + bfloat16_t(float f) { (*this) = f; } + constexpr bfloat16_t(uint16_t r, bool) : raw_bits_(r) {} + + template ::value>::type> + bfloat16_t(const IntegerType i) + : raw_bits_ {convert_bits_of_normal_or_zero(bit_cast(static_cast(i)))} {} + + bfloat16_t &operator=(float f) { + auto iraw = bit_cast>(f); + switch (std::fpclassify(f)) { + case FP_SUBNORMAL: + case FP_ZERO: + raw_bits_ = iraw[1]; + raw_bits_ &= 0x8000; + break; + case FP_INFINITE: raw_bits_ = iraw[1]; break; + case FP_NAN: + raw_bits_ = iraw[1]; + raw_bits_ |= 1 << 6; + break; + case FP_NORMAL: + const uint32_t rounding_bias = 0x00007FFF + (iraw[1] & 0x1); + const uint32_t int_raw = bit_cast(f) + rounding_bias; + iraw = bit_cast>(int_raw); + raw_bits_ = iraw[1]; + break; + } + + return *this; + } + + template ::value>::type> + bfloat16_t &operator=(const IntegerType i) { + return (*this) = bfloat16_t {i}; + } + + operator float() const { + std::array iraw = {{0, raw_bits_}}; + return bit_cast(iraw); + } + + bfloat16_t &operator+=(const float a) { + (*this) = float {*this} + a; + return *this; + } + + static void cvt_float_to_bfloat16(const float *src, bfloat16_t *dst, int size); + static void cvt_bfloat16_to_float(const bfloat16_t *src, float *dst, int size); + static void float_add_bfloat16(const float *src1, const bfloat16_t *src2, float *dst, int size); + +private: + constexpr uint16_t convert_bits_of_normal_or_zero(const uint32_t bits) { + return uint32_t {bits + uint32_t {0x7FFFU + (uint32_t {bits >> 16} & 1U)}} >> 16; + } + + uint16_t raw_bits_; +}; + +static_assert(sizeof(bfloat16_t) == 2, "bfloat16_t must be 2 bytes"); + +inline void bfloat16_t::cvt_float_to_bfloat16(const float *src, bfloat16_t *dst, int size) { + constexpr int kStep = 16; + + const __m512i nan = _mm512_set1_epi32(0xffff); + const __m512i ones = _mm512_set1_epi32(0x1); + const __m512i vec_bias = _mm512_set1_epi32(0x7fff); + +#if (__GNUC__ > 12) || ((__GNUC__ == 12) && (__GNUC_MINOR__ >= 3)) + auto cvt_fp32_to_bf16 = [&](const __m512 input_vector) { return (__m256i)_mm512_cvtneps_pbh(input_vector); }; +#else + auto cvt_fp32_to_bf16 = [&](const __m512 input_vector) { + __m512i value = _mm512_castps_si512(input_vector); + auto mask = _mm512_cmp_ps_mask(input_vector, input_vector, _CMP_ORD_Q); + auto result = _mm512_and_si512(_mm512_srli_epi32(value, 16), ones); + result = _mm512_add_epi32(result, vec_bias); + result = _mm512_add_epi32(result, value); + result = _mm512_srli_epi32(result, 16); + result = _mm512_mask_blend_epi32(mask, nan, result); + return _mm512_cvtusepi32_epi16(result); + }; +#endif + + for (int i = 0; i < size; i += kStep) { + __m512 input_vector = _mm512_loadu_ps(src + i); + __m256i output_vector = cvt_fp32_to_bf16(input_vector); + _mm256_mask_storeu_epi16(dst + i, 0xffff, output_vector); + } + + int remainder = size % kStep; + if (remainder != 0) { + __mmask16 mask = 0xFFFF >> (16 - remainder); + __m512 input_vector = _mm512_maskz_loadu_ps(mask, src + size - remainder); + __m256i output_vector = cvt_fp32_to_bf16(input_vector); + _mm256_mask_storeu_epi16(dst + size - remainder, mask, output_vector); + } +} + +inline void bfloat16_t::cvt_bfloat16_to_float(const bfloat16_t *src, float *dst, int size) { + constexpr int kStep = 16; + + auto cvt_bf16_to_fp32 = [](const __m256i src) { + auto y = _mm512_cvtepu16_epi32(src); + return _mm512_castsi512_ps(_mm512_bslli_epi128(y, 2)); + }; + + for (int i = 0; i < size; i += kStep) { + __m256i input_vector = _mm256_maskz_loadu_epi16(0xFFFF, src + i); + __m512 output_vector = cvt_bf16_to_fp32(input_vector); + _mm512_storeu_ps(dst + i, output_vector); + } + + int remainder = size % kStep; + if (remainder != 0) { + __mmask16 mask = 0xFFFF >> (16 - remainder); + __m256i input_vector = _mm256_maskz_loadu_epi16(mask, src + size - remainder); + __m512 output_vector = cvt_bf16_to_fp32(input_vector); + _mm512_mask_storeu_ps(dst + size - remainder, mask, output_vector); + } +} + +inline void bfloat16_t::float_add_bfloat16(const float *src1, const bfloat16_t *src2, float *dst, int size) { + constexpr int kStep = 16; + + auto cvt_bf16_to_fp32 = [](const __m256i src) { + auto y = _mm512_cvtepu16_epi32(src); + return _mm512_castsi512_ps(_mm512_bslli_epi128(y, 2)); + }; + + for (int i = 0; i < size; i += kStep) { + __m512 vec1 = _mm512_loadu_ps(src1 + i); + __m256i _t = _mm256_maskz_loadu_epi16(0xFFFF, src2 + i); + __m512 vec2 = cvt_bf16_to_fp32(_t); + _mm512_storeu_ps(dst + i, vec1 + vec2); + } + + int remainder = size % kStep; + if (remainder != 0) { + __mmask16 mask = 0xFFFF >> (16 - remainder); + __m512 vec1 = _mm512_maskz_loadu_ps(mask, src1 + size - remainder); + __m256i _t = _mm256_maskz_loadu_epi16(mask, src2 + size - remainder); + __m512 vec2 = cvt_bf16_to_fp32(_t); + _mm512_mask_storeu_ps(dst + size - remainder, mask, vec1 + vec2); + } +} diff --git a/src/common/bit_cast.h b/src/common/bit_cast.h new file mode 100644 index 0000000..ba8e7ab --- /dev/null +++ b/src/common/bit_cast.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +template +inline T1 bit_cast(const T2 &u) { + static_assert(sizeof(T1) == sizeof(T2), "Bit-casting must preserve size."); + static_assert(std::is_trivial::value, "T1 must be trivially copyable."); + static_assert(std::is_trivial::value, "T2 must be trivially copyable."); + + T1 t; + uint8_t *t_ptr = reinterpret_cast(&t); + const uint8_t *u_ptr = reinterpret_cast(&u); + for (size_t i = 0; i < sizeof(T2); i++) + t_ptr[i] = u_ptr[i]; + return t; +} diff --git a/src/common/float16.h b/src/common/float16.h new file mode 100644 index 0000000..b93867c --- /dev/null +++ b/src/common/float16.h @@ -0,0 +1,175 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "bit_cast.h" + +class float16_t { +public: + float16_t() = default; + float16_t(float val) { (*this) = val; } + constexpr float16_t(uint16_t bits, bool) : raw_(bits) {} + + float16_t &operator=(float val); + + float16_t &operator+=(float16_t a) { + (*this) = float(f() + a.f()); + return *this; + } + + operator float() const; + + static void cvt_float_to_float16(const float *src, float16_t *dst, int size); + static void cvt_float16_to_float(const float16_t *src, float *dst, int size); + static void float_add_float16(const float *src1, const float16_t *src2, float *dst, int size); + +private: + float f() { return (float)(*this); } + + uint16_t raw_; +}; + +static_assert(sizeof(float16_t) == 2, "float16_t must be 2 bytes"); + +inline float16_t &float16_t::operator=(float f) { + uint32_t i = bit_cast(f); + uint32_t s = i >> 31; + uint32_t e = (i >> 23) & 0xFF; + uint32_t m = i & 0x7FFFFF; + + uint32_t ss = s; + uint32_t mm = m >> 13; + uint32_t r = m & 0x1FFF; + uint32_t ee = 0; + int32_t eee = (e - 127) + 15; + + if (0 == e) { + ee = 0; + mm = 0; + } else if (0xFF == e) { + ee = 0x1F; + if (0 != m && 0 == mm) mm = 1; + } else if (0 < eee && eee < 0x1F) { + ee = eee; + if (r > (0x1000 - (mm & 1))) { + mm++; + if (mm == 0x400) { + mm = 0; + ee++; + } + } + } else if (0x1F <= eee) { + ee = 0x1F; + mm = 0; + } else { + float ff = fabsf(f) + 0.5; + uint32_t ii = bit_cast(ff); + ee = 0; + mm = ii & 0x7FF; + } + + this->raw_ = (ss << 15) | (ee << 10) | mm; + return *this; +} + +inline float16_t::operator float() const { + uint32_t ss = raw_ >> 15; + uint32_t ee = (raw_ >> 10) & 0x1F; + uint32_t mm = raw_ & 0x3FF; + + uint32_t s = ss; + uint32_t eee = ee - 15 + 127; + uint32_t m = mm << 13; + uint32_t e; + + if (0 == ee) { + if (0 == mm) { + e = 0; + } else { + return (ss ? -1 : 1) * std::scalbn((float)mm, -24); + } + } else if (0x1F == ee) { + e = 0xFF; + } else { + e = eee; + } + + uint32_t f = (s << 31) | (e << 23) | m; + + return bit_cast(f); +} + +inline void float16_t::cvt_float_to_float16(const float *src, float16_t *dst, int size) { + // Round to nearest even mode + constexpr int rounding_mode = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC; + + // Process 16 floats (AVX512 is a 512-bit SIMD register) + constexpr int kStep = 16; + + // Process blocks of 16 floats at a time + for (int i = 0; i < size; i += kStep) { + // Load the input floats into a AVX512 register + __m512 input_vector = _mm512_loadu_ps(src + i); + + // Convert the floats to float16_t using AVX512 intrinsics + __m256i output_vector = _mm512_cvtps_ph(input_vector, rounding_mode); + + // Store the converted values in the output array + _mm256_mask_storeu_epi16(reinterpret_cast(dst + i), 0xffff, output_vector); + } + + int remainder = size % kStep; + if (remainder != 0) { + __mmask16 mask = 0xFFFF >> (16 - remainder); + __m512 input_vector = _mm512_maskz_loadu_ps(mask, src + size - remainder); + __m256i output_vector = _mm512_cvtps_ph(input_vector, rounding_mode); + _mm256_mask_storeu_epi16(reinterpret_cast(dst + size - remainder), mask, output_vector); + } +} + +inline void float16_t::cvt_float16_to_float(const float16_t *src, float *dst, int size) { + // Process 16 floats (AVX512 is a 512-bit SIMD register) + constexpr int kStep = 16; + + for (int i = 0; i < size; i += kStep) { + __m256i input_vector = _mm256_maskz_loadu_epi16(0xffff, src + i); + __m512 output_vector = _mm512_cvtph_ps(input_vector); + _mm512_storeu_ps(dst + i, output_vector); + } + + int remainder = size % kStep; + if (remainder != 0) { + __mmask16 mask = 0xFFFF >> (16 - remainder); + __m256i input_vector = _mm256_maskz_loadu_epi16(mask, src + size - remainder); + __m512 output_vector = _mm512_cvtph_ps(input_vector); + _mm512_mask_storeu_ps(dst + size - remainder, mask, output_vector); + } +} + +inline void float16_t::float_add_float16(const float *src1, const float16_t *src2, float *dst, int size) { + constexpr int kStep = 16; + + for (int i = 0; i < size; i += kStep) { + __m512 vec1 = _mm512_loadu_ps(src1 + i); + __m256i _t = _mm256_maskz_loadu_epi16(0xffff, src2 + i); + __m512 vec2 = _mm512_cvtph_ps(_t); + _mm512_storeu_ps(dst + i, vec1 + vec2); + } + + int remainder = size % kStep; + if (remainder != 0) { + __mmask16 mask = 0xFFFF >> (16 - remainder); + __m512 vec1 = _mm512_maskz_loadu_ps(mask, src1 + size - remainder); + __m256i _t = _mm256_maskz_loadu_epi16(mask, src2 + size - remainder); + __m512 vec2 = _mm512_cvtph_ps(_t); + _mm512_mask_storeu_ps(dst + size - remainder, mask, vec1 + vec2); + } +} diff --git a/src/common/kvcache_tensor.h b/src/common/kvcache_tensor.h new file mode 100644 index 0000000..2e16765 --- /dev/null +++ b/src/common/kvcache_tensor.h @@ -0,0 +1,146 @@ +#pragma once +#include +#include +#include +#include +#include + +/** + * Tensor specially designed for KV Cache + * Naturaly, it could be represented in the shape of [seq_length][batch_size][head_num][head_size] + * + * |________ bs[0] ________|_______|_______ bs[N-1] _______| + * seq=0 | | | | ... | | | | + * seq=1 | | | | ... | | | | + * seq=2 | head0 | head1 | head2 | ... | head0 | head1 | head2 | + * ... | | | | ... | | | | + * | | | | ... | | | | + * ````````````````````````````````````````````````````````` + * For better performance, it can be represented as [batch_size][head_num][seq_length][head_size] + * __________________ + * | | ^ + * | | | + * | head0 | | + * | | | + * | | | + * |_______| | + * | | | + * | | | + * | head1 | bs[0] + * | | | + * | | | + * |_______| | + * | | | + * | | | + * | head2 | | + * | | | + * | | v + * `````````````````` + * .... + * Note: The batch size in KVCache can be larger than the batch size in model inference (when beam size > 1) + * The batch size of model inference is smaller to save the computing + * The batch size of KV Cache is larger to make the KV cache expanding easier +*/ +template +class KVCacheTensor { +public: + KVCacheTensor() : maxSeqLen(0), batchSize(0), headNum(0), headSize(0), data(nullptr), allocSize(0) {} + + ~KVCacheTensor() { + if (this->data) { free(this->data); } + } + + void resize(int maxSeqLen, int batchSize, int headNum, int headSize) { + this->maxSeqLen = maxSeqLen; + this->batchSize = batchSize; + this->headNum = headNum; + this->headSize = headSize; + + uint64_t requiredSize = (uint64_t)maxSeqLen * batchSize * headNum * headSize; + if (requiredSize > allocSize) { + this->data = (T *)aligned_alloc(64, requiredSize * sizeof(T)); + if (!this->data) { + printf("Failed to alloc mem for KV Cache [%d][%d][%d][%d].\n", maxSeqLen, batchSize, headNum, headSize); + exit(-1); + } + + allocSize = requiredSize; + } + } + + int getBatchSize() const { return batchSize; } + int getHeadNum() const { return headNum; } + int getHeadSize() const { return headSize; } + + T *getData() { return data; } + + // Get a vector for a specified sequence + T *getSequence(int seqIdx, int batchIdx, int headIdx) { + return data + (seqIdx * batchSize + batchIdx) * (headNum * headSize) + headIdx * headSize; + } + + // Get a head matrix, return the start address and the stride + std::pair getHead(int batchIdx, int headIdx) { + T *addr = data + batchIdx * headNum * headSize + headIdx * headSize; + return std::make_pair(addr, batchSize * headNum * headSize); + } + + /** + * Expand the tensor by broadcasting each sample to multiple beams. + * It is needed when beam_size > 1 by just passing the unique user side samples to do the inference. + * For example, when user_side_bs=2, beam_size=3, it will expand: + * _______________________________________________ + * | bs0 | bs1 | | | | | + * ``````````````````````````````````````````````` + * to + * _______________________________________________ + * | bs0 | bs0 | bs0 | bs1 | bs1 | bs1 | + * ``````````````````````````````````````````````` + */ + void expandAllSequence(int userSideBS, int beamSize, int seqLen) { + if (userSideBS * beamSize != batchSize) { + printf("Cannot expand the KV Cache as userSideBS(%d) * beamSize(%d) != batchSize(%d)\n", userSideBS, + beamSize, batchSize); + return; + } + +#pragma omp parallel for + for (int seq = 0; seq < seqLen; ++seq) { + for (int b = batchSize - 1; b > 0; --b) { + T *dst = getSequence(seq, b, 0); + T *src = getSequence(seq, b / beamSize, 0); + memcpy(dst, src, headNum * headSize * sizeof(T)); + } + } + } + + void expandOneSequence(int userSideBS, int beamSize, int seq) { + for (int b = batchSize - 1; b > 0; --b) { + T *dst = getSequence(seq, b, 0); + T *src = getSequence(seq, b / beamSize, 0); + memcpy(dst, src, headNum * headSize * sizeof(T)); + } + } + + // Below implementation could be a little faster (100.6 vs. 100.9), but also need to modify expand and reorder function + + // // Get a vector for a specified sequence + // T *getSequence(int seqIdx, int batchIdx, int headIdx) { + // return data + batchIdx * headNum * maxSeqLen * headSize + headIdx * maxSeqLen * headSize + seqIdx * headSize; + // } + + // // Get a head matrix, return the start address and the stride + // std::pair getHead(int batchIdx, int headIdx) { + // T *addr = data + batchIdx * headNum * maxSeqLen * headSize + headIdx * maxSeqLen * headSize; + // return std::make_pair(addr, headSize); + // } + +private: + int maxSeqLen; + int batchSize; + int headNum; + int headSize; + + T *data; + uint64_t allocSize; +}; \ No newline at end of file diff --git a/src/common/my_types.h b/src/common/my_types.h new file mode 100644 index 0000000..96b1c34 --- /dev/null +++ b/src/common/my_types.h @@ -0,0 +1,356 @@ +#pragma once + +#include +#include +#include +#include + +typedef int8_t s8; +typedef uint8_t u8; + +#define unlikely(x) __builtin_expect((x), 0) + +extern "C" { +void *xft_numa_alloc(size_t size); +void xft_numa_free(void *start, size_t size); +} + +namespace hpj { + +template +struct is_quantization_type { + static const bool value = false; +}; +template <> +struct is_quantization_type { + static const bool value = true; +}; +template <> +struct is_quantization_type { + static const bool value = true; +}; + +template ::value> +struct MatData { + // A sub matrix of others, if true + bool shadow; + + int buf_alloc_size; + T *buf; + + MatData() { + this->shadow = false; + this->buf = NULL; + this->buf_alloc_size = 0; + } + MatData(T *buf) { + this->shadow = true; + this->buf = buf; + this->buf_alloc_size = 0; + } + void Assign(T *buf) { + if (shadow) { + this->buf = buf; + } else { + Release(); + this->shadow = true; + this->buf = buf; + this->buf_alloc_size = 0; + } + } + void Resize(int rows, int cols, int stride) { + assert(!shadow); + int size = rows * stride; + if (this->buf_alloc_size >= size) { + return; + } else { + if (buf) { xft_numa_free(buf, sizeof(T) * buf_alloc_size); } + this->buf_alloc_size = size; + buf = (T *)xft_numa_alloc(sizeof(T) * size); + if (buf == NULL) { throw std::bad_alloc(); } + } + } + void Release() { + if (!shadow && buf) { + xft_numa_free(buf, sizeof(T) *buf_alloc_size); + buf = NULL; + } + buf_alloc_size = 0; + } + ~MatData() { Release(); } +}; + +enum QuantizationScheme { + qscheme_undefined = 0, + per_tensor_symmetric = 1, + per_tensor_affine = 2, + per_channel_symmetric = 3, + per_channel_affine = 4, +}; + +// The matrix is quantized per row/channel +template +struct MatData { + // A sub matrix of others, if true + bool shadow; + + int buf_alloc_size; + + T *buf; + + QuantizationScheme qscheme; + + union QParam { + struct QParamPerTensor { + float scale; + int32_t zp; + } per_t; + struct QParamPerChannel { + float *scales; + int32_t *zps; + int alloc_size; + } per_c; + } qparam; + + MatData() { + this->shadow = false; + this->buf = NULL; + this->buf_alloc_size = 0; + this->qscheme = qscheme_undefined; + } + void Resize(int rows, int cols, int stride) { + assert(!shadow); + int size = rows * stride; + if (this->buf_alloc_size < size) { + if (buf) { xft_numa_free(buf, sizeof(T) * buf_alloc_size); } + this->buf_alloc_size = size; + buf = (T *)xft_numa_alloc(sizeof(T) * size); + if (buf == NULL) { throw std::bad_alloc(); } + } + // Check the scale and zero point buffer + if ((this->qscheme == per_channel_symmetric || this->qscheme == per_channel_affine) + && this->qparam.per_c.alloc_size < rows) { + if (this->qparam.per_c.scales) { free(this->qparam.per_c.scales); } + this->qparam.per_c.scales = (float *)aligned_alloc(64, sizeof(float) * rows); + if (this->qparam.per_c.scales == NULL) { throw std::bad_alloc(); } + this->qparam.per_c.alloc_size = rows; + // For per_channel_affine, need to check buffer for zero point + if (this->qscheme == per_channel_affine) { + if (this->qparam.per_c.zps) { free(this->qparam.per_c.zps); } + this->qparam.per_c.zps = (int32_t *)aligned_alloc(64, sizeof(int32_t) * rows); + if (this->qparam.per_c.zps == NULL) { throw std::bad_alloc(); } + } + } + } + void Release() { + if (!shadow && buf) { + xft_numa_free(buf, sizeof(T) * buf_alloc_size); + buf = NULL; + } + if (!shadow && (this->qscheme == per_channel_symmetric || this->qscheme == per_channel_affine)) { + free(this->qparam.per_c.scales); + this->qparam.per_c.scales = NULL; + if (this->qscheme == per_channel_affine) { + free(this->qparam.per_c.zps); + this->qparam.per_c.zps = NULL; + } + this->qparam.per_c.alloc_size = 0; + } + buf_alloc_size = 0; + } + void SetQScheme(QuantizationScheme scheme) { + if (scheme == per_tensor_symmetric || scheme == per_tensor_affine) { + // From per_channel to per_tensor + if (unlikely(this->qscheme == per_channel_symmetric || this->qscheme == per_channel_affine)) { + if (this->qparam.per_c.scales) { free(this->qparam.per_c.scales); } + if (this->qparam.per_c.zps) { free(this->qparam.per_c.zps); } + } + this->qparam.per_t.scale = 1.0; + this->qparam.per_t.zp = 0; + } else if (scheme == per_channel_symmetric || scheme == per_channel_affine) { + // From non_per_channel to per_channel + if (this->qscheme != per_channel_symmetric && this->qscheme != per_channel_affine) { + this->qparam.per_c.scales = NULL; + this->qparam.per_c.zps = NULL; + this->qparam.per_c.alloc_size = 0; + } + } + this->qscheme = scheme; + } + float *Scales() { + if (this->qscheme == per_tensor_symmetric || this->qscheme == per_tensor_affine) { + return &qparam.per_t.scale; + } else if (this->qscheme == per_channel_symmetric || this->qscheme == per_channel_affine) { + return qparam.per_c.scales; + } else { + return NULL; + } + } + int32_t *ZeroPoint() { + if (this->qscheme == per_tensor_symmetric || this->qscheme == per_tensor_affine) { + return &qparam.per_t.zp; + } else if (this->qscheme == per_channel_symmetric || this->qscheme == per_channel_affine) { + return qparam.per_c.zps; + } else { + return NULL; + } + } + ~MatData() { Release(); } +}; + +template +class Matrix { +private: + int rows; + int cols; + int stride; + + MatData data; + + Matrix &operator=(const Matrix &m); + +public: + Matrix() { + this->rows = 0; + this->cols = 0; + this->stride = 0; + } + + Matrix(Matrix &m, int start_row, int rows, int start_col, int cols) + : data(m.data.buf + start_row * m.stride + start_col) { + this->rows = rows; + this->cols = cols; + this->stride = m.stride; + } + + Matrix(Matrix &m) : data(m.data) { + this->rows = m.rows; + this->cols = m.cols; + this->stride = m.stride; + } + + // Create dilated matrix, for example, if dilation = 2, then select the 1st, 3rd, 5th, ... lines + Matrix(Matrix &m, int start_row, int dilation, bool unused) : data(m.data.buf + start_row * m.stride) { + this->rows = m.rows / dilation; + this->cols = m.cols; + this->stride = m.stride * dilation; + } + + Matrix(Matrix &m, int start_row, int rows) : data(m.data.buf + start_row * m.stride) { + this->rows = rows; + this->cols = m.cols; + this->stride = m.stride; + } + + Matrix(T *buf, int rows, int cols, int stride) : data(buf) { + this->rows = rows; + this->cols = cols; + this->stride = stride; + } + + ~Matrix() { this->Release(); } + + void Assign(T *buf, int rows, int cols, int stride) { + this->data.Assign(buf); + this->rows = rows; + this->cols = cols; + this->stride = stride; + } + + void Resize(int rows, int cols) { + assert(!data.shadow); + + if (this->rows == rows && this->cols == cols) { return; } + if (rows <= 0 || cols <= 0) { + this->Release(); + return; + } + + // Previously, we used to pad the matrix when the columns aligned with the boundary of 1024. + // However, we discovered that this did not enhance the performance. + // As a result, we have decided to remove this approach. + this->stride = cols; + this->rows = rows; + this->cols = cols; + this->data.Resize(rows, cols, stride); + } + void Resize(int rows, int cols, int stride) { + assert(!data.shadow); + + if (this->rows == rows && this->cols == cols && this->stride == stride) { return; } + if (rows <= 0 || cols <= 0 || stride <= 0) { + this->Release(); + return; + } + this->rows = rows; + this->cols = cols; + this->stride = stride; + this->data.Resize(rows, cols, stride); + } + T *Data() { return data.buf; } + const T *Data() const { return data.buf; } + void SetQScheme(QuantizationScheme qscheme) { data.SetQScheme(qscheme); } + float *Scales() { return data.Scales(); } + int32_t *ZeroPoint() { return data.ZeroPoint(); } + void Release() { + this->data.Release(); + this->rows = 0; + this->cols = 0; + this->stride = 0; + } + int Rows() const { return this->rows; } + int Cols() const { return this->cols; } + int Stride() const { return this->stride; } + T *Row(const int idx) { + // assert(idx < rows_ && idx >= 0); + return this->data.buf + this->stride * idx; + } + const T *Row(const int idx) const { return this->data.buf + this->stride * idx; } + T &operator()(int r, int c) { + // assert(r >= 0 && r < rows_ && c >= 0 && c < cols_); + return *(this->data.buf + r * this->stride + c); + } +}; + +template +class Vector { +private: + T *data; + int size; + int alloc_size; + +public: + Vector() { + data = NULL; + size = 0; + alloc_size = 0; + } + ~Vector() { this->Release(); } + void Resize(int size) { + if (size <= 0) { + this->Release(); + return; + } + if (this->alloc_size >= size) { // space is enough + this->size = size; + return; + } + if (this->data) { xft_numa_free(this->data, sizeof(T) * alloc_size); } + this->alloc_size = size + (16 - (size % 16)) % 16; + this->size = size; + this->data = (T *)xft_numa_alloc(sizeof(T) * alloc_size); + if (this->data == NULL) { throw std::bad_alloc(); } + } + void SetZero() { memset(data, 0, sizeof(T) * size); } + T *Data() { return data; } + void Release() { + if (data) { + xft_numa_free(data, sizeof(T) * alloc_size); + data = NULL; + } + size = 0; + alloc_size = 0; + } + int Size() { return size; } +}; +} // namespace hpj \ No newline at end of file diff --git a/src/common/transformer_ctx.h b/src/common/transformer_ctx.h new file mode 100644 index 0000000..860e557 --- /dev/null +++ b/src/common/transformer_ctx.h @@ -0,0 +1,179 @@ +#pragma once +#include + +#include +#include +#include + +#include "my_types.h" + +struct DecoderContext { + // # of mini-batch + int batchSize; + // # of tokens + int inputSeqLen; + // For custom usage + int reserved1; + + // Other configuration + int vocabSize; + int embeddingSize; + int maxPositions; + int layers; + + // For BERT-base, hidden_size=768 + const int hiddenSize; + // For BERT-base, intermediate_size=3072 + const int intermediateSize; + // For BERT-base, attHeadNum=12 + const int attHeadNum; + const int kvHeadNum; + // attHeadSize = hiddenSize / attHeadNum + int attHeadSize; + // attFactor = 1 / sqrtf(attHeadSize) + float attFactor; + + // norm epsilon + float epsilon; + + // Which split this context is for + const int splitIdx; + // # of splits (the same as NUMA node number in the system) + const int numSplit; + + enum ActivationType { RELU, GELU, SWIGLU, SILU }; + ActivationType actType; + + // # of thread + int numThreads; + + float *qkScores; // attention score + + // Please look into the comments in resize function to see how buffers are arranged + hpj::Matrix normBuf; // buf for the first layer norm + hpj::Matrix tmpBuf; // tmp buffer, same size as output + hpj::Matrix qkvMatMul; // query, key, value + hpj::Matrix imOut; // intermediate output + +private: + float *rawBuffer; + int rawBufSize; // how many floats + +public: + DecoderContext(int _layers, int _hiddenSize, int _attHeadNum, int _kvHeadNum, int _imSize, const std::string &act, + float epsilon, int _vocabSize, int _embeddingSize, int _maxPositions, int _splitIdx, int _splits, + int numThreads = 0) + : layers(_layers) + , hiddenSize(_hiddenSize) + , intermediateSize(_imSize) + , attHeadNum(_attHeadNum) + , kvHeadNum(_kvHeadNum) + , vocabSize(_vocabSize) + , embeddingSize(_embeddingSize) + , maxPositions(_maxPositions) + , splitIdx(_splitIdx) + , numSplit(_splits) + , epsilon(epsilon) { + if (attHeadNum != 0) { + this->attHeadSize = hiddenSize / attHeadNum; + this->attFactor = 1 / sqrtf(attHeadSize); + } + + // Set the default value (don't worry, it can be changed later) + this->batchSize = 1; + this->inputSeqLen = 1; + this->numThreads = numThreads; + + if (numThreads == 0) { +#pragma omp parallel + { + int tid = omp_get_thread_num(); + if (tid == 0) { this->numThreads = omp_get_num_threads(); } + } + } + + this->rawBufSize = 4 * 32 * intermediateSize + 4 * attHeadNum * 32 * 32; // assume bs=4, seq=32 + this->rawBuffer = (float *)aligned_alloc(64, sizeof(float) * rawBufSize); + + if (act == "relu") { + this->actType = RELU; + } else if (act == "gelu") { + this->actType = GELU; + } else if (act == "silu") { + this->actType = SILU; + } else if (act == "swiglu") { + this->actType = SWIGLU; + } else { + printf("unsupported activation: %s\n", act.c_str()); + exit(-1); + } + } + + void dump() { + printf("batch_size=%d\n", batchSize); + printf("inputSeqLen=%d\n", inputSeqLen); + + printf("hiddenSize=%d\n", hiddenSize); + printf("intermediateSize=%d\n", intermediateSize); + printf("attHeadNum=%d\n", attHeadNum); + printf("kvHeadNum=%d\n", kvHeadNum); + printf("attHeadSize=%d\n", attHeadSize); + printf("attFactor=%f\n", attFactor); + + printf("numThreads=%d\n", numThreads); + } + + // Resize to make sure the buffer is big enough + // |---------|---------|--------| + // | normBuf |qkvMatMul|qkScores| + // | | imOut | tmpBuf | + void resize(int batchSize, int inputSeqLen, bool preSeqLen) { + this->batchSize = batchSize; + this->inputSeqLen = inputSeqLen; + + // Check total required size + const int pad = 0; // 4; + int hiddenStride = (hiddenSize % 512 == 0 ? hiddenSize + pad + : hiddenSize); // stride for matrix with columns of hiddenSize + int qkvCols = hiddenSize * 3 / numSplit; + int qkvStride = (qkvCols % 512 == 0 ? qkvCols + pad : qkvCols); // stride for the concated QKV + int imCols = intermediateSize / numSplit; + int imStride = (imCols % 512 == 0 ? imCols + pad : imCols); // stride for intermediate output + + int normSize = batchSize * inputSeqLen * hiddenStride; + int qkvSize = batchSize * inputSeqLen * qkvStride; + int imOutSize = batchSize * inputSeqLen * imStride; + + int headPerSplit = attHeadNum / numSplit; + int presentSeqLen = preSeqLen + 1; + int paddedSize = (presentSeqLen + 15) / 16 * 16; + + // Note: the score buffer for first token generation is not padded + int scoreBufSize = preSeqLen > 0 ? batchSize * headPerSplit * inputSeqLen * paddedSize + : batchSize * headPerSplit * inputSeqLen * inputSeqLen; + int tmpBufSize = batchSize * inputSeqLen * hiddenStride; + + int size1 = normSize; + int size2 = qkvSize < imOutSize ? imOutSize : qkvSize; + int size3 = tmpBufSize < scoreBufSize ? scoreBufSize : tmpBufSize; + + int total = size1 + size2 + size3; + if (total > this->rawBufSize) { + this->rawBufSize = total; + free(this->rawBuffer); + + this->rawBuffer = (float *)aligned_alloc(64, sizeof(float) * rawBufSize); + } + + // Assign the buffer + this->qkScores = this->rawBuffer + size1 + size2; + normBuf.Assign(this->rawBuffer, batchSize * inputSeqLen, hiddenSize, hiddenStride); + tmpBuf.Assign(this->qkScores, batchSize * inputSeqLen, hiddenSize, hiddenStride); + imOut.Assign(this->rawBuffer + size1, batchSize * inputSeqLen, intermediateSize / numSplit, imStride); + qkvMatMul.Assign(this->rawBuffer + size1, batchSize * inputSeqLen, hiddenSize / numSplit * 3, qkvStride); + } + + ~DecoderContext() { + free(this->rawBuffer); + } +}; \ No newline at end of file diff --git a/src/kernels/CMakeLists.txt b/src/kernels/CMakeLists.txt new file mode 100644 index 0000000..29c8e54 --- /dev/null +++ b/src/kernels/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.15.1) + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} KERNELS_SRCS) + +add_library(kernels OBJECT ${KERNELS_SRCS}) +add_dependencies(kernels utils) \ No newline at end of file diff --git a/src/kernels/gemm_kernel_ext.cpp b/src/kernels/gemm_kernel_ext.cpp new file mode 100644 index 0000000..388a5ad --- /dev/null +++ b/src/kernels/gemm_kernel_ext.cpp @@ -0,0 +1,392 @@ +#include "gemm_kernel_ext.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "compile_util.h" + +template +void small_gemm_transb(const float *A, const float *B, float *C, int K, int lda, int ldb, int ldc) { + // vc[0] vc[1] ... vc[N-1] + // vc[N] vc[N+1] ... + // .. + // vc[(M-1)*N] ... + __m512 vc[M * N]; + + int vecs = (K + 15) / 16; // vector size in AVX512 + __mmask16 mask = (K % 16 == 0 ? 0xffff : (1 << (K % 16)) - 1); // mask for last vector + + compile_time_for::op([&vc](auto i) { vc[i] = _mm512_set1_ps(0); }); + + // The last vector is not included + for (int v = 0; v < vecs - 1; ++v) { + const float *pA = A + v * 16; + const float *pB = B + v * 16; + __m512 vb[N]; + __m512 va; + + compile_time_for::op([&](auto i) { + constexpr int idx = i; + // Load from A when reach to first column in vc matrix + if constexpr (idx % N == 0) { + va = _mm512_loadu_ps(pA); + pA += lda; + } + // Load from B when reach to first row in vc matrix + if constexpr (idx < N) { + vb[idx] = _mm512_loadu_ps(pB); + pB += ldb; + } + constexpr int col = idx % N; + vc[idx] = _mm512_fmadd_ps(va, vb[col], vc[idx]); + }); + } + + // The last vector computing, together with data store + { + __m512 vb[N]; + __m512 va; + + const float *pA = A + (vecs - 1) * 16; + const float *pB = B + (vecs - 1) * 16; + float *pC = C; + + compile_time_for::op([&](auto i) { + constexpr int idx = i; + if constexpr (idx % N == 0) { + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + } + if constexpr (idx < N) { + vb[idx] = _mm512_maskz_loadu_ps(mask, pB); + pB += ldb; + } + constexpr int col = idx % N; + vc[idx] = _mm512_fmadd_ps(va, vb[col], vc[idx]); + pC[col] = _mm512_reduce_add_ps(vc[idx]); + // Reach to the row end + if constexpr (i % N == N - 1) { pC += ldc; } + }); + } +} + +void small_gemm_transb_6x4(const float *A, const float *B, float *C, int K, int lda, int ldb, int ldc) { + // vc[00] vc[01] vc[02] vc[03] + // vc[04] vc[05] vc[06] vc[07] + // vc[08] vc[09] vc[10] vc[11] + // vc[12] vc[13] vc[14] vc[15] + // vc[16] vc[17] vc[18] vc[19] + // vc[20] vc[21] vc[22] vc[23] + __m512 vc[24]; + + int vecs = (K + 15) / 16; // vector size in AVX512 + __mmask16 mask = (K % 16 == 0 ? 0xffff : (1 << (K % 16)) - 1); // mask for last vector + + vc[0] = _mm512_set1_ps(0); + vc[1] = _mm512_set1_ps(0); + vc[2] = _mm512_set1_ps(0); + vc[3] = _mm512_set1_ps(0); + vc[4] = _mm512_set1_ps(0); + vc[5] = _mm512_set1_ps(0); + vc[6] = _mm512_set1_ps(0); + vc[7] = _mm512_set1_ps(0); + vc[8] = _mm512_set1_ps(0); + vc[9] = _mm512_set1_ps(0); + vc[10] = _mm512_set1_ps(0); + vc[11] = _mm512_set1_ps(0); + vc[12] = _mm512_set1_ps(0); + vc[13] = _mm512_set1_ps(0); + vc[14] = _mm512_set1_ps(0); + vc[15] = _mm512_set1_ps(0); + vc[16] = _mm512_set1_ps(0); + vc[17] = _mm512_set1_ps(0); + vc[18] = _mm512_set1_ps(0); + vc[19] = _mm512_set1_ps(0); + vc[20] = _mm512_set1_ps(0); + vc[21] = _mm512_set1_ps(0); + vc[22] = _mm512_set1_ps(0); + vc[23] = _mm512_set1_ps(0); + + // The last vector is not included + for (int v = 0; v < vecs - 1; ++v) { + const float *pA = A + v * 16; + const float *pB = B + v * 16; + __m512 vb[4]; + + __m512 va = _mm512_loadu_ps(pA); + pA += lda; + vb[0] = _mm512_loadu_ps(pB); + pB += ldb; + vc[0] = _mm512_fmadd_ps(va, vb[0], vc[0]); + vb[1] = _mm512_loadu_ps(pB); + pB += ldb; + vc[1] = _mm512_fmadd_ps(va, vb[1], vc[1]); + vb[2] = _mm512_loadu_ps(pB); + pB += ldb; + vc[2] = _mm512_fmadd_ps(va, vb[2], vc[2]); + vb[3] = _mm512_loadu_ps(pB); + pB += ldb; + vc[3] = _mm512_fmadd_ps(va, vb[3], vc[3]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[4] = _mm512_fmadd_ps(va, vb[0], vc[4]); + vc[5] = _mm512_fmadd_ps(va, vb[1], vc[5]); + vc[6] = _mm512_fmadd_ps(va, vb[2], vc[6]); + vc[7] = _mm512_fmadd_ps(va, vb[3], vc[7]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[8] = _mm512_fmadd_ps(va, vb[0], vc[8]); + vc[9] = _mm512_fmadd_ps(va, vb[1], vc[9]); + vc[10] = _mm512_fmadd_ps(va, vb[2], vc[10]); + vc[11] = _mm512_fmadd_ps(va, vb[3], vc[11]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[12] = _mm512_fmadd_ps(va, vb[0], vc[12]); + vc[13] = _mm512_fmadd_ps(va, vb[1], vc[13]); + vc[14] = _mm512_fmadd_ps(va, vb[2], vc[14]); + vc[15] = _mm512_fmadd_ps(va, vb[3], vc[15]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[16] = _mm512_fmadd_ps(va, vb[0], vc[16]); + vc[17] = _mm512_fmadd_ps(va, vb[1], vc[17]); + vc[18] = _mm512_fmadd_ps(va, vb[2], vc[18]); + vc[19] = _mm512_fmadd_ps(va, vb[3], vc[19]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[20] = _mm512_fmadd_ps(va, vb[0], vc[20]); + vc[21] = _mm512_fmadd_ps(va, vb[1], vc[21]); + vc[22] = _mm512_fmadd_ps(va, vb[2], vc[22]); + vc[23] = _mm512_fmadd_ps(va, vb[3], vc[23]); + } + + // The last vector computing, together with data store + const float *pA = A + (vecs - 1) * 16; + const float *pB = B + (vecs - 1) * 16; + float *pC = C; + __m512 vb[4]; + + __m512 va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vb[0] = _mm512_maskz_loadu_ps(mask, pB); + pB += ldb; + vc[0] = _mm512_fmadd_ps(va, vb[0], vc[0]); + pC[0] = _mm512_reduce_add_ps(vc[0]); + vb[1] = _mm512_maskz_loadu_ps(mask, pB); + pB += ldb; + vc[1] = _mm512_fmadd_ps(va, vb[1], vc[1]); + pC[1] = _mm512_reduce_add_ps(vc[1]); + vb[2] = _mm512_maskz_loadu_ps(mask, pB); + pB += ldb; + vc[2] = _mm512_fmadd_ps(va, vb[2], vc[2]); + pC[2] = _mm512_reduce_add_ps(vc[2]); + vb[3] = _mm512_maskz_loadu_ps(mask, pB); + pB += ldb; + vc[3] = _mm512_fmadd_ps(va, vb[3], vc[3]); + pC[3] = _mm512_reduce_add_ps(vc[3]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[4] = _mm512_fmadd_ps(va, vb[0], vc[4]); + pC[0] = _mm512_reduce_add_ps(vc[4]); + vc[5] = _mm512_fmadd_ps(va, vb[1], vc[5]); + pC[1] = _mm512_reduce_add_ps(vc[5]); + vc[6] = _mm512_fmadd_ps(va, vb[2], vc[6]); + pC[2] = _mm512_reduce_add_ps(vc[6]); + vc[7] = _mm512_fmadd_ps(va, vb[3], vc[7]); + pC[3] = _mm512_reduce_add_ps(vc[7]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[8] = _mm512_fmadd_ps(va, vb[0], vc[8]); + pC[0] = _mm512_reduce_add_ps(vc[8]); + vc[9] = _mm512_fmadd_ps(va, vb[1], vc[9]); + pC[1] = _mm512_reduce_add_ps(vc[9]); + vc[10] = _mm512_fmadd_ps(va, vb[2], vc[10]); + pC[2] = _mm512_reduce_add_ps(vc[10]); + vc[11] = _mm512_fmadd_ps(va, vb[3], vc[11]); + pC[3] = _mm512_reduce_add_ps(vc[11]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[12] = _mm512_fmadd_ps(va, vb[0], vc[12]); + pC[0] = _mm512_reduce_add_ps(vc[12]); + vc[13] = _mm512_fmadd_ps(va, vb[1], vc[13]); + pC[1] = _mm512_reduce_add_ps(vc[13]); + vc[14] = _mm512_fmadd_ps(va, vb[2], vc[14]); + pC[2] = _mm512_reduce_add_ps(vc[14]); + vc[15] = _mm512_fmadd_ps(va, vb[3], vc[15]); + pC[3] = _mm512_reduce_add_ps(vc[15]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[16] = _mm512_fmadd_ps(va, vb[0], vc[16]); + pC[0] = _mm512_reduce_add_ps(vc[16]); + vc[17] = _mm512_fmadd_ps(va, vb[1], vc[17]); + pC[1] = _mm512_reduce_add_ps(vc[17]); + vc[18] = _mm512_fmadd_ps(va, vb[2], vc[18]); + pC[2] = _mm512_reduce_add_ps(vc[18]); + vc[19] = _mm512_fmadd_ps(va, vb[3], vc[19]); + pC[3] = _mm512_reduce_add_ps(vc[19]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[20] = _mm512_fmadd_ps(va, vb[0], vc[20]); + pC[0] = _mm512_reduce_add_ps(vc[20]); + vc[21] = _mm512_fmadd_ps(va, vb[1], vc[21]); + pC[1] = _mm512_reduce_add_ps(vc[21]); + vc[22] = _mm512_fmadd_ps(va, vb[2], vc[22]); + pC[2] = _mm512_reduce_add_ps(vc[22]); + vc[23] = _mm512_fmadd_ps(va, vb[3], vc[23]); + pC[3] = _mm512_reduce_add_ps(vc[23]); +} + +// M is a fixed small number +template +void small_gemm_transb(const float *A, const float *B, float *C, int N, int K, int lda, int ldb, int ldc) { + int j = 0; + const float *pA = A; + constexpr int NB = 4; + + for (; j + NB - 1 < N; j += NB) { + const float *pB = B + j * ldb; + if constexpr (M == 6 && NB == 4) { + small_gemm_transb_6x4(pA, pB, C + j, K, lda, ldb, ldc); + } else { + small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); + } + } + + // Remain part in B + if (j < N) { + const float *pB = B + j * ldb; + switch (N - j) { + case 2: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 1: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 3: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + } + } +} + +// A: M x K +// B: N x K +// C: M x N +void small_gemm_transb(const float *A, const float *B, float *C, int M, int N, int K, int lda, int ldb, int ldc) { + int i = 0; + constexpr int MB = 6; + + for (i = 0; i + MB - 1 < M; i += MB) { + const float *pA = A + i * lda; + small_gemm_transb(pA, B, C + i * ldc, N, K, lda, ldb, ldc); + } + + // Remain part in A + if (i < M) { + const float *pA = A + i * lda; + const int remain = M - i; + + switch (remain) { + case 2: small_gemm_transb<2>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 4: small_gemm_transb<4>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 1: small_gemm_transb<1>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 3: small_gemm_transb<3>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 5: small_gemm_transb<5>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// To check if the block can be skipped by checking if the attention mask is the lowest value +// lda: leading dimension of the attention mask +template +inline bool isBlockSkippable(const float *attnMask, int lda, int rows) { + if constexpr (COLS != 4) { + printf("COLS=%d is not supported in isBlockSkippable\n", COLS); + exit(-1); + } + + const float lowest = std::numeric_limits::lowest(); + + if (*(attnMask + (rows - 1) * lda) == lowest) { // last row is the lowest, then most likely all lowest + for (int i = 0; i < rows; ++i) { + const float *p = attnMask + i * lda; + if (p[0] != lowest || p[1] != lowest || p[2] != lowest || p[3] != lowest) { return false; } + } + return true; + } + + return false; +} + +// M is a fixed small number +template +void small_gemm_transb( + const float *attnMask, const float *A, const float *B, float *C, int N, int K, int lda, int ldb, int ldc) { + int j = 0; + const float *pA = A; + constexpr int NB = 4; + + for (; j + NB - 1 < N; j += NB) { + const float *pB = B + j * ldb; + if (isBlockSkippable(attnMask + j, N, M)) { continue; } + + if constexpr (M == 6 && NB == 4) { + small_gemm_transb_6x4(pA, pB, C + j, K, lda, ldb, ldc); + } else { + small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); + } + } + + // Remain part in B + if (j < N) { + const float *pB = B + j * ldb; + switch (N - j) { + case 2: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 1: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 3: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + } + } +} + +// If attention mask is the lowest value in some position, skip the computation +// attnMask: attention mask with the shape of (M, N) +void small_gemm_transb(const float *attnMask, const float *A, const float *B, float *C, int M, int N, int K, int lda, + int ldb, int ldc) { + int i = 0; + constexpr int MB = 6; + + for (i = 0; i + MB - 1 < M; i += MB) { + const float *pA = A + i * lda; + small_gemm_transb(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); + } + + // Remain part in A + if (i < M) { + const float *pA = A + i * lda; + const int remain = M - i; + + switch (remain) { + case 2: small_gemm_transb<2>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 4: small_gemm_transb<4>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 1: small_gemm_transb<1>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 3: small_gemm_transb<3>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 5: small_gemm_transb<5>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + } + } +} \ No newline at end of file diff --git a/src/kernels/gemm_kernel_ext.h b/src/kernels/gemm_kernel_ext.h new file mode 100644 index 0000000..0c96040 --- /dev/null +++ b/src/kernels/gemm_kernel_ext.h @@ -0,0 +1,12 @@ +#pragma once +#include "float16.h" + +// Single thread small gemm +void small_gemm_transb(const float *A, const float *B, float *C, int M, int N, int K, int lda, int ldb, int ldc); +void small_gemm_transb(const float *A, const float16_t *B, float *C, int M, int N, int K, int lda, int ldb, int ldc); + +// Single thread small gemm with attention mask (skip skippable computation according to attnMask) +void small_gemm_transb(const float *attnMask, const float *A, const float *B, float *C, int M, int N, int K, int lda, + int ldb, int ldc); +void small_gemm_transb(const float *attnMask, const float *A, const float16_t *B, float *C, int M, int N, int K, + int lda, int ldb, int ldc); diff --git a/src/kernels/gemm_kernel_ext_fp16.cpp b/src/kernels/gemm_kernel_ext_fp16.cpp new file mode 100644 index 0000000..9f99d1a --- /dev/null +++ b/src/kernels/gemm_kernel_ext_fp16.cpp @@ -0,0 +1,407 @@ +// Generated from below script: +// with open("./gemm_kernel_ext.cpp", "r") as in_file, open("./gemm_kernel_ext_fp16.cpp", "w") as out_file: +// # Loop over each line in the input file +// for line in in_file: +// if "float *B" in line: +// line = line.replace("float *B", "float16_t *B") +// if "const float *pB" in line: +// line = line.replace("const float *pB", "const float16_t *pB") +// # Change the method to load from B +// if "_mm512_loadu_ps(pB)" in line: +// line = line.replace("_mm512_loadu_ps(pB)", "_mm512_cvtph_ps(_mm256_maskz_loadu_epi16(0xFFFF, pB))") +// if "_mm512_maskz_loadu_ps(mask, pB)" in line: +// line = line.replace("_mm512_maskz_loadu_ps(mask, pB)", "_mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask, pB))") +// # Write the modified line to the output file +// out_file.write(line) +#include "gemm_kernel_ext.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "compile_util.h" + +template +void small_gemm_transb(const float *A, const float16_t *B, float *C, int K, int lda, int ldb, int ldc) { + // vc[0] vc[1] ... vc[N-1] + // vc[N] vc[N+1] ... + // .. + // vc[(M-1)*N] ... + __m512 vc[M * N]; + + int vecs = (K + 15) / 16; // vector size in AVX512 + __mmask16 mask = (K % 16 == 0 ? 0xffff : (1 << (K % 16)) - 1); // mask for last vector + + compile_time_for::op([&vc](auto i) { vc[i] = _mm512_set1_ps(0); }); + + // The last vector is not included + for (int v = 0; v < vecs - 1; ++v) { + const float *pA = A + v * 16; + const float16_t *pB = B + v * 16; + __m512 vb[N]; + __m512 va; + + compile_time_for::op([&](auto i) { + constexpr int idx = i; + // Load from A when reach to first column in vc matrix + if constexpr (idx % N == 0) { + va = _mm512_loadu_ps(pA); + pA += lda; + } + // Load from B when reach to first row in vc matrix + if constexpr (idx < N) { + vb[idx] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(0xFFFF, pB)); + pB += ldb; + } + constexpr int col = idx % N; + vc[idx] = _mm512_fmadd_ps(va, vb[col], vc[idx]); + }); + } + + // The last vector computing, together with data store + { + __m512 vb[N]; + __m512 va; + + const float *pA = A + (vecs - 1) * 16; + const float16_t *pB = B + (vecs - 1) * 16; + float *pC = C; + + compile_time_for::op([&](auto i) { + constexpr int idx = i; + if constexpr (idx % N == 0) { + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + } + if constexpr (idx < N) { + vb[idx] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask, pB)); + pB += ldb; + } + constexpr int col = idx % N; + vc[idx] = _mm512_fmadd_ps(va, vb[col], vc[idx]); + pC[col] = _mm512_reduce_add_ps(vc[idx]); + // Reach to the row end + if constexpr (i % N == N - 1) { pC += ldc; } + }); + } +} + +void small_gemm_transb_6x4(const float *A, const float16_t *B, float *C, int K, int lda, int ldb, int ldc) { + // vc[00] vc[01] vc[02] vc[03] + // vc[04] vc[05] vc[06] vc[07] + // vc[08] vc[09] vc[10] vc[11] + // vc[12] vc[13] vc[14] vc[15] + // vc[16] vc[17] vc[18] vc[19] + // vc[20] vc[21] vc[22] vc[23] + __m512 vc[24]; + + int vecs = (K + 15) / 16; // vector size in AVX512 + __mmask16 mask = (K % 16 == 0 ? 0xffff : (1 << (K % 16)) - 1); // mask for last vector + + vc[0] = _mm512_set1_ps(0); + vc[1] = _mm512_set1_ps(0); + vc[2] = _mm512_set1_ps(0); + vc[3] = _mm512_set1_ps(0); + vc[4] = _mm512_set1_ps(0); + vc[5] = _mm512_set1_ps(0); + vc[6] = _mm512_set1_ps(0); + vc[7] = _mm512_set1_ps(0); + vc[8] = _mm512_set1_ps(0); + vc[9] = _mm512_set1_ps(0); + vc[10] = _mm512_set1_ps(0); + vc[11] = _mm512_set1_ps(0); + vc[12] = _mm512_set1_ps(0); + vc[13] = _mm512_set1_ps(0); + vc[14] = _mm512_set1_ps(0); + vc[15] = _mm512_set1_ps(0); + vc[16] = _mm512_set1_ps(0); + vc[17] = _mm512_set1_ps(0); + vc[18] = _mm512_set1_ps(0); + vc[19] = _mm512_set1_ps(0); + vc[20] = _mm512_set1_ps(0); + vc[21] = _mm512_set1_ps(0); + vc[22] = _mm512_set1_ps(0); + vc[23] = _mm512_set1_ps(0); + + // The last vector is not included + for (int v = 0; v < vecs - 1; ++v) { + const float *pA = A + v * 16; + const float16_t *pB = B + v * 16; + __m512 vb[4]; + + __m512 va = _mm512_loadu_ps(pA); + pA += lda; + vb[0] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(0xFFFF, pB)); + pB += ldb; + vc[0] = _mm512_fmadd_ps(va, vb[0], vc[0]); + vb[1] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(0xFFFF, pB)); + pB += ldb; + vc[1] = _mm512_fmadd_ps(va, vb[1], vc[1]); + vb[2] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(0xFFFF, pB)); + pB += ldb; + vc[2] = _mm512_fmadd_ps(va, vb[2], vc[2]); + vb[3] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(0xFFFF, pB)); + pB += ldb; + vc[3] = _mm512_fmadd_ps(va, vb[3], vc[3]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[4] = _mm512_fmadd_ps(va, vb[0], vc[4]); + vc[5] = _mm512_fmadd_ps(va, vb[1], vc[5]); + vc[6] = _mm512_fmadd_ps(va, vb[2], vc[6]); + vc[7] = _mm512_fmadd_ps(va, vb[3], vc[7]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[8] = _mm512_fmadd_ps(va, vb[0], vc[8]); + vc[9] = _mm512_fmadd_ps(va, vb[1], vc[9]); + vc[10] = _mm512_fmadd_ps(va, vb[2], vc[10]); + vc[11] = _mm512_fmadd_ps(va, vb[3], vc[11]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[12] = _mm512_fmadd_ps(va, vb[0], vc[12]); + vc[13] = _mm512_fmadd_ps(va, vb[1], vc[13]); + vc[14] = _mm512_fmadd_ps(va, vb[2], vc[14]); + vc[15] = _mm512_fmadd_ps(va, vb[3], vc[15]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[16] = _mm512_fmadd_ps(va, vb[0], vc[16]); + vc[17] = _mm512_fmadd_ps(va, vb[1], vc[17]); + vc[18] = _mm512_fmadd_ps(va, vb[2], vc[18]); + vc[19] = _mm512_fmadd_ps(va, vb[3], vc[19]); + + va = _mm512_loadu_ps(pA); + pA += lda; + vc[20] = _mm512_fmadd_ps(va, vb[0], vc[20]); + vc[21] = _mm512_fmadd_ps(va, vb[1], vc[21]); + vc[22] = _mm512_fmadd_ps(va, vb[2], vc[22]); + vc[23] = _mm512_fmadd_ps(va, vb[3], vc[23]); + } + + // The last vector computing, together with data store + const float *pA = A + (vecs - 1) * 16; + const float16_t *pB = B + (vecs - 1) * 16; + float *pC = C; + __m512 vb[4]; + + __m512 va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vb[0] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask, pB)); + pB += ldb; + vc[0] = _mm512_fmadd_ps(va, vb[0], vc[0]); + pC[0] = _mm512_reduce_add_ps(vc[0]); + vb[1] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask, pB)); + pB += ldb; + vc[1] = _mm512_fmadd_ps(va, vb[1], vc[1]); + pC[1] = _mm512_reduce_add_ps(vc[1]); + vb[2] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask, pB)); + pB += ldb; + vc[2] = _mm512_fmadd_ps(va, vb[2], vc[2]); + pC[2] = _mm512_reduce_add_ps(vc[2]); + vb[3] = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask, pB)); + pB += ldb; + vc[3] = _mm512_fmadd_ps(va, vb[3], vc[3]); + pC[3] = _mm512_reduce_add_ps(vc[3]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[4] = _mm512_fmadd_ps(va, vb[0], vc[4]); + pC[0] = _mm512_reduce_add_ps(vc[4]); + vc[5] = _mm512_fmadd_ps(va, vb[1], vc[5]); + pC[1] = _mm512_reduce_add_ps(vc[5]); + vc[6] = _mm512_fmadd_ps(va, vb[2], vc[6]); + pC[2] = _mm512_reduce_add_ps(vc[6]); + vc[7] = _mm512_fmadd_ps(va, vb[3], vc[7]); + pC[3] = _mm512_reduce_add_ps(vc[7]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[8] = _mm512_fmadd_ps(va, vb[0], vc[8]); + pC[0] = _mm512_reduce_add_ps(vc[8]); + vc[9] = _mm512_fmadd_ps(va, vb[1], vc[9]); + pC[1] = _mm512_reduce_add_ps(vc[9]); + vc[10] = _mm512_fmadd_ps(va, vb[2], vc[10]); + pC[2] = _mm512_reduce_add_ps(vc[10]); + vc[11] = _mm512_fmadd_ps(va, vb[3], vc[11]); + pC[3] = _mm512_reduce_add_ps(vc[11]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[12] = _mm512_fmadd_ps(va, vb[0], vc[12]); + pC[0] = _mm512_reduce_add_ps(vc[12]); + vc[13] = _mm512_fmadd_ps(va, vb[1], vc[13]); + pC[1] = _mm512_reduce_add_ps(vc[13]); + vc[14] = _mm512_fmadd_ps(va, vb[2], vc[14]); + pC[2] = _mm512_reduce_add_ps(vc[14]); + vc[15] = _mm512_fmadd_ps(va, vb[3], vc[15]); + pC[3] = _mm512_reduce_add_ps(vc[15]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[16] = _mm512_fmadd_ps(va, vb[0], vc[16]); + pC[0] = _mm512_reduce_add_ps(vc[16]); + vc[17] = _mm512_fmadd_ps(va, vb[1], vc[17]); + pC[1] = _mm512_reduce_add_ps(vc[17]); + vc[18] = _mm512_fmadd_ps(va, vb[2], vc[18]); + pC[2] = _mm512_reduce_add_ps(vc[18]); + vc[19] = _mm512_fmadd_ps(va, vb[3], vc[19]); + pC[3] = _mm512_reduce_add_ps(vc[19]); + pC += ldc; + + va = _mm512_maskz_loadu_ps(mask, pA); + pA += lda; + vc[20] = _mm512_fmadd_ps(va, vb[0], vc[20]); + pC[0] = _mm512_reduce_add_ps(vc[20]); + vc[21] = _mm512_fmadd_ps(va, vb[1], vc[21]); + pC[1] = _mm512_reduce_add_ps(vc[21]); + vc[22] = _mm512_fmadd_ps(va, vb[2], vc[22]); + pC[2] = _mm512_reduce_add_ps(vc[22]); + vc[23] = _mm512_fmadd_ps(va, vb[3], vc[23]); + pC[3] = _mm512_reduce_add_ps(vc[23]); +} + +// M is a fixed small number +template +void small_gemm_transb(const float *A, const float16_t *B, float *C, int N, int K, int lda, int ldb, int ldc) { + int j = 0; + const float *pA = A; + constexpr int NB = 4; + + for (; j + NB - 1 < N; j += NB) { + const float16_t *pB = B + j * ldb; + if constexpr (M == 6 && NB == 4) { + small_gemm_transb_6x4(pA, pB, C + j, K, lda, ldb, ldc); + } else { + small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); + } + } + + // Remain part in B + if (j < N) { + const float16_t *pB = B + j * ldb; + switch (N - j) { + case 2: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 1: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 3: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + } + } +} + +// A: M x K +// B: N x K +// C: M x N +void small_gemm_transb(const float *A, const float16_t *B, float *C, int M, int N, int K, int lda, int ldb, int ldc) { + int i = 0; + constexpr int MB = 6; + + for (i = 0; i + MB - 1 < M; i += MB) { + const float *pA = A + i * lda; + small_gemm_transb(pA, B, C + i * ldc, N, K, lda, ldb, ldc); + } + + // Remain part in A + if (i < M) { + const float *pA = A + i * lda; + const int remain = M - i; + + switch (remain) { + case 2: small_gemm_transb<2>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 4: small_gemm_transb<4>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 1: small_gemm_transb<1>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 3: small_gemm_transb<3>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 5: small_gemm_transb<5>(pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// To check if the block can be skipped by checking if the attention mask is the lowest value +// lda: leading dimension of the attention mask +template +inline bool isBlockSkippable(const float *attnMask, int lda, int rows) { + if constexpr (COLS != 4) { + printf("COLS=%d is not supported in isBlockSkippable\n", COLS); + exit(-1); + } + + const float lowest = std::numeric_limits::lowest(); + + if (*(attnMask + (rows - 1) * lda) == lowest) { // last row is the lowest, then most likely all lowest + for (int i = 0; i < rows; ++i) { + const float *p = attnMask + i * lda; + if (p[0] != lowest || p[1] != lowest || p[2] != lowest || p[3] != lowest) { return false; } + } + return true; + } + + return false; +} + +// M is a fixed small number +template +void small_gemm_transb( + const float *attnMask, const float *A, const float16_t *B, float *C, int N, int K, int lda, int ldb, int ldc) { + int j = 0; + const float *pA = A; + constexpr int NB = 4; + + for (; j + NB - 1 < N; j += NB) { + const float16_t *pB = B + j * ldb; + if (isBlockSkippable(attnMask + j, N, M)) { continue; } + + if constexpr (M == 6 && NB == 4) { + small_gemm_transb_6x4(pA, pB, C + j, K, lda, ldb, ldc); + } else { + small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); + } + } + + // Remain part in B + if (j < N) { + const float16_t *pB = B + j * ldb; + switch (N - j) { + case 2: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 1: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + case 3: small_gemm_transb(pA, pB, C + j, K, lda, ldb, ldc); break; + } + } +} + +// If attention mask is the lowest value in some position, skip the computation +// attnMask: attention mask with the shape of (M, N) +void small_gemm_transb(const float *attnMask, const float *A, const float16_t *B, float *C, int M, int N, int K, int lda, + int ldb, int ldc) { + int i = 0; + constexpr int MB = 6; + + for (i = 0; i + MB - 1 < M; i += MB) { + const float *pA = A + i * lda; + small_gemm_transb(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); + } + + // Remain part in A + if (i < M) { + const float *pA = A + i * lda; + const int remain = M - i; + + switch (remain) { + case 2: small_gemm_transb<2>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 4: small_gemm_transb<4>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 1: small_gemm_transb<1>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 3: small_gemm_transb<3>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + case 5: small_gemm_transb<5>(attnMask + i * N, pA, B, C + i * ldc, N, K, lda, ldb, ldc); break; + } + } +} \ No newline at end of file diff --git a/src/kernels/layernorm_kernels.cpp b/src/kernels/layernorm_kernels.cpp new file mode 100644 index 0000000..630419c --- /dev/null +++ b/src/kernels/layernorm_kernels.cpp @@ -0,0 +1,142 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#include + +#include "bfloat16.h" +#include "dtype.h" +#include "float16.h" +#include "intrinsic_ext.h" +#include "layernorm_kernels.h" +#include "my_types.h" + +namespace xft { + +void invokeLayerNorm(float *output, const float *input, const float *gamma, const float *beta, const int rows, + const int size, int iStride, int oStride, const float epsilon) { + + if (iStride == -1) iStride = size; + if (oStride == -1) oStride = size; + +#pragma omp parallel for + for (int r = 0; r < rows; ++r) { + const float *px = input + r * iStride; + float *py = output + r * oStride; + + float sum = 0; + float squareSum = 0; + + __m512 vsum = _mm512_set1_ps(0); + __m512 vsqare = _mm512_set1_ps(0); + + for (int col = 0; col < size; col += 16) { + int remain = size - col; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + // SUM(x) + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + vsum = _mm512_add_ps(vsum, vx); + + // SUM(x*x) + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + + sum = _mm512_reduce_add_ps(vsum); + squareSum = _mm512_reduce_add_ps(vsqare); + + // Mean + float mean = sum / size; + __m512 vmean = _mm512_set1_ps(mean); + + // Variance + float var = 1 / sqrt(squareSum / size - mean * mean + epsilon); + __m512 vvar = _mm512_set1_ps(var); + + for (int col = 0; col < size; col += 16) { + int remain = size - col; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + __m512 vgamma = _mm512_maskz_loadu_ps(mask, gamma + col); + __m512 vbeta = _mm512_maskz_loadu_ps(mask, beta + col); + __m512 vy = (vx - vmean) * vgamma * vvar + vbeta; + _mm512_mask_storeu_ps(py + col, mask, vy); + } + } +} + +void invokeLayerNorm(bfloat16_t *output, const bfloat16_t *input, const bfloat16_t *gamma, const bfloat16_t *beta, + const int rows, const int size, int iStride, int oStride, const float epsilon) { + + if (iStride == -1) iStride = size; + if (oStride == -1) oStride = size; + +#pragma omp parallel for + for (int r = 0; r < rows; ++r) { + const bfloat16_t *px = input + r * iStride; + bfloat16_t *py = output + r * oStride; + + float sum = 0; + float squareSum = 0; + + __m512 vsum = _mm512_set1_ps(0); + __m512 vsqare = _mm512_set1_ps(0); + + for (int col = 0; col < size; col += 16) { + int remain = size - col; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + // SUM(x) + __m512 vx = _mm512_maskz_loadu_pbh(mask, px + col); + vsum = _mm512_add_ps(vsum, vx); + + // SUM(x*x) + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + + sum = _mm512_reduce_add_ps(vsum); + squareSum = _mm512_reduce_add_ps(vsqare); + + // Mean + float mean = sum / size; + __m512 vmean = _mm512_set1_ps(mean); + + // Variance + float var = 1 / sqrt(squareSum / size - mean * mean + epsilon); + __m512 vvar = _mm512_set1_ps(var); + + for (int col = 0; col < size; col += 16) { + int remain = size - col; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + __m512 vx = _mm512_maskz_loadu_pbh(mask, px + col); + __m512 vgamma = _mm512_maskz_loadu_pbh(mask, gamma + col); + __m512 vbeta = _mm512_maskz_loadu_pbh(mask, beta + col); + __m512 vy = (vx - vmean) * vgamma * vvar + vbeta; + _mm512_mask_storeu_pbh(py + col, mask, vy); + } + } +} + +void invokeLayerNorm(DataType dt, void *output, const void *input, const void *gamma, const void *beta, const int rows, + const int size, int iStride, int oStride, const float epsilon) { + if (dt == DataType::bf16) { + invokeLayerNorm((bfloat16_t *)output, (const bfloat16_t *)input, (const bfloat16_t *)gamma, + (const bfloat16_t *)beta, rows, size, iStride, oStride, epsilon); + } +} + +} // namespace xft \ No newline at end of file diff --git a/src/kernels/layernorm_kernels.h b/src/kernels/layernorm_kernels.h new file mode 100644 index 0000000..4b7e79b --- /dev/null +++ b/src/kernels/layernorm_kernels.h @@ -0,0 +1,90 @@ +#pragma once + +#include + +#include "my_types.h" +#include "float16.h" + +namespace xft { + +template +struct LayerNormWeight { + const T *gamma = nullptr; + const T *beta = nullptr; +}; + +template +inline void invokeLayerNorm(T *output, const T *input, + const T *gamma, const T *beta, + const int rows, const int size, + int iStride = -1, int oStride = -1, + const float epsilon = 1e-5) { + if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v) { + printf("Type %s not supported!\n", typeid(T).name()); + exit(-1); + } else { + printf("Type %s not supported!\n", typeid(T).name()); + exit(-1); + } +} + +template <> +inline void invokeLayerNorm(float *output, const float *input, + const float *gamma, const float *beta, + const int rows, const int size, + int iStride, int oStride, + const float epsilon) { + + if (iStride == -1) iStride = size; + if (oStride == -1) oStride = size; + +#pragma omp parallel for + for (int r = 0; r < rows; ++r) { + const float *px = input + r * iStride; + float *py = output + r * oStride; + + float sum = 0; + float squareSum = 0; + + __m512 vsum = _mm512_set1_ps(0); + __m512 vsqare = _mm512_set1_ps(0); + + for (int col = 0; col < size; col += 16) { + int remain = size - col; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + // SUM(x) + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + vsum = _mm512_add_ps(vsum, vx); + + // SUM(x*x) + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + + sum = _mm512_reduce_add_ps(vsum); + squareSum = _mm512_reduce_add_ps(vsqare); + + // Mean + float mean = sum / size; + __m512 vmean = _mm512_set1_ps(mean); + + // Variance + float var = 1 / sqrt(squareSum / size - mean * mean + epsilon); + __m512 vvar = _mm512_set1_ps(var); + + for (int col = 0; col < size; col += 16) { + int remain = size - col; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + __m512 vgamma = _mm512_maskz_loadu_ps(mask, gamma + col); + __m512 vbeta = _mm512_maskz_loadu_ps(mask, beta + col); + __m512 vy = (vx - vmean) * vgamma * vvar + vbeta; + _mm512_mask_storeu_ps(py + col, mask, vy); + } + } +} +} // namespace xft \ No newline at end of file diff --git a/src/kernels/rmsnorm_kernels.cpp b/src/kernels/rmsnorm_kernels.cpp new file mode 100644 index 0000000..25cd991 --- /dev/null +++ b/src/kernels/rmsnorm_kernels.cpp @@ -0,0 +1,138 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#include + +#include "bfloat16.h" +#include "dtype.h" +#include "float16.h" +#include "intrinsic_ext.h" +#include "my_types.h" +#include "rmsnorm_kernels.h" + +namespace xft { + +void invokeRmsNorm(float *output, const float *input, const float *weight, int rows, int cols, int iStride, int oStride, + float epsilon) { + int size = cols; + + if (iStride == -1) iStride = cols; + if (oStride == -1) oStride = cols; + +#pragma omp parallel for + for (int r = 0; r < rows; ++r) { + const float *px = input + r * iStride; + float *py = output + r * oStride; + + float squareSum = 0; + + __m512 vsqare = _mm512_set1_ps(0); + + int col = 0; + for (; col + 15 < size; col += 16) { + // SUM(x*x) + __m512 vx = _mm512_loadu_ps(px + col); + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + if (col < size) { + __mmask16 mask = (1 << (size - col)) - 1; + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + + squareSum = _mm512_reduce_add_ps(vsqare); + + // Variance + float var = 1 / sqrt(squareSum / size + epsilon); + __m512 vvar = _mm512_set1_ps(var); + + for (col = 0; col + 15 < size; col += 16) { + __m512 vx = _mm512_loadu_ps(px + col); + __m512 vw = _mm512_loadu_ps(weight + col); + __m512 vy = vx * vvar * vw; + _mm512_storeu_ps(py + col, vy); + } + if (col < size) { + __mmask16 mask = (1 << (size - col)) - 1; + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + __m512 vw = _mm512_maskz_loadu_ps(mask, weight + col); + __m512 vy = vx * vvar * vw; + _mm512_mask_storeu_ps(py + col, mask, vy); + } + } // end for rows +} + +void invokeRmsNorm(bfloat16_t *output, const bfloat16_t *input, const bfloat16_t *weight, int rows, int cols, + int iStride, int oStride, float epsilon) { + int size = cols; + + if (iStride == -1) iStride = cols; + if (oStride == -1) oStride = cols; + +#pragma omp parallel for + for (int r = 0; r < rows; ++r) { + const bfloat16_t *px = input + r * iStride; + bfloat16_t *py = output + r * oStride; + + float squareSum = 0; + + __m512 vsqare = _mm512_set1_ps(0); + + int col = 0; + for (; col + 15 < size; col += 16) { + // SUM(x*x) + __m512 vx = _mm512_loadu_pbh(px + col); + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + if (col < size) { + __mmask16 mask = (1 << (size - col)) - 1; + __m512 vx = _mm512_maskz_loadu_pbh(mask, px + col); + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + + squareSum = _mm512_reduce_add_ps(vsqare); + + // Variance + float var = 1 / sqrt(squareSum / size + epsilon); + __m512 vvar = _mm512_set1_ps(var); + + for (col = 0; col + 15 < size; col += 16) { + __m512 vx = _mm512_loadu_pbh(px + col); + __m512 vw = _mm512_loadu_pbh(weight + col); + __m512 vy = vx * vvar * vw; + _mm512_storeu_pbh(py + col, vy); + } + if (col < size) { + __mmask16 mask = (1 << (size - col)) - 1; + __m512 vx = _mm512_maskz_loadu_pbh(mask, px + col); + __m512 vw = _mm512_maskz_loadu_pbh(mask, weight + col); + __m512 vy = vx * vvar * vw; + _mm512_mask_storeu_pbh(py + col, mask, vy); + } + } // end for rows +} + +void invokeRmsNorm(DataType dt, void *output, const void *input, const void *weight, int rows, int cols, int iStride, + int oStride, float epsilon) { + if (dt == DataType::bf16) { + invokeRmsNorm((bfloat16_t *)output, (const bfloat16_t *)input, (const bfloat16_t *)weight, rows, cols, iStride, + oStride, epsilon); + } +} + +} // namespace xft \ No newline at end of file diff --git a/src/kernels/rmsnorm_kernels.h b/src/kernels/rmsnorm_kernels.h new file mode 100644 index 0000000..51869a5 --- /dev/null +++ b/src/kernels/rmsnorm_kernels.h @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once + +#include + +#include "bfloat16.h" +#include "float16.h" +#include "my_types.h" + +namespace xft { + +void invokeRmsNorm(float *output, const float *input, const float *weight, int rows, int cols, int iStride = -1, + int oStride = -1, float epsilon = 1e-6); + +void invokeRmsNorm(bfloat16_t *output, const bfloat16_t *input, const bfloat16_t *weight, int rows, int cols, + int iStride = -1, int oStride = -1, float epsilon = 1e-6); + +} // namespace xft \ No newline at end of file diff --git a/src/layers/CMakeLists.txt b/src/layers/CMakeLists.txt new file mode 100644 index 0000000..b68c87b --- /dev/null +++ b/src/layers/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.15.1) + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} LAYERS_SRCS) + +add_library(layers OBJECT ${LAYERS_SRCS}) +add_dependencies(layers utils) diff --git a/src/layers/alibi_embedding.cpp b/src/layers/alibi_embedding.cpp new file mode 100644 index 0000000..4ee7ef5 --- /dev/null +++ b/src/layers/alibi_embedding.cpp @@ -0,0 +1,56 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#include "alibi_embedding.h" +#include +#include "compile_util.h" + +AlibiEmbedding::AlibiEmbedding(const int headNum, const int seqLen) { + maxLen = seqLen; + maxHeadNums = headNum; + alibiGetRelativePos(maxLen); + alibiGetSlope(maxHeadNums); +} + +void AlibiEmbedding::alibiGetBias(const int headIdx, const int seqLen, float *biasMatrx) { + REQUIRES(headIdx < maxHeadNums, "Alibi Embedding ERROR, headIdx is exceeds max head nums."); + if (seqLen > maxLen) { + maxLen = seqLen; + alibiGetRelativePos(maxLen); + } + for (size_t i = 0; i < seqLen; i++) { + for (size_t j = 0; j < seqLen; j++) { + int index = i * seqLen + j; + biasMatrx[index] = posMatrix[index] * slopeM[headIdx]; + } + } +} + +void AlibiEmbedding::alibiGetRelativePos(const int seqLen) { + posMatrix = (int *)aligned_alloc(64, seqLen * seqLen * sizeof(int)); + for (int i = 0; i < seqLen; i++) { + for (int j = 0; j < seqLen; j++) { + posMatrix[i * seqLen + j] = j - i; + } + } +} + +void AlibiEmbedding::alibiGetSlope(const int headNum) { + slopeM = (float *)aligned_alloc(64, headNum * sizeof(float)); + float x = std::pow(2, 8); + x = std::pow(x, 1.0 / headNum); + for (int i = 0; i < headNum; i++) { + slopeM[i] = 1 / std::pow(x, i + 1); + } +} \ No newline at end of file diff --git a/src/layers/alibi_embedding.h b/src/layers/alibi_embedding.h new file mode 100644 index 0000000..fbf68bf --- /dev/null +++ b/src/layers/alibi_embedding.h @@ -0,0 +1,41 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once +#include + +class AlibiEmbedding { +public: + AlibiEmbedding(const int headNum, const int seqLen); + + ~AlibiEmbedding() { + maxLen = 0; + maxHeadNums = 0; + if (posMatrix != nullptr) free(posMatrix); + if (slopeM != nullptr) free(slopeM); + } + + void alibiGetRelativePos(const int seqLen); + + void alibiGetSlope(const int headNum); + + // headIdx is [0,n] + void alibiGetBias(const int headIdx, const int seqLen, float *bias_matrx); + +private: + int maxLen = 0; + int maxHeadNums = 0; + int *posMatrix; + float *slopeM; +}; diff --git a/src/layers/attention.h b/src/layers/attention.h new file mode 100644 index 0000000..74ca094 --- /dev/null +++ b/src/layers/attention.h @@ -0,0 +1,619 @@ +#pragma once +#include + +#include "bfloat16.h" +#include "debugger.h" +#include "decoder_util.h" +#include "float16.h" +#include "gemm_kernel_ext.h" +#include "kvcache_tensor.h" +#include "matmul_helper.h" +#include "transformer_ctx.h" +#include "transformer_util.h" + +// WeiT: weight data type +// QKPO_CLS: class for post operation of query/key, it is generally the rotary embedding +// NORM_CLS: class for layernorm or other norms +// INPUT_AS_RESID: input as residential or not, most models use input as residential, +// but there are exceptions like ChatGLM use values after layernorm as residential +template +class Attention { +public: + Attention(int layerId, DecoderContext *ctx) : layerId(layerId), qkpo(ctx->attHeadSize) { + // Group attention or multi-head attention (multi-head attn is a special case of group attn) + if (ctx->attHeadNum % ctx->kvHeadNum == 0) { + // We are responsible for the range [startQHead, endQHead) + auto range = getTaskRange(ctx->attHeadNum, ctx->numSplit, ctx->splitIdx); + this->startQHead = range.first; + this->endQHead = range.second; + + int expandFactor = ctx->attHeadNum / ctx->kvHeadNum; + this->startKVHead = startQHead / expandFactor; + this->endKVHead = (this->endQHead - 1) / expandFactor + 1; + } + + // Unexpected case + else { + printf("Not supported yet: QHeads=%d, KVHeads=%d\n", ctx->attHeadNum, ctx->kvHeadNum); + exit(-1); + } + } + + // The inerface is for PyTorch, thus the weights are already transposed + void setWeights(DecoderContext *ctx, const float *queryWeight, const float *queryBias, const float *keyWeight, + const float *keyBias, const float *valueWeight, const float *valueBias, const float *attnOutWeight, + const float *attnOutBias, const float *gamma1, const float *beta1, bool trans = true) { + int hiddenSize = ctx->hiddenSize; + int headSize = ctx->attHeadSize; + + // Merged weights, dimension is like: hiddenSize * (hiddenSize + 2 * kvHiddenSize) + // Vertically split the QKV weights + int qResponsibleCols = (this->endQHead - this->startQHead) * headSize; + int kvResponsibleCols = (this->endKVHead - this->startKVHead) * headSize; + int responsibleCols = qResponsibleCols + 2 * kvResponsibleCols; + qkvWeight.Resize(hiddenSize, responsibleCols); + + float *concatBuf = (float *)malloc(hiddenSize * responsibleCols * sizeof(float)); + if (trans) { + memcpy(concatBuf, queryWeight + this->startQHead * headSize * hiddenSize, + hiddenSize * qResponsibleCols * sizeof(float)); + memcpy(concatBuf + hiddenSize * qResponsibleCols, keyWeight + this->startKVHead * headSize * hiddenSize, + hiddenSize * kvResponsibleCols * sizeof(float)); + memcpy(concatBuf + hiddenSize * (qResponsibleCols + kvResponsibleCols), + valueWeight + this->startKVHead * headSize * hiddenSize, + hiddenSize * kvResponsibleCols * sizeof(float)); + } else { + int qkvStride = (ctx->attHeadNum + ctx->kvHeadNum + ctx->kvHeadNum) * ctx->attHeadSize; + for (int i = 0; i < hiddenSize; ++i) { + memcpy(concatBuf + i * responsibleCols, queryWeight + i * qkvStride + this->startQHead * headSize, + qResponsibleCols * sizeof(float)); + } + for (int i = 0; i < hiddenSize; ++i) { + memcpy(concatBuf + i * responsibleCols + qResponsibleCols, + keyWeight + i * qkvStride + this->startKVHead * headSize, kvResponsibleCols * sizeof(float)); + } + for (int i = 0; i < hiddenSize; ++i) { + memcpy(concatBuf + i * responsibleCols + qResponsibleCols + kvResponsibleCols, + valueWeight + i * qkvStride + this->startKVHead * headSize, kvResponsibleCols * sizeof(float)); + } + } + + hpj::Matrix convertedqkvWeight; + MMHelper::convertWeight( + trans, hiddenSize, responsibleCols, concatBuf, convertedqkvWeight, qkvWeightScale, qkvWeightZero); + MMHelper::packWeight(trans, convertedqkvWeight, qkvWeight); + + free(concatBuf); + +#ifdef DEBUG + dbg.debugPrint("attention qkv weight: [%d, %d] (%d)\n", convertedqkvWeight.Rows(), convertedqkvWeight.Cols(), + convertedqkvWeight.Stride()); + dbg.dumpMatrix(convertedqkvWeight); + dbg.debugPrint( + "attention qkv packed weight: [%d, %d] (%d)\n", qkvWeight.Rows(), qkvWeight.Cols(), qkvWeight.Stride()); + dbg.dumpMatrix(qkvWeight); +#endif + + // Merged bias + if (queryBias && keyBias && valueBias) { + //responsibleCols = hiddenSize / ctx->numSplit; + qkvBias.Resize(responsibleCols); + memcpy(qkvBias.Data(), queryBias + ctx->splitIdx * qResponsibleCols, sizeof(float) * qResponsibleCols); + memcpy(qkvBias.Data() + qResponsibleCols, keyBias + this->startKVHead * headSize, + sizeof(float) * kvResponsibleCols); + memcpy(qkvBias.Data() + qResponsibleCols + kvResponsibleCols, valueBias + this->startKVHead * headSize, + sizeof(float) * kvResponsibleCols); + } + + // Weights for attention output + // Horizontally split the weight, as the source (PyTorch weight) is transposed, thus looks like vertically + hpj::Matrix convertedWeight; + MMHelper::convertWeight(ctx, trans, hiddenSize, hiddenSize, attnOutWeight, false, convertedWeight, + attnOutputWeightScale, attnOutputWeightZero); + MMHelper::packWeight(trans, convertedWeight, attnOutputWeight); + +#ifdef DEBUG + dbg.debugPrint("attention output weight: [%d, %d] (%d)\n", convertedWeight.Rows(), convertedWeight.Cols(), + convertedWeight.Stride()); + dbg.dumpMatrix(convertedWeight); + dbg.debugPrint("attention output packed weight: [%d, %d] (%d)\n", attnOutputWeight.Rows(), + attnOutputWeight.Cols(), attnOutputWeight.Stride()); + dbg.dumpMatrix(attnOutputWeight); +#endif + + // Attention output bias + if (attnOutBias) { + this->attnOutputBias.Resize(hiddenSize); + if (ctx->splitIdx == 0) { + memcpy(this->attnOutputBias.Data(), attnOutBias, sizeof(float) * hiddenSize); + } else { // For other splits, set bias to 0, to avoid duplicated calculation + memset(this->attnOutputBias.Data(), 0, sizeof(float) * hiddenSize); + } + } + + // LayerNorm + this->norm.setWeight(gamma1, beta1, hiddenSize); + } + +#ifdef DEBUG + void setDebugger(const Debugger &debugger) { this->dbg = debugger; } +#endif + + /** + * Forward computing for the whole encoder/decoder layer + * Inputs: + * - input: (bs * seq_len) x hidden_size (input buffer) + * - imBuf: (bs * seq_len) x hidden_size (intermediate buffer, output is in ctx->tmpBuf) + * - attnMask: (bs, 1, tgt_len, src_len) (tgt_len is the length of query, src_len is the length of key) + * - presentKeys, presentValues: past key/values concats current key/values + * - pastSeqLen: the sequence length in pastKeys and pastValues + * - useSelfAttn: use self attention or not, self attention is used to gen first token + * - doLnBefore: Do layer norm before or not. If true, will do layer norm as the first step + * - returnAttn: return attention values or not (this option is not used any more) + * - returnKVs: return present key/values or not (this option is not used any more) + * - forPT: is it for PyTorch or not (this option is not used any more), now cached keys/values are always controlled by us + * Internal Buffers: + * _________ _________ _________ _________ _________ + * | |------------->| |------------->| |------------->| |------------->| | + * ``````````` layerNorm ``````````` QKV Linear ``````````` MHA ``````````` out Linear ``````````` + * input resultBuffer1 qkvMatMul resultBuffer1 resultBuffer2 + */ + template + void forward(DecoderContext *ctx, float *input, float *imBuf, const float *attnMask, + KVCacheTensor &presentKey, KVCacheTensor &presentValue, int inputSeqLen, int pastSeqLen, + bool useSelfAttn, bool doLnBefore, bool returnAttn, bool returnKVs, bool forPT = true, + int *positionIds = nullptr) { + if (forPT) { + printf("For better perf, need to manage cached key/vaues by ourself, PyTorch extension is not supported " + "any more.\n"); + exit(-1); + } + + hpj::Matrix inputBuffer(input, ctx->batchSize * inputSeqLen, ctx->hiddenSize, ctx->hiddenSize); + hpj::Matrix imBuffer(imBuf, ctx->batchSize * inputSeqLen, ctx->hiddenSize, ctx->hiddenSize); + + auto hiddenSize = ctx->hiddenSize; + auto &qkvMatMul = ctx->qkvMatMul; + auto &resultBuffer1 = imBuffer; + auto &resultBuffer2 = ctx->tmpBuf; + +#ifdef DEBUG + dbg.debugPrint("---- DecoderLayer.forward (useSelfAttn=%d) ----\n", useSelfAttn); + dbg.debugPrint("input:\n"); + dbg.dumpMatrix(inputBuffer); +#endif + + if (doLnBefore) { + TimeLine t1("input.layer_norm"); + norm.forward(inputBuffer.Data(), resultBuffer1.Data(), inputBuffer.Rows(), inputBuffer.Stride(), + resultBuffer1.Stride()); + } +#ifdef DEBUG + dbg.debugPrint("layer norm:\n"); + dbg.dumpMatrix(resultBuffer1); + dbg.debugPrint("qkvWeight [%d, %d]:\n", this->qkvWeight.Rows(), this->qkvWeight.Cols()); + dbg.dumpMatrix(this->qkvWeight); +#endif + + // Query, Key, Value computed together + TimeLine t2("QKV.linear"); + DecoderUtil::dense(resultBuffer1, qkvWeight, qkvWeightScale, qkvWeightZero, qkvBias, qkvMatMul); + t2.release(); + + int cols = hiddenSize / ctx->numSplit; + hpj::Matrix query(qkvMatMul, 0, inputBuffer.Rows(), 0, cols); + hpj::Matrix key(qkvMatMul, 0, inputBuffer.Rows(), cols, cols); + hpj::Matrix value(qkvMatMul, 0, inputBuffer.Rows(), cols * 2, cols); + +#ifdef DEBUG + dbg.debugPrint("Q:\n"); + dbg.dumpMatrix(query); + dbg.debugPrint("K:\n"); + dbg.dumpMatrix(key); +#endif + + // Apply post operattions on query and key + TimeLine t3("QKPO"); + int qk_shape[4] = {ctx->batchSize, ctx->inputSeqLen, ctx->attHeadNum / ctx->numSplit, ctx->attHeadSize}; + if (positionIds != nullptr) { + qkpo.forward(query.Data(), key.Data(), query.Stride(), key.Stride(), qk_shape, positionIds); + } else { + // Use the default position ids + std::vector position_ids(ctx->inputSeqLen); + if (inputSeqLen == 1) { + position_ids[0] = pastSeqLen; + } else { + std::iota(position_ids.begin(), position_ids.end(), 0); + } + qkpo.forward(query.Data(), key.Data(), query.Stride(), key.Stride(), qk_shape, position_ids.data()); + } + t3.release(); + +#ifdef DEBUG + dbg.debugPrint("Q after post op:\n"); + dbg.dumpMatrix(query); + dbg.debugPrint("K after post op:\n"); + dbg.dumpMatrix(key); +#endif + + // Revise attnFactor before softmax (for some models, attnFactor may be not the default value) + // We initially introduced the code for ChatGLM, but eventually found it has no difference and was unnecessary. + // However, we have chosen to keep it in the codebase in case it becomes useful for future models. + if (getScalingCoeff() != 0) { ctx->attFactor = getScalingCoeff(); } + + TimeLine t4("MHA"); + if constexpr (!INPUT_AS_RESID) { + auto presult = resultBuffer1.Data(); + int rows = resultBuffer1.Rows(), cols = resultBuffer1.Cols(), stride = resultBuffer1.Stride(); + resultBuffer1.Assign(inputBuffer.Data(), inputBuffer.Rows(), inputBuffer.Cols(), inputBuffer.Stride()); + inputBuffer.Assign(presult, rows, cols, stride); + } + if (ctx->inputSeqLen > 256 && pastSeqLen == 0) + flashAttention(ctx, qkvMatMul, resultBuffer2, resultBuffer1, presentKey, presentValue, attnMask, pastSeqLen); + else + fusedAttention(ctx, query, key, value, resultBuffer1, presentKey, presentValue, attnMask, pastSeqLen); + t4.release(); + + // For multiple nodes inference, not the whole result buffer + hpj::Matrix attnSplit(resultBuffer1.Data(), resultBuffer1.Rows(), resultBuffer1.Cols() / ctx->numSplit, + resultBuffer1.Stride()); + +#ifdef DEBUG + dbg.debugPrint("attention_%d (softmax * value): [%d, %d] (%d)\n", ctx->splitIdx, attnSplit.Rows(), + attnSplit.Cols(), attnSplit.Stride()); + dbg.dumpMatrix(attnSplit); +#endif + + TimeLine t5("Output"); + // Output/projection in attention, only add the input in the first split + if (ctx->splitIdx == 0) { + float gamma = getResidentialScale(); + + // denseWithScaledSum should be enough, but as the performance of denseWithScaledSum is not verified, + // So here still use denseWithSum + if (gamma == 1) { + DecoderUtil::denseWithSum(attnSplit, attnOutputWeight, attnOutputWeightScale, attnOutputWeightZero, + attnOutputBias, inputBuffer, resultBuffer2); + } else { + DecoderUtil::denseWithScaledSum(attnSplit, attnOutputWeight, attnOutputWeightScale, + attnOutputWeightZero, attnOutputBias, gamma, inputBuffer, resultBuffer2); + } + } else { + DecoderUtil::dense(attnSplit, attnOutputWeight, attnOutputWeightScale, attnOutputWeightZero, attnOutputBias, + resultBuffer2); + } + t5.release(); + +#ifdef DEBUG + dbg.debugPrint("attention output/projection:\n"); + dbg.dumpMatrix(resultBuffer2); +#endif + + if (!doLnBefore) { + TimeLine t6("result.layer_norm"); + norm.forward(resultBuffer2.Data(), resultBuffer2.Data(), resultBuffer2.Rows(), resultBuffer2.Stride(), + resultBuffer2.Stride()); +#ifdef DEBUG + dbg.debugPrint("LayerNorm after attention: [%d, %d] (%d)\n", resultBuffer2.Rows(), resultBuffer2.Cols(), + resultBuffer2.Stride()); + dbg.dumpMatrix(resultBuffer2); +#endif + } + } + +protected: + template + void fusedAttention(DecoderContext *ctx, hpj::Matrix &query, hpj::Matrix &key, + hpj::Matrix &value, hpj::Matrix &result, KVCacheTensor &presentKey, + KVCacheTensor &presentValue, const float *attnMask, int pastSeqLen) { + // How many heads this task should do + int responsibleHeads = this->endQHead - this->startQHead; + int batchSize = ctx->batchSize; + int groupNum = ctx->attHeadNum / ctx->kvHeadNum; + + // If M_dimension/input_seq_len is big (1K, 2K, etc), need to split it to make sure the intermediate result in cache + // to make sure it works better (the logic here is trying to make sure each head of BMM result [seq * seq] in cache) + // WARN: reserve field in context is used to make it effective for all layers, do not change it in other places + int &mBlockSize = ctx->reserved1; + if (layerId == 0) { + if (pastSeqLen == 0) { + const int l2CacheSize = 2 * 1024 * 1024; // TODO: get it dynamically + const int sizeA = ctx->inputSeqLen * ctx->attHeadSize; + const int sizeB = ctx->inputSeqLen * ctx->attHeadSize; + const int sizeC = ctx->inputSeqLen * ctx->inputSeqLen; + + // As we do the split along M dimension, so to make sure: + // (sizeA / splits) + sizeB + (sizeC / splits) <= cacheSize + int splits = std::ceil(1.0f * (sizeA + sizeC) / (l2CacheSize / sizeof(float) - sizeB)); + mBlockSize = (ctx->inputSeqLen + splits - 1) / splits; + if (mBlockSize <= 0) { + mBlockSize = ctx->inputSeqLen > 6 ? 6 : ctx->inputSeqLen; + } else if (mBlockSize > ctx->inputSeqLen) { + mBlockSize = ctx->inputSeqLen; + } + } else { + mBlockSize = ctx->inputSeqLen; + } + } + + // How many blocks in M dimension + int mBlockNum = (ctx->inputSeqLen + mBlockSize - 1) / mBlockSize; + + // To get score buffer according to openmp thread ID or not (see below) + int scoreBufSize = batchSize * (ctx->attHeadNum / ctx->numSplit) * ctx->inputSeqLen * ctx->inputSeqLen; + bool scoreBufByThread = (ctx->numThreads * mBlockSize * (pastSeqLen + ctx->inputSeqLen) <= scoreBufSize); + + // For group attention, as #kvHeads != #qHeads, need to copy current key/values to cache seperately + // When M dimension is split, also multiple tasks per copy, so do copy seperately + bool kvCopied = false; + if (ctx->kvHeadNum < ctx->attHeadNum || mBlockSize != ctx->inputSeqLen) { +#pragma omp parallel for collapse(3) + for (int b = 0; b < batchSize; ++b) { + for (int i = 0; i < (this->endKVHead - this->startKVHead); ++i) { + // Copy current key/value to cached keys/values + // Re-layout is needed: (bs, seq=1, hidden_size) -> (seq=1, bs, hidden_size) + // Be noted: for group attention, the key/value is less than query + for (int seq = 0; seq < ctx->inputSeqLen; ++seq) { + auto srcK = key.Row(b * ctx->inputSeqLen + seq) + i * ctx->attHeadSize; + auto dstK = presentKey.getSequence(pastSeqLen + seq, b, i); + + auto srcV = value.Row(b * ctx->inputSeqLen + seq) + i * ctx->attHeadSize; + auto dstV = presentValue.getSequence(pastSeqLen + seq, b, i); + + if constexpr (std::is_same_v) { + memcpy(dstK, srcK, ctx->attHeadSize * sizeof(float)); + memcpy(dstV, srcV, ctx->attHeadSize * sizeof(float)); + } else if constexpr (std::is_same_v) { + float16_t::cvt_float_to_float16(srcK, dstK, ctx->attHeadSize); + float16_t::cvt_float_to_float16(srcV, dstV, ctx->attHeadSize); + } + } + } + } + kvCopied = true; + } + +#pragma omp parallel for collapse(3) + for (int b = 0; b < batchSize; ++b) { + for (int i = 0; i < responsibleHeads; ++i) { + for (int mb = 0; mb < mBlockNum; ++mb) { + const int startSeq = mb * mBlockSize; + const int endSeq + = startSeq + mBlockSize < ctx->inputSeqLen ? startSeq + mBlockSize : ctx->inputSeqLen; + + // Copy current key to cached keys + // Re-layout is needed: (bs, seq=1, hidden_size) -> (seq=1, bs, hidden_size) + if (!kvCopied) { + for (int seq = 0; seq < ctx->inputSeqLen; ++seq) { + auto src = key.Row(b * ctx->inputSeqLen + seq) + i * ctx->attHeadSize; + auto dst = presentKey.getSequence(pastSeqLen + seq, b, i); + if constexpr (std::is_same_v) { + memcpy(dst, src, ctx->attHeadSize * sizeof(float)); + } else if constexpr (std::is_same_v) { + float16_t::cvt_float_to_float16(src, dst, ctx->attHeadSize); + } + } + } + + // Q * K + auto keyMatInfo = presentKey.getHead(b, i / groupNum); + int m = endSeq - startSeq; + int k = ctx->attHeadSize; + int n = pastSeqLen + ctx->inputSeqLen; + int lda = query.Stride(); + int ldb = keyMatInfo.second; + int strideC = pastSeqLen > 0 ? (pastSeqLen + ctx->inputSeqLen + 15) / 16 * 16 : ctx->inputSeqLen; + int ldc = strideC; + auto A = query.Row(b * ctx->inputSeqLen + startSeq) + i * ctx->attHeadSize; // updated + auto B = keyMatInfo.first; + // Some special case, maximum required buffer size: max_thread_num * (mBlockSize * strideC) + // = bs * heads * mBlockNum * (mBlockSize * strideC) + // = bs * heads * (~seqLen) * seqLen, may be bigger than bs * heads * seqLen * seqLen + auto C = scoreBufByThread ? ctx->qkScores + omp_get_thread_num() * mBlockSize * strideC + : ctx->qkScores + (b * responsibleHeads + i) * ctx->inputSeqLen * strideC + + startSeq * strideC; + + const int queryLen = ctx->inputSeqLen; + const int keyLen = pastSeqLen + ctx->inputSeqLen; + + small_gemm_transb( + attnMask + b * queryLen * keyLen + startSeq * keyLen, A, B, C, m, n, k, lda, ldb, ldc); + +#ifdef DEBUG + if (b == 0 && i == 0) { + dbg.debugPrint("Q * K, first head:\n"); + auto p = ctx->qkScores; + dbg.debugPrint("%f, %f, %f ... %f %f %f\n", p[0] * ctx->attFactor, p[1] * ctx->attFactor, + p[2] * ctx->attFactor, p[keyLen - 3] * ctx->attFactor, p[keyLen - 2] * ctx->attFactor, + p[keyLen - 1] * ctx->attFactor); + } +#endif + + // Softmax(Q * K) + if (pastSeqLen == 0) + for (int seq = 0; seq < endSeq - startSeq; ++seq) { + DecoderUtil::softmaxSkipMask(ctx, C + seq * strideC, + attnMask + b * queryLen * keyLen + (seq + startSeq) * keyLen, keyLen); + } + else + for (int seq = 0; seq < endSeq - startSeq; ++seq) { + DecoderUtil::computeSoftmax(ctx, C + seq * strideC, + attnMask + b * queryLen * keyLen + (seq + startSeq) * keyLen, keyLen); + } + +#ifdef DEBUG + if (b == 0 && i == 0) { + dbg.debugPrint("Softmax(Q * K), first head:\n"); + auto p = ctx->qkScores; + dbg.debugPrint("%f, %f, %f ... %f %f %f\n", p[0], p[1], p[2], p[keyLen - 3], p[keyLen - 2], + p[keyLen - 1]); + } +#endif + + // Copy current value to cached values + // Re-layout is needed: (bs, seq, hidden_size) -> (seq, bs, hidden_size) + if (!kvCopied) { + for (int seq = 0; seq < ctx->inputSeqLen; ++seq) { + auto src = value.Row(b * ctx->inputSeqLen + seq) + i * ctx->attHeadSize; + auto dst = presentValue.getSequence(pastSeqLen + seq, b, i); + if constexpr (std::is_same_v) { + memcpy(dst, src, ctx->attHeadSize * sizeof(float)); + } else if constexpr (std::is_same_v) { + float16_t::cvt_float_to_float16(src, dst, ctx->attHeadSize); + } + } + } + + // Softmax * V + auto valueMatInfo = presentValue.getHead(b, i / groupNum); + std::swap(k, n); + lda = strideC; + ldb = valueMatInfo.second; + ldc = result.Stride(); + A = C; + B = valueMatInfo.first; + C = result.Row(b * ctx->inputSeqLen + startSeq) + i * ctx->attHeadSize; + + if constexpr (std::is_same_v) { + ig_sgemm_single_thread(false, false, m, n, k, 1.0f, A, lda, B, ldb, 0.0f, C, ldc); + } else if constexpr (std::is_same_v) { + ig_sgemm_f32f16f32_single_thread(false, false, m, n, k, 1.0f, A, lda, B, ldb, 0.0f, C, ldc); + } + +#ifdef DEBUG + if (b == 0 && i == 0) { + dbg.debugPrint("Softmax(Q * K) * V, first head:\n"); + auto p = C; + dbg.debugPrint("%f, %f, %f ... %f %f %f\n", p[0], p[1], p[2], p[ctx->attHeadSize - 3], + p[ctx->attHeadSize - 2], p[ctx->attHeadSize - 1]); + } +#endif + } // end for mb + } // end for i + } // end for b + } + + template + void flashAttention(DecoderContext *ctx, hpj::Matrix &qkvMatMul, hpj::Matrix &tmpRes, + hpj::Matrix &result, KVCacheTensor &presentKey, KVCacheTensor &presentValue, + const float *attnMask, int pastSeqLen) { + + // How many heads this task should do + int batchSize = ctx->batchSize; + int respQHeads = this->endQHead - this->startQHead; + int respKVHeads = this->endKVHead - this->startKVHead; + int qkvCols = respQHeads + respKVHeads * 2; + int headSize = ctx->attHeadSize; + float scale = ctx->attFactor; + int srcLen = ctx->inputSeqLen; + int tgtLen = pastSeqLen + srcLen; + + float *transQKV = (float*)malloc(sizeof(float) * batchSize * qkvCols * srcLen * headSize); + + DecoderUtil::transposeQKV(qkvMatMul.Data(), transQKV, batchSize, srcLen, respQHeads, respKVHeads, headSize); + + float *query = transQKV; + float *key = transQKV + batchSize * respQHeads * srcLen * headSize; + float *value = transQKV + batchSize * (respQHeads + respKVHeads) * srcLen * headSize; + + DecoderUtil::scaledDpAttention(query, key, value, attnMask, scale, batchSize, srcLen, tgtLen, respQHeads, + respKVHeads, headSize, tmpRes.Data()); + DecoderUtil::transposeAttnResult(tmpRes.Data(), result.Data(), batchSize, srcLen, respQHeads, headSize, + result.Stride()); + + // For group attention, as #kvHeads != #qHeads, need to copy current key/values to cache seperately + // When M dimension is split, also multiple tasks per copy, so do copy seperately +#pragma omp parallel for collapse(3) + for (int b = 0; b < batchSize; ++b) { + for (int i = 0; i < (this->endKVHead - this->startKVHead); ++i) { + // Copy current key/value to cached keys/values + // Re-layout is needed: (bs, seq=1, hidden_size) -> (seq=1, bs, hidden_size) + // Be noted: for group attention, the key/value is less than query + for (int seq = 0; seq < tgtLen; ++seq) { + auto srcK = key + b * respKVHeads * tgtLen * headSize + i * tgtLen * headSize + seq * headSize; + auto dstK = presentKey.getSequence(pastSeqLen + seq, b, i); + + auto srcV = value + b * respKVHeads * tgtLen * headSize + i * tgtLen * headSize + seq * headSize; + auto dstV = presentValue.getSequence(pastSeqLen + seq, b, i); + + if constexpr (std::is_same_v) { + memcpy(dstK, srcK, headSize * sizeof(float)); + memcpy(dstV, srcV, headSize * sizeof(float)); + } else if constexpr (std::is_same_v) { + float16_t::cvt_float_to_float16(srcK, dstK, headSize); + float16_t::cvt_float_to_float16(srcV, dstV, headSize); + } + } + } + } + free(transQKV); + } + +private: + std::pair getTaskRange(int N, int splits, int splitIdx) { + int startId, endId; + + if (N % splits == 0) { + int tasksPerSplit = N / splits; + startId = splitIdx * tasksPerSplit; + endId = startId + tasksPerSplit; + } else { + int baseTasksPerSplit = N / splits; + int remainingTasks = N % splits; + + // Each split has (baseTasksPerSplit + 1) tasks + if (splitIdx < remainingTasks) { + int tasksPerSplit = baseTasksPerSplit + 1; + startId = splitIdx * tasksPerSplit; + endId = startId + tasksPerSplit; + } + // Each split has 'baseTasksPerSplit' tasks + else { + int taskOffset = (baseTasksPerSplit + 1) * remainingTasks; + startId = taskOffset + (splitIdx - remainingTasks) * baseTasksPerSplit; + endId = startId + baseTasksPerSplit; + } + } + + return std::make_pair(startId, endId); + } + +protected: + virtual float getResidentialScale() { + return 1; // directly add the residential + } + + // Used in computeSoftmax + virtual float getScalingCoeff() { + return 0; // 0 means using the default value + } + + // query, key, value weighs + hpj::Matrix qkvWeight; + hpj::Vector qkvWeightScale; // if weighs is int8 + hpj::Vector qkvWeightZero; // if weighs is int8 + // query, key, value bias + hpj::Vector qkvBias; + + hpj::Matrix attnOutputWeight; + hpj::Vector attnOutputWeightScale; // if weighs is int8 + hpj::Vector attnOutputWeightZero; // if weighs is int8 + hpj::Vector attnOutputBias; + + // Query/Key post op + QKPO_CLS qkpo; + + // layerNorm param + NORM_CLS norm; + int layerId; + + // The responsible head in the global view + // If in single instance, startQHead=startKVHead=0, and endQHead-startQHead=qHeadNum + int startQHead; + int endQHead; + int startKVHead; + int endKVHead; +#ifdef DEBUG + Debugger dbg; +#endif +}; diff --git a/src/layers/attn_baichuan.h b/src/layers/attn_baichuan.h new file mode 100644 index 0000000..1a8748d --- /dev/null +++ b/src/layers/attn_baichuan.h @@ -0,0 +1,67 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once +#include +#include +#include + +#include "common_decoder.h" +#include "layers_norm.h" +#include "attention.h" + +template +class BaichuanAttention : public Attention { +public: + BaichuanAttention(int layerId, DecoderContext *ctx) : Attention(layerId, ctx) { + alibiSlopes = nullptr; + } + + virtual ~BaichuanAttention() { + if (alibiSlopes != nullptr) + delete[] alibiSlopes; + } + + const float* getAlibiSlopes(int attHeadNum) { + if (alibiSlopes == nullptr) { + int responsibleHeads = this->endQHead - this->startQHead; + alibiSlopes = new float[responsibleHeads]; + // alibi mask element + float ratio = std::pow(2, 8); + int closestPowerOf2 = std::pow(2, int(std::log2(attHeadNum))); + float x0 = std::pow(ratio, 1.0 / closestPowerOf2); + float x1 = std::pow(ratio, 1.0 / (closestPowerOf2 * 2)); + for (int i = 0, h = this->startQHead; i < responsibleHeads; ++i, ++h) { + if (h < closestPowerOf2) + alibiSlopes[i] = 1 / std::pow(x0, h + 1); + else + alibiSlopes[i] = 1 / std::pow(x1, 2 * (h - closestPowerOf2) + 1); + } + } + return alibiSlopes; + } + + const int getResponsibleHeads() { + return this->endQHead - this->startQHead; + } + +protected: + + const float* getMask(const float* attnMask, int bId, int hId, int srcLen, int tgtLen) override { + return attnMask + hId * srcLen * tgtLen; + } + +private: + float* alibiSlopes; +}; diff --git a/src/layers/attn_chatglm.h b/src/layers/attn_chatglm.h new file mode 100644 index 0000000..7e59845 --- /dev/null +++ b/src/layers/attn_chatglm.h @@ -0,0 +1,28 @@ +#pragma once +#include + +#include "attention.h" + +template +class ChatGlmAttention : public Attention { +public: + ChatGlmAttention(int layerId, DecoderContext *ctx) : Attention(layerId, ctx) { + residScale = std::sqrt(2 * ctx->layers); + scalingCoeff = 1.0f / (std::sqrt(ctx->attHeadSize) * (layerId + 1)); + } + +protected: + float getResidentialScale() override { return residScale; } + + // Do NOT needed, in ChatGLM, query_layer is divided by 'query_key_layer_scaling_coeff', + // but 'attention_scores' is multiplied by the same factor before softmax + // float getScalingCoeff() override { + // return scalingCoeff; + // } + +private: + // Residential scale + float residScale; + // query_key_layer_scaling_coeff + float scalingCoeff; +}; \ No newline at end of file diff --git a/src/layers/attn_chatglm2.h b/src/layers/attn_chatglm2.h new file mode 100644 index 0000000..ad7b541 --- /dev/null +++ b/src/layers/attn_chatglm2.h @@ -0,0 +1,167 @@ +#pragma once +#include + +#include "attention.h" + +template +class ChatGLM2Attention : public Attention { +public: + ChatGLM2Attention(int layerId, DecoderContext *ctx) + : Attention(layerId, ctx) {} + virtual ~ChatGLM2Attention() { } + +public: + template + void forward(DecoderContext *ctx, float *input, float *output, const float *attnMask, KVCacheTensor &presentKey, + KVCacheTensor &presentValue, int inputSeqLen, int pastSeqLen, bool useSelfAttn, bool doLnBefore, + bool returnAttn, bool returnKVs, bool forPT = true, int *positionIds = nullptr) { + if (forPT) { + printf("For better perf, need to manage cached key/vaues by ourself, PyTorch extension is not supported " + "any more.\n"); + exit(-1); + } + + KVCacheT *presentKeys = presentKey.getData(); + KVCacheT *presentValues = presentValue.getData(); + + hpj::Matrix inputBuffer(input, ctx->batchSize * inputSeqLen, ctx->hiddenSize, ctx->hiddenSize); + hpj::Matrix outBuffer(output, ctx->batchSize * inputSeqLen, ctx->hiddenSize, ctx->hiddenSize); + + auto hiddenSize = ctx->hiddenSize; + auto &qkvMatMul = ctx->qkvMatMul; + auto &resultBuffer1 = (ctx->numSplit == 1 ? outBuffer : ctx->normBuf); + auto &resultBuffer2 = ctx->tmpBuf; + float epsilon = ctx->epsilon; + + // //init group_qkvBuffer + int attHeadSize = ctx->attHeadSize; + int qkvRows = ctx->batchSize * inputSeqLen; + // multi query attention + int q_cols = (this->endQHead - this->startQHead) * attHeadSize; + int kv_cols = (this->endKVHead - this->startKVHead) * attHeadSize; + int qkCols = q_cols + kv_cols; + int qkvCols = qkCols + kv_cols; + + int qkvStride = qkvCols; + hpj::Matrix qkvGroupMatMul(qkvMatMul.Data(), qkvRows, qkvCols, qkvStride); + +#ifdef DEBUG + this->dbg.debugPrint("---- GLM2 DecoderLayer.forward (useSelfAttn=%d) ----\n", useSelfAttn); + this->dbg.debugPrint("input [%d, %d, %d]:\n", ctx->batchSize * inputSeqLen, ctx->hiddenSize, ctx->hiddenSize); + this->dbg.dumpMatrix(inputBuffer); +#endif + + if (doLnBefore) { + TimeLine t1("input.layer_norm"); + this->norm.forward(inputBuffer.Data(), resultBuffer1.Data(), inputBuffer.Rows(), inputBuffer.Stride(), + resultBuffer1.Stride(), epsilon); + } +#ifdef DEBUG + this->dbg.debugPrint( + "layer norm [%d, %d, %d]:\n", ctx->batchSize * inputSeqLen, ctx->hiddenSize, ctx->hiddenSize); + this->dbg.dumpMatrix(resultBuffer1); +#endif + // Query, Key, Value computed together + TimeLine t2("QKV.linear"); + DecoderUtil::dense(resultBuffer1, this->qkvWeight, this->qkvWeightScale, this->qkvWeightZero, this->qkvBias, + qkvGroupMatMul); + t2.release(); + +#ifdef DEBUG + this->dbg.debugPrint("dense [%d, %d, %d]:\n", ctx->batchSize * inputSeqLen, ctx->hiddenSize, qkvCols); + this->dbg.dumpMatrix(qkvGroupMatMul); +#endif + + // Apply post operattions on query and key + TimeLine t3("QKPO"); + if (positionIds != nullptr) { + this->qkpo.forward(qkvGroupMatMul.Data(), qkvGroupMatMul.Stride(), ctx->batchSize, inputSeqLen, qkCols, + attHeadSize, positionIds); + } else { + std::vector position_ids(ctx->inputSeqLen); + if (inputSeqLen == 1) { + position_ids[0] = pastSeqLen; + } else { + std::iota(position_ids.begin(), position_ids.end(), 0); + } + this->qkpo.forward(qkvGroupMatMul.Data(), qkvGroupMatMul.Stride(), ctx->batchSize, inputSeqLen, qkCols, + attHeadSize, position_ids.data()); + } + t3.release(); + +#ifdef DEBUG + this->dbg.debugPrint("qkpo [%d, %d, %d]:\n", ctx->batchSize * inputSeqLen, ctx->hiddenSize, qkvCols); + this->dbg.dumpMatrix(qkvGroupMatMul); +#endif + + // this->expand_to_qkv(qkvMatMul, qkvGroupMatMul, qkvRows, q_cols, kv_cols, kvHeadNum); + // printf("q_cols=%d, kv_cols=%d, qk_cols=%d\n", q_cols, kv_cols, qkCols); + hpj::Matrix query(qkvGroupMatMul, 0, inputBuffer.Rows(), 0, q_cols); + hpj::Matrix key(qkvGroupMatMul, 0, inputBuffer.Rows(), q_cols, kv_cols); + hpj::Matrix value(qkvGroupMatMul, 0, inputBuffer.Rows(), qkCols, kv_cols); + +#ifdef DEBUG + this->dbg.debugPrint("Q [%d, %d]:\n", query.Rows(), query.Cols()); + this->dbg.dumpMatrix(query); + this->dbg.debugPrint("K [%d, %d]:\n", key.Rows(), key.Cols()); + this->dbg.dumpMatrix(key); + this->dbg.debugPrint("V [%d, %d]:\n", value.Rows(), value.Cols()); + this->dbg.dumpMatrix(value); +#endif + + if (this->getScalingCoeff() != 0) { ctx->attFactor = this->getScalingCoeff(); } + TimeLine t4("MHA"); + if constexpr (!INPUT_AS_RESID) { + auto presult = resultBuffer1.Data(); + int rows = resultBuffer1.Rows(), cols = resultBuffer1.Cols(), stride = resultBuffer1.Stride(); + resultBuffer1.Assign(inputBuffer.Data(), inputBuffer.Rows(), inputBuffer.Cols(), inputBuffer.Stride()); + inputBuffer.Assign(presult, rows, cols, stride); + } + if (ctx->inputSeqLen > 1024 && pastSeqLen == 0) + this->flashAttention( + ctx, qkvGroupMatMul, resultBuffer2, resultBuffer1, presentKey, presentValue, attnMask, pastSeqLen); + else + this->fusedAttention( + ctx, query, key, value, resultBuffer1, presentKey, presentValue, attnMask, pastSeqLen); + t4.release(); + hpj::Matrix attnSplit(resultBuffer1.Data(), resultBuffer1.Rows(), resultBuffer1.Cols() / ctx->numSplit, + resultBuffer1.Stride()); + +#ifdef DEBUG + this->dbg.debugPrint("attention_%d (softmax * value):\n", ctx->splitIdx); + this->dbg.dumpMatrix(attnSplit); +#endif + + TimeLine t5("Output"); + // Output/projection in attention, only add the input in the first split + if (ctx->splitIdx == 0) { + float gamma = this->getResidentialScale(); + + // denseWithScaledSum should be enough, but as the performance of denseWithScaledSum is not verified, + // So here still use denseWithSum + if (gamma == 1) { + DecoderUtil::denseWithSum(attnSplit, this->attnOutputWeight, this->attnOutputWeightScale, + this->attnOutputWeightZero, this->attnOutputBias, inputBuffer, resultBuffer2); + } else { + DecoderUtil::denseWithScaledSum(attnSplit, this->attnOutputWeight, this->attnOutputWeightScale, + this->attnOutputWeightZero, this->attnOutputBias, gamma, inputBuffer, resultBuffer2); + } + } else { + DecoderUtil::dense(attnSplit, this->attnOutputWeight, this->attnOutputWeightScale, + this->attnOutputWeightZero, this->attnOutputBias, resultBuffer2); + } + t5.release(); + +#ifdef DEBUG + this->dbg.debugPrint("attention output/projection: [%d, %d] (%d)\n", resultBuffer2.Rows(), resultBuffer2.Cols(), + resultBuffer2.Stride()); + this->dbg.dumpMatrix(resultBuffer2); +#endif + + if (!doLnBefore) { + TimeLine t6("result.layer_norm"); + this->norm.forward(resultBuffer2.Data(), resultBuffer1.Data(), resultBuffer2.Rows(), resultBuffer2.Stride(), + resultBuffer1.Stride()); + } + } +}; diff --git a/src/layers/decoder_layer.h b/src/layers/decoder_layer.h new file mode 100644 index 0000000..e006b29 --- /dev/null +++ b/src/layers/decoder_layer.h @@ -0,0 +1,124 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attention.h" +#include "debugger.h" +#include "kvcache_tensor.h" +#include "timeline.h" + +template +class Decoder { +public: + Decoder(DecoderContext *_ctx, int _layerIdx) + : layerIdx(_layerIdx) + , attn(_layerIdx, _ctx) + , mlp(_ctx) +#ifdef DEBUG + , dbg(Debugger::formatStr("%d_%d.csv", _layerIdx, _ctx->splitIdx)) +#endif + { +#ifdef DEBUG + attn.setDebugger(dbg); + mlp.setDebugger(dbg); +#endif + } + + virtual ~Decoder() {} + + int getLayerId() { return layerIdx; } + + void setWeights(DecoderContext *ctx, std::vector ¶ms, bool trans = true) { + const float *queryWeight = params[0]; + const float *queryBias = params[1]; + const float *keyWeight = params[2]; + const float *keyBias = params[3]; + const float *valueWeight = params[4]; + const float *valueBias = params[5]; + const float *attnOutWeight = params[6]; + const float *attnOutBias = params[7]; + const float *gamma1 = params[8]; + const float *beta1 = params[9]; + + attn.setWeights(ctx, queryWeight, queryBias, keyWeight, keyBias, valueWeight, valueBias, attnOutWeight, + attnOutBias, gamma1, beta1, trans); + + std::vector mlpParams(params.begin() + 10, params.end()); + mlp.setWeights(ctx, mlpParams, trans); + } + + template + void forwardAttention(DecoderContext *ctx, float *input, float *output, const float *attnMask, KVCacheTensor &presentKey, + KVCacheTensor &presentValue, int inputSeqLen, int pastSeqLen, bool useSelfAttn, bool doLnBefore, + bool returnAttn, bool returnKVs, bool forPT = true, int *positionIds = nullptr) { + TimeLine t("Decoder.forwardAttention"); + attn.forward(ctx, input, output, attnMask, presentKey, presentValue, inputSeqLen, pastSeqLen, useSelfAttn, + doLnBefore, returnAttn, returnKVs, forPT, positionIds); + } + + void forwardFFN(DecoderContext *ctx, float *input, float *output, int iStride, int oStride, bool doLnBefore = true) { + TimeLine t("Decoder.forwardFFN"); + mlp.forward(ctx, input, output, iStride, oStride, doLnBefore); + } + +private: + void copyWeights(hpj::Matrix &w, int start_col, int end_col, const float *data) { + hpj::Matrix subW(w, 0, w.Rows(), start_col, end_col - start_col); + copyWeights(subW, data); + } + + // Copy the transposed weight into the non-transposed matrix + void copyWeights(hpj::Matrix &w, const float *data) { + for (int j = 0; j < w.Cols(); ++j) { + for (int i = 0; i < w.Rows(); ++i) { + w(i, j) = *data++; + } + } + } + + void copyTransposed(hpj::Matrix &dst, hpj::Matrix &src) { + dst.Resize(src.Cols(), src.Rows()); + for (int i = 0; i < dst.Rows(); ++i) { + for (int j = 0; j < dst.Cols(); ++j) { + dst(i, j) = src(j, i); + } + } + } + + // Add bias to matrix + void biasAdd(hpj::Matrix &m, hpj::Vector &bias) { + float *pbias = bias.Data(); +#pragma omp parallel for + for (int i = 0; i < m.Rows(); ++i) { + float *p = m.Row(i); +#pragma omp simd + for (int j = 0; j < m.Cols(); ++j) { + p[j] += pbias[j]; + } + } + } + +private: + // For debug usage + int layerIdx; + + ATTN_CLS attn; + MLP_CLS mlp; + +#ifdef DEBUG + Debugger dbg; +#endif +}; diff --git a/src/layers/dist_linear.h b/src/layers/dist_linear.h new file mode 100644 index 0000000..9a0ad73 --- /dev/null +++ b/src/layers/dist_linear.h @@ -0,0 +1,98 @@ +#pragma once +#include "float16.h" +#include "matmul_helper.h" +#include "timeline.h" + +/** + * Distributed linear impl. by vertically spliting the weight + */ +template +class DistLinear { +public: + DistLinear(int inDim, int outDim, int splitIdx, int splits) { + this->inputSize = inDim; + this->outputSize = outDim; + this->splitIdx = splitIdx; + this->splits = splits; + + this->bias = nullptr; + } + + ~DistLinear() { + if (bias) free(bias); + } + + // Note: the weight passed in is transposed + // + // _______________inputSize(K)______________ + // | | + // | | splitSize(N) + // |_________________________________________| + // | | + // | | splitSize(N) + // |_________________________________________| + void setWeight(const float *w, const float *b) { + this->splitSize = outputSize / splits; + this->splitOffset = this->splitSize * splitIdx; + + if (splitIdx < outputSize % splits) { + this->splitSize += 1; + this->splitOffset += splitIdx; + } else { + this->splitOffset += outputSize % splits; + } + + int K = inputSize; + int N = this->splitSize; + weight.Resize(K, N); + scaleWeight.Resize(N); + zeroWeight.Resize(N); + + hpj::Matrix quantizedWeight; + MMHelper::convertWeight(true, K, N, w + splitOffset * K, quantizedWeight, scaleWeight, zeroWeight); + MMHelper::packWeight(true, quantizedWeight, weight); + + // Copy Bias + if (b) { + bias = (float *)aligned_alloc(64, N * sizeof(float)); + memcpy(bias, b + splitOffset, N * sizeof(float)); + } + } + + // input is in the shape of (batchSize, inputSize) + void forward(const float *input, float *output, int batchSize) { + TimeLine t("DistLinear.forward"); + if (bias) { + MMHelper::compute_bias(false, batchSize, splitSize, inputSize, 1.0f, input, inputSize, weight.Data(), + scaleWeight.Data(), zeroWeight.Data(), 0.0f, output, splitSize, bias); + + } else { + MMHelper::compute(false, batchSize, splitSize, inputSize, 1.0f, input, inputSize, weight.Data(), + scaleWeight.Data(), zeroWeight.Data(), 0.0f, output, splitSize); + } + } + + int getInputSize() { return inputSize; } + + int getOutputSize() { return outputSize; } + + int getSplitSize() { return splitSize; } + + int getSplitOffset() { return splitOffset; } + +private: + int inputSize; + int outputSize; + + int splitIdx; + int splits; + + // = outputSize/splits, but need to consider the case of not divisible + int splitSize; + int splitOffset; + + hpj::Matrix weight; + hpj::Vector scaleWeight; // if weighs is int8 + hpj::Vector zeroWeight; // if weighs is int8 + float *bias; +}; \ No newline at end of file diff --git a/src/layers/layer_norm.cpp b/src/layers/layer_norm.cpp new file mode 100644 index 0000000..26551c3 --- /dev/null +++ b/src/layers/layer_norm.cpp @@ -0,0 +1,52 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#include + +#include +#include + +#include "layernorm_kernels.h" +#include "layers_norm.h" +#include "timeline.h" + +namespace xft { + +// Layer normalization: only support the norm along last dimension +LayerNorm::LayerNorm() { + weights = nullptr; + normSize = 0; +} + +LayerNorm::~LayerNorm() { + if (weights) { free(weights); } +} + +void LayerNorm::setWeight(const float *gamma, const float *beta, int size) { + this->normSize = size; + this->weights = (float *)aligned_alloc(64, 2 * size * sizeof(float)); + memcpy(weights, gamma, size * sizeof(float)); + memcpy(weights + size, beta, size * sizeof(float)); +} + +// input and output are in shape of (rows, normSize) +// TODO: column-wise parallel +void LayerNorm::forward(const float *input, float *output, int rows, int iStride, int oStride) { + TimeLine t("LayerNorm.forward"); + const float *pgamma = weights; + const float *pbeta = weights + normSize; + invokeLayerNorm(output, input, pgamma, pbeta, rows, normSize, iStride, oStride); +} + +} // namespace xft \ No newline at end of file diff --git a/src/layers/layer_norm.h b/src/layers/layer_norm.h new file mode 100644 index 0000000..cdc7f9e --- /dev/null +++ b/src/layers/layer_norm.h @@ -0,0 +1,39 @@ +#pragma once +#include +#include + +#include "layernorm_kernels.h" + +// Layer normalization: only support the norm along last dimension +class LayerNorm { +public: + LayerNorm() { + weights = nullptr; + normSize = 0; + } + + ~LayerNorm() { + if (weights) { free(weights); } + } + + void setWeight(const float *gamma, const float *beta, int size) { + this->normSize = size; + this->weights = (float *)aligned_alloc(64, 2 * size * sizeof(float)); + memcpy(weights, gamma, size * sizeof(float)); + memcpy(weights + size, beta, size * sizeof(float)); + } + + // input and output are in shape of (rows, normSize) + // TODO: column-wise parallel + void forward(const float *input, float *output, int rows, int iStride = -1, int oStride = -1) { + const float *pgamma = weights; + const float *pbeta = weights + normSize; + xft::invokeLayerNorm(output, input, pgamma, pbeta, rows, normSize, iStride, oStride); + } + +private: + int normSize; + + // the weights contains gamma and beta concated together + float *weights; +}; \ No newline at end of file diff --git a/src/layers/mlp_chatglm.h b/src/layers/mlp_chatglm.h new file mode 100644 index 0000000..a5ffa1d --- /dev/null +++ b/src/layers/mlp_chatglm.h @@ -0,0 +1,17 @@ +#pragma once +#include + +#include "mlp_standard.h" + +template +class ChatGlmMLP : public MLP { +public: + ChatGlmMLP(DecoderContext *ctx) : MLP(ctx) { residScale = std::sqrt(2 * ctx->layers); } + +protected: + float getResidentialScale() override { return residScale; } + +private: + // Residential scale + float residScale; +}; \ No newline at end of file diff --git a/src/layers/mlp_chatglm2.h b/src/layers/mlp_chatglm2.h new file mode 100644 index 0000000..c027930 --- /dev/null +++ b/src/layers/mlp_chatglm2.h @@ -0,0 +1,86 @@ +#pragma once +#include +#include "mlp_llama.h" + +template +class ChatGLM2MLP : public LlamaMLP { +public: + ChatGLM2MLP(DecoderContext *ctx) : LlamaMLP(ctx) { } + + // The inerface is for PyTorch, thus the weights are already transposed + void setWeights(DecoderContext *ctx, std::vector ¶ms, bool trans = true) { + int hiddenSize = ctx->hiddenSize; + int intermediateSize = ctx->intermediateSize; + + const float *gate_upW = params[0]; + const float *downW = params[2]; + const float *normW = params[4]; + + REQUIRES(ctx->actType == DecoderContext::SWIGLU, "unsupported activation."); + + // Vertically split the gate weight and up weight + hpj::Matrix convertedGateWeight, convertedUpWeight, convertedDownWeight; + + int colSplit = intermediateSize / ctx->numSplit; + float *gateW = (float *)malloc(hiddenSize * colSplit * sizeof(float)); + float *upW = (float *)malloc(hiddenSize * colSplit * sizeof(float)); + if (trans) { + int blockSize = colSplit * hiddenSize; + memcpy(gateW, gate_upW + ctx->splitIdx * blockSize, blockSize * sizeof(float)); + memcpy(upW, gate_upW + intermediateSize * hiddenSize + ctx->splitIdx * blockSize, + blockSize * sizeof(float)); + } else { + const float *weightPTR = gate_upW; + for (int i = 0; i < hiddenSize; i++) { + memcpy(gateW + i * colSplit, weightPTR + ctx->splitIdx * colSplit, colSplit * sizeof(float)); + weightPTR += intermediateSize; + memcpy(upW + i * colSplit, weightPTR + ctx->splitIdx * colSplit, colSplit * sizeof(float)); + weightPTR += intermediateSize; + } + } + + MMHelper::convertWeight( + trans, hiddenSize, colSplit, gateW, convertedGateWeight, this->gateWeightScale, this->gateWeightZero); + MMHelper::packWeight(trans, convertedGateWeight, this->gateWeight); + + MMHelper::convertWeight(trans, hiddenSize, colSplit, upW, convertedUpWeight, this->upWeightScale, this->upWeightZero); + MMHelper::packWeight(trans, convertedUpWeight, this->upWeight); + + free(gateW); + free(upW); + + // Horizontally split the down weight + MMHelper::convertWeight(ctx, trans, intermediateSize, hiddenSize, downW, false, convertedDownWeight, + this->downWeightScale, this->downWeightZero); + MMHelper::packWeight(trans, convertedDownWeight, this->downWeight); +#ifdef DEBUG + this->dbg.debugPrint("convertedGateWeight [%d, %d](%d):\n", convertedGateWeight.Rows(), convertedGateWeight.Cols(), + convertedGateWeight.Stride()); + this->dbg.dumpMatrix(convertedGateWeight); + + this->dbg.debugPrint("packed convertedGateWeight [%d, %d](%d):\n", this->gateWeight.Rows(), this->gateWeight.Cols(), + this->gateWeight.Stride()); + this->dbg.dumpMatrix(this->gateWeight); + + this->dbg.debugPrint("convertedUpWeight [%d, %d](%d):\n", convertedUpWeight.Rows(), convertedUpWeight.Cols(), + convertedUpWeight.Stride()); + this->dbg.dumpMatrix(convertedUpWeight); + + this->dbg.debugPrint("packed convertedUpWeight [%d, %d](%d):\n", this->upWeight.Rows(), this->upWeight.Cols(), this->upWeight.Stride()); + this->dbg.dumpMatrix(this->upWeight); + + this->dbg.debugPrint("convertedDownWeight [%d, %d](%d):\n", convertedDownWeight.Rows(), convertedDownWeight.Cols(), + convertedDownWeight.Stride()); + this->dbg.dumpMatrix(convertedDownWeight); + + this->dbg.debugPrint("packed convertedDownWeight [%d, %d](%d):\n", this->downWeight.Rows(), this->downWeight.Cols(), + this->downWeight.Stride()); + this->dbg.dumpMatrix(this->downWeight); +#endif + // norm.setWeight(normW, NULL, hiddenSize); + if (normW) { + this->normWeight.Resize(hiddenSize); + memcpy(this->normWeight.Data(), normW, sizeof(float) * hiddenSize); + } + } +}; diff --git a/src/layers/mlp_llama.h b/src/layers/mlp_llama.h new file mode 100644 index 0000000..d151170 --- /dev/null +++ b/src/layers/mlp_llama.h @@ -0,0 +1,211 @@ +#pragma once +#include "bert_util.h" +#include "debugger.h" +#include "decoder_util.h" +#include "matmul_helper.h" +#include "timeline.h" + +// C++ implementation for the python code in modeling_llama.py: +// residual = hidden_states +// hidden_states = self.post_attention_layernorm(hidden_states) +// hidden_states = self.mlp(hidden_states) +// hidden_states = residual + hidden_states +// +// While LlamaMLP is like: +// def forward(self, x): +// return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) +// But please also be noted: we extended the MLP to include layer norm +template +class LlamaMLP { +public: + LlamaMLP(DecoderContext *ctx) { + if (ctx->intermediateSize % ctx->numSplit != 0) { + printf("Unsupported: split ffn (size=%d) into %d splits.\n", ctx->intermediateSize, ctx->numSplit); + exit(-1); + } + } + + // The inerface is for PyTorch, thus the weights are already transposed + void setWeights(DecoderContext *ctx, std::vector ¶ms, bool trans = true) { + int hiddenSize = ctx->hiddenSize; + int imSize = ctx->intermediateSize; + + // Refer to CommonDecoder for parameters order + const float *gateW = params[0]; + const float *upW = params[2]; + const float *normW = params[4]; + const float *downW = params[6]; + + REQUIRES(ctx->actType == DecoderContext::SILU, "unsupported activation."); + + // Vertically split the gate weight and up weight + hpj::Matrix quantizedGateWeight, quantizedUpWeight, quantizedDownWeight; + + MMHelper::convertWeight( + ctx, trans, hiddenSize, imSize, gateW, true, quantizedGateWeight, gateWeightScale, gateWeightZero); + MMHelper::packWeight(trans, quantizedGateWeight, gateWeight); + + MMHelper::convertWeight( + ctx, trans, hiddenSize, imSize, upW, true, quantizedUpWeight, upWeightScale, upWeightZero); + MMHelper::packWeight(trans, quantizedUpWeight, upWeight); + + // Horizontally split the down weight + MMHelper::convertWeight( + ctx, trans, imSize, hiddenSize, downW, false, quantizedDownWeight, downWeightScale, downWeightZero); + MMHelper::packWeight(trans, quantizedDownWeight, downWeight); + +#ifdef DEBUG + dbg.debugPrint("quantizedGateWeight:\n"); + dbg.dumpMatrix(quantizedGateWeight); + + dbg.debugPrint("quantizedUpWeight:\n"); + dbg.dumpMatrix(quantizedUpWeight); + + dbg.debugPrint("quantizedDownWeight:\n"); + dbg.dumpMatrix(quantizedDownWeight); +#endif + + // LlamaRMSNorm + if (normW) { + normWeight.Resize(hiddenSize); + memcpy(normWeight.Data(), normW, sizeof(float) * hiddenSize); + } + } + +#ifdef DEBUG + void setDebugger(const Debugger &debugger) { + this->dbg = debugger; + } +#endif + + // Forward for FFN (Feed Forward Network) + void forward(DecoderContext *ctx, float *input, float *output, int iStride, int oStride, + bool doLnBefore = true /*not used*/) { + TimeLine t("LlamaMLP"); + const int M = ctx->batchSize * ctx->inputSeqLen; + const int hiddenSize = ctx->hiddenSize; + + hpj::Matrix inBuffer(input, M, hiddenSize, iStride); + hpj::Matrix outBuffer(output, M, hiddenSize, oStride); + auto &normBuffer = ctx->normBuf; + auto &imBuffer = ctx->imOut; + + DecoderUtil::rmsNorm(inBuffer, normBuffer, normWeight, 1e-6); + +#ifdef DEBUG + dbg.debugPrint("LayerNorm before MLP:\n"); + dbg.dumpMatrix(normBuffer); +#endif + + gateProj(normBuffer, imBuffer); + +#ifdef DEBUG + dbg.debugPrint("gateWeight:\n"); + dbg.dumpMatrix(gateWeight); + dbg.debugPrint("gate output:\n"); + dbg.dumpMatrix(imBuffer); +#endif + + upProj(normBuffer, imBuffer); + +#ifdef DEBUG + dbg.debugPrint("upWeight:\n"); + dbg.dumpMatrix(upWeight); + dbg.debugPrint("up output:\n"); + dbg.dumpMatrix(imBuffer); +#endif + + downProj(imBuffer, outBuffer, inBuffer, ctx->splitIdx == 0); + +#ifdef DEBUG + dbg.debugPrint("downWeight:\n"); + dbg.dumpMatrix(downWeight); + dbg.debugPrint("residential:\n"); + dbg.dumpMatrix(inBuffer); + dbg.debugPrint("final output:\n"); + dbg.dumpMatrix(outBuffer); +#endif + } + +private: + void gateProj(hpj::Matrix &input, hpj::Matrix &output) { + TimeLine t("GateProj"); + + assert(input.Rows() == output.Rows()); + assert(input.Cols() == gateWeight.Rows()); + assert(gateWeight.Cols() == output.Cols()); + + int M = input.Rows(), N = output.Cols(), K = input.Cols(); + int lda = input.Stride(), ldc = output.Stride(); + + const float *A = input.Data(); + const WeiT *B = gateWeight.Data(); + const float *scaleB = gateWeightScale.Data(); + const float *zeroB = gateWeightZero.Data(); + float *C = output.Data(); + + MMHelper::compute_silu(false, M, N, K, 1.0f, A, lda, B, scaleB, zeroB, 0.0f, C, ldc); + } + + void upProj(hpj::Matrix &input, hpj::Matrix &output) { + TimeLine t("UpProj"); + + assert(input.Rows() == output.Rows()); + assert(input.Cols() == upWeight.Rows()); + assert(upWeight.Cols() == output.Cols()); + + int M = input.Rows(), N = output.Cols(), K = input.Cols(); + int lda = input.Stride(), ldc = output.Stride(); + + const float *A = input.Data(); + const WeiT *B = upWeight.Data(); + const float *scaleB = upWeightScale.Data(); + const float *zeroB = upWeightZero.Data(); + float *C = output.Data(); + + MMHelper::compute_resmul(false, M, N, K, 1.0f, A, lda, B, scaleB, zeroB, 0.0f, C, ldc, C, ldc); + } + + void downProj( + hpj::Matrix &input, hpj::Matrix &output, hpj::Matrix &residential, bool isMaster) { + TimeLine t("DownProj"); + + assert(input.Rows() == output.Rows()); + assert(input.Cols() == downWeight.Rows()); + assert(downWeight.Cols() == output.Cols()); + + int M = input.Rows(), N = output.Cols(), K = input.Cols(); + int lda = input.Stride(), ldc = output.Stride(), ldr = residential.Stride(); + + const float *A = input.Data(); + const WeiT *B = downWeight.Data(); + const float *scaleB = downWeightScale.Data(); + const float *zeroB = downWeightZero.Data(); + float *C = output.Data(); + const float *R = residential.Data(); + + if (isMaster) { + MMHelper::compute_residential(false, M, N, K, 1.0f, A, lda, B, scaleB, zeroB, 0.0f, C, ldc, NULL, R, ldr); + } else { + MMHelper::compute(false, M, N, K, 1.0f, A, lda, B, scaleB, zeroB, 0.0f, C, ldc); + } + } + +protected: + hpj::Matrix gateWeight; + hpj::Vector gateWeightScale; // For int8_t weight + hpj::Vector gateWeightZero; // For int8_t weight + hpj::Matrix upWeight; + hpj::Vector upWeightScale; // For int8_t weight + hpj::Vector upWeightZero; // For int8_t weight + hpj::Matrix downWeight; + hpj::Vector downWeightScale; // For int8_t weight + hpj::Vector downWeightZero; // For int8_t weight + + // LlamaRMSNorm param + hpj::Vector normWeight; + +#ifdef DEBUG + Debugger dbg; +#endif +}; diff --git a/src/layers/mlp_standard.h b/src/layers/mlp_standard.h new file mode 100644 index 0000000..db2fd8d --- /dev/null +++ b/src/layers/mlp_standard.h @@ -0,0 +1,217 @@ +#pragma once +#include "bert_util.h" +#include "debugger.h" +#include "decoder_util.h" +#include "matmul_helper.h" + +// WeiT: weight data type +// INPUT_AS_RESID: input as residential or not, most models use input as residential, +// but there are exceptions like ChatGLM use values after layernorm as residential +template +class MLP { +public: + MLP(DecoderContext *ctx) { + if (ctx->intermediateSize % ctx->numSplit != 0) { + printf("Unsupported: split ffn (size=%d) into %d splits.\n", ctx->intermediateSize, ctx->numSplit); + exit(-1); + } + } + + // The inerface is for PyTorch, thus the weights are already transposed + void setWeights(DecoderContext *ctx, std::vector ¶ms, bool trans = true) { + int hiddenSize = ctx->hiddenSize; + int intermediateSize = ctx->intermediateSize; + + const float *_imWeight = params[0]; + const float *_imBias = params[1]; + const float *_outputWeight = params[2]; + const float *_outputBias = params[3]; + const float *_gamma2 = params[4]; + const float *_beta2 = params[5]; + + // Vertically split intermediate(FC1) weight + hpj::Matrix quantizedIntermediateWeight; + MMHelper::convertWeight(ctx, trans, hiddenSize, intermediateSize, _imWeight, true, quantizedIntermediateWeight, + intermediateWeightScale, intermediateWeightZero); + MMHelper::packWeight(trans, quantizedIntermediateWeight, intermediateWeight); + + // Intermediate bias + int colsPerSplit = intermediateSize / ctx->numSplit; + intermediateBias.Resize(colsPerSplit); + memcpy(intermediateBias.Data(), _imBias + colsPerSplit * ctx->splitIdx, sizeof(float) * colsPerSplit); + + // Horizontally split the output(FC2) weight + hpj::Matrix quantizedOutputWeight; + MMHelper::convertWeight(ctx, trans, intermediateSize, hiddenSize, _outputWeight, false, quantizedOutputWeight, + outputWeightScale, outputWeightZero); + MMHelper::packWeight(trans, quantizedOutputWeight, outputWeight); + + // Output bias + outputBias.Resize(hiddenSize); + if (ctx->splitIdx == 0) { + memcpy(outputBias.Data(), _outputBias, sizeof(float) * hiddenSize); + } else { // For other splits, set bias to 0, to avoid duplicated calculation + memset(outputBias.Data(), 0, sizeof(float) * hiddenSize); + } + + // gamma and beta for layer norm + if (_gamma2 && _beta2) { + gamma2.Resize(hiddenSize); + beta2.Resize(hiddenSize); + memcpy(gamma2.Data(), _gamma2, sizeof(float) * hiddenSize); + memcpy(beta2.Data(), _beta2, sizeof(float) * hiddenSize); + } + } + +#ifdef DEBUG + void setDebugger(const Debugger &debugger) { + this->dbg = debugger; + } +#endif + + // Forward for FFN (Feed Forward Network) + void forward(DecoderContext *ctx, float *input, float *output, int iStride, int oStride, bool doLnBefore) { + TimeLine t("StandardMLP"); + int M = ctx->batchSize * ctx->inputSeqLen; + hpj::Matrix outBuffer(output, M, ctx->hiddenSize, ctx->hiddenSize); + + auto &resultBuffer1 = outBuffer; + auto &resultBuffer2 = ctx->tmpBuf; + auto &imBuffer = ctx->imOut; + + // When doLnBefore=true, conduct layernorm in the beginning + // Note: input is resultBuffer2 as of some history reason + if (doLnBefore) { + // Need to keep input unmodified + if constexpr (INPUT_AS_RESID) { + DecoderUtil::layerNorm(resultBuffer2, resultBuffer1, gamma2, beta2); + } else { + DecoderUtil::layerNorm(resultBuffer2, resultBuffer2, gamma2, beta2); + } + } + + auto &imInput = doLnBefore ? (INPUT_AS_RESID ? resultBuffer1 : resultBuffer2) : resultBuffer2; + +#ifdef DEBUG + dbg.debugPrint("layer norm after attention:\n"); + dbg.dumpMatrix(imInput); +#endif + + // intermediate + switch (ctx->actType) { + case DecoderContext::RELU: intermediate_relu(imInput, imBuffer); break; + case DecoderContext::GELU: intermediate_gelu(imInput, imBuffer); break; + } + +#ifdef DEBUG + dbg.debugPrint("intermediate:\n"); + dbg.dumpMatrix(imBuffer); +#endif + + // dense in output + if (ctx->splitIdx == 0) { + float gamma = getResidentialScale(); + + // denseWithScaledSum is enough, but as the perf of denseWithScaledSum is not verified, so denseWithSum is still here + if (gamma == 1) { + DecoderUtil::denseWithSum(imBuffer, outputWeight, outputWeightScale, outputWeightZero, outputBias, + resultBuffer2, resultBuffer1); + } else { + DecoderUtil::denseWithScaledSum(imBuffer, outputWeight, outputWeightScale, outputWeightZero, outputBias, + gamma, resultBuffer2, resultBuffer1); + } + } else { + DecoderUtil::dense(imBuffer, outputWeight, outputWeightScale, outputWeightZero, outputBias, resultBuffer1); + } + +#ifdef DEBUG + dbg.debugPrint("output:\n"); + dbg.dumpMatrix(resultBuffer1); +#endif + + // layerNorm + if (!doLnBefore) { DecoderUtil::layerNorm(resultBuffer1, resultBuffer1, gamma2, beta2); } + +#ifdef DEBUG + dbg.debugPrint("final output:\n"); + dbg.dumpMatrix(resultBuffer1); +#endif + } + +protected: + void intermediate_relu(hpj::Matrix &input, hpj::Matrix &output) { + MMHelper::compute_biasadd_relu(false, input.Rows(), output.Cols(), input.Cols(), 1.0f, input.Data(), + input.Stride(), intermediateWeight.Data(), intermediateWeightScale.Data(), + intermediateWeightZero.Data(), 0.0f, output.Data(), output.Stride(), intermediateBias.Data()); + } + + void intermediate_gelu(hpj::Matrix &input, hpj::Matrix &output) { + MMHelper::compute(false, input.Rows(), output.Cols(), input.Cols(), 1.0f, input.Data(), input.Stride(), + intermediateWeight.Data(), intermediateWeightScale.Data(), intermediateWeightZero.Data(), 0.0f, + output.Data(), output.Stride()); + + float *pbias = intermediateBias.Data(); + float factor = 0.7978845608; // np.sqrt(2 / np.pi) + +#pragma omp parallel for + for (int i = 0; i < output.Rows(); ++i) { + // int tid = omp_get_thread_num(); + // float *pout = output.Row(i); + // #pragma omp simd + // for (int j = 0; j < output.Cols(); ++j) { + // float x = pout[j] + pbias[j]; + // ctx->erf_buffer[tid][j] = x; + // pout[j] = factor * (x + 0.044715f * x * x * x); + // } + // vsTanh(output.Cols(), pout, pout); + // #pragma omp simd + // for (int j = 0; j < output.Cols(); ++j) { + // pout[j] = ctx->erf_buffer[tid][j] * 0.5f * (1 + pout[j]); + // } + float *pout = output.Row(i); + __m512 c1 = _mm512_set1_ps(0.044715f); + __m512 c2 = _mm512_set1_ps(factor); + __m512 vone = _mm512_set1_ps(1); + __m512 vtwo = _mm512_set1_ps(2); + __m512 vhalf = _mm512_set1_ps(0.5f); + + for (int off = 0; off < output.Cols(); off += 16) { + int remain = output.Cols() - off; + __mmask16 mask = (remain >= 16 ? 0xffff : (1 << remain) - 1); + + __m512 vx = _mm512_maskz_loadu_ps(mask, pout + off); + vx = vx + _mm512_maskz_loadu_ps(mask, pbias + off); + + __m512 vt = c2 * (vx + c1 * vx * vx * vx); + vt = BertUtil::vexp(vt * vtwo); + vt = vone - vtwo * _mm512_rcp14_ps(vt + vone); // tanh + __m512 vy = vx * (vone + vt) * vhalf; + + _mm512_mask_storeu_ps(pout + off, mask, vy); + } + } + } + + // protected: + virtual float getResidentialScale() { + return 1; // directly add the residential + } + + // private: + hpj::Matrix intermediateWeight; + hpj::Vector intermediateWeightScale; + hpj::Vector intermediateWeightZero; + hpj::Vector intermediateBias; + + hpj::Matrix outputWeight; + hpj::Vector outputWeightScale; + hpj::Vector outputWeightZero; + hpj::Vector outputBias; + + // layerNorm param + hpj::Vector gamma2, beta2; + +#ifdef DEBUG + Debugger dbg; +#endif +}; diff --git a/src/layers/opt_embedding.h b/src/layers/opt_embedding.h new file mode 100644 index 0000000..93946b7 --- /dev/null +++ b/src/layers/opt_embedding.h @@ -0,0 +1,89 @@ +#pragma once +#include "float16.h" +#include "transformer_ctx.h" + +template +class OptEmbedding { +public: + OptEmbedding(DecoderContext *ctx) { + this->vocabSize = ctx->vocabSize; + this->embeddingSize = ctx->embeddingSize; + this->maxPositions = ctx->maxPositions; + this->hiddenSize = ctx->hiddenSize; + } + + void setWeights(float *tokenEmb, float *positionEmb) { + int size1 = vocabSize * embeddingSize; + embTable = (T *)aligned_alloc(64, size1 * sizeof(T)); + + int size2 = maxPositions * hiddenSize; + positionalTable = (T *)aligned_alloc(64, size2 * sizeof(T)); + + if constexpr (std::is_same_v) { + memcpy(embTable, tokenEmb, size1 * sizeof(T)); + memcpy(positionalTable, positionEmb, size2 * sizeof(T)); + } else if constexpr (std::is_same_v) { + float16_t::cvt_float_to_float16(tokenEmb, embTable, size1); + float16_t::cvt_float_to_float16(positionEmb, positionalTable, size2); + } else { + printf("Type %s not supported!\n", typeid(T).name()); + exit(-1); + } + } + + // TODO: mask is not considered + // tokenIds and positions are 2-dimension array with batchSize rows, and seqLen cols + void forward(int *tokenIds, int *positions, float *output, int batchSize, int seqLen) { + if (embeddingSize != hiddenSize) { + printf("Not supported yet: embeddingSize != hiddenSize\n"); + exit(-1); + } + + int row = 0; + + if constexpr (std::is_same_v) { + for (int i = 0; i < batchSize; ++i) { + for (int j = 0; j < seqLen; ++j) { + // Embedding + int id = tokenIds[i * seqLen + j]; + float16_t::cvt_float16_to_float( + embTable + id * embeddingSize, output + row * hiddenSize, embeddingSize); + + // Positional embedding + int pos = positions[i * seqLen + j]; + // # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 + // # and adjust num_embeddings appropriately. Other models don't have this hack + // Do not add the offset if the embedding table is already handled it (like FasterTransformer) + //pos += 2; + float16_t::float_add_float16(output + row * hiddenSize, positionalTable + pos * hiddenSize, + output + row * hiddenSize, hiddenSize); + + row += 1; + } + } + } else { + printf("Type %s not supported!\n", typeid(T).name()); + exit(-1); + } + } + + int getVocabSize() { return vocabSize; } + + int getEmbeddingSize() { return embeddingSize; } + + int getMaxPositions() { return maxPositions; } + + int getHiddenSize() { return hiddenSize; } + +private: + // Embedding in OPT models are like: + // self.embed_tokens = nn.Embedding(config.vocab_size, config.word_embed_proj_dim, self.padding_idx) + // self.embed_positions = OPTLearnedPositionalEmbedding(config.max_position_embeddings, config.hidden_size) + int vocabSize; + int embeddingSize; + int maxPositions; + int hiddenSize; + + T *embTable; + T *positionalTable; +}; diff --git a/src/layers/rms_norm.cpp b/src/layers/rms_norm.cpp new file mode 100644 index 0000000..9acca4d --- /dev/null +++ b/src/layers/rms_norm.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#include + +#include +#include + +#include "layers_norm.h" +#include "rmsnorm_kernels.h" +#include "timeline.h" + +namespace xft { + +RmsNorm::RmsNorm() { + weight = nullptr; + normSize = 0; +} + +RmsNorm::~RmsNorm() { + if (weight) { free(weight); } +} + +void RmsNorm::setWeight(const float *w, const float *, int size) { + this->normSize = size; + this->weight = (float *)aligned_alloc(64, size * sizeof(float)); + memcpy(weight, w, size * sizeof(float)); +} + +// input and output are in shape of (rows, normSize) +void RmsNorm::forward(const float *input, float *output, int rows, int iStride, int oStride, float epsilon) { + TimeLine t("RmsNorm.forward"); + invokeRmsNorm(output, input, weight, rows, normSize, iStride, oStride, epsilon); +} + +} // namespace xft \ No newline at end of file diff --git a/src/layers/rms_norm.h b/src/layers/rms_norm.h new file mode 100644 index 0000000..7b53c74 --- /dev/null +++ b/src/layers/rms_norm.h @@ -0,0 +1,87 @@ +#pragma once +#include + +#include +#include + +#include "timeline.h" +#include "transformer_util.h" + +// Layer normalization: only support the norm along last dimension +class RmsNorm { +public: + RmsNorm() { + weight = nullptr; + normSize = 0; + } + + ~RmsNorm() { + if (weight) { free(weight); } + } + + void setWeight(const float *w, const float *, int size) { + this->normSize = size; + this->weight = (float *)aligned_alloc(64, size * sizeof(float)); + memcpy(weight, w, size * sizeof(float)); + } + + // input and output are in shape of (rows, normSize) + void forward( + const float *input, float *output, int rows, int iStride = -1, int oStride = -1, float epsilon = 1e-6) { + TimeLine t("RmsNorm.forward"); + int size = normSize; + + if (iStride == -1) iStride = normSize; + if (oStride == -1) oStride = normSize; + +#pragma omp parallel for + for (int r = 0; r < rows; ++r) { + const float *px = input + r * iStride; + float *py = output + r * oStride; + + float squareSum = 0; + + __m512 vsqare = _mm512_set1_ps(0); + + int col = 0; + for (; col + 15 < size; col += 16) { + // SUM(x*x) + __m512 vx = _mm512_loadu_ps(px + col); + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + if (col < size) { + __mmask16 mask = (1 << (size - col)) - 1; + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + __m512 tmp = _mm512_mul_ps(vx, vx); + vsqare = _mm512_add_ps(vsqare, tmp); + } + + squareSum = _mm512_reduce_add_ps(vsqare); + + // Variance + float var = 1 / sqrt(squareSum / size + epsilon); + __m512 vvar = _mm512_set1_ps(var); + + for (col = 0; col + 15 < size; col += 16) { + __m512 vx = _mm512_loadu_ps(px + col); + __m512 vw = _mm512_loadu_ps(weight + col); + __m512 vy = vx * vvar * vw; + _mm512_storeu_ps(py + col, vy); + } + if (col < size) { + __mmask16 mask = (1 << (size - col)) - 1; + __m512 vx = _mm512_maskz_loadu_ps(mask, px + col); + __m512 vw = _mm512_maskz_loadu_ps(mask, weight + col); + __m512 vy = vx * vvar * vw; + _mm512_mask_storeu_ps(py + col, mask, vy); + } + } // end for rows + } + +private: + int normSize; + + // the scale weight + float *weight; +}; \ No newline at end of file diff --git a/src/layers/rope_2d.cpp b/src/layers/rope_2d.cpp new file mode 100644 index 0000000..377c169 --- /dev/null +++ b/src/layers/rope_2d.cpp @@ -0,0 +1,172 @@ +#include "rope_2d.h" + +#include "compile_util.h" +#include "float16.h" + +static int max_seq_len_cached = -1; +static int inv_freq_size = -1; +static float *inv_freq; +static float *emb_cos = nullptr; +static float *emb_sin = nullptr; + +bool RotaryEmbedding2D::initialized = false; + +// dim: equals to head size, but in rotary_2d, we need to be a half of head size +RotaryEmbedding2D::RotaryEmbedding2D(const int _dim, const int max_position_embeddings, const float base) { + int dim = _dim / 2; + + if (!initialized) { + initialized = true; + + max_seq_len_cached = max_position_embeddings; + inv_freq_size = (dim + 1) / 2; + inv_freq = (float *)malloc(inv_freq_size * sizeof(float)); + for (size_t i = 0; i < inv_freq_size; i++) { + inv_freq[i] = 1.0 / pow(base, float(i * 2) / dim); + } + + // Convert to FP16 (align with ChatGLM) + float16_t *buf = (float16_t *)malloc(inv_freq_size * sizeof(float16_t)); + float16_t::cvt_float_to_float16(inv_freq, buf, inv_freq_size); + float16_t::cvt_float16_to_float(buf, inv_freq, inv_freq_size); + free(buf); + + prepareEmbedding(); + } else if (dim != inv_freq_size * 2) { + printf("Incorrect dim=%d, inv_freq_size=%d\n", dim, inv_freq_size); + exit(-1); + } +}; + +void RotaryEmbedding2D::prepareEmbedding() { + emb_cos = (float *)aligned_alloc(64, max_seq_len_cached * (inv_freq_size * 2) * sizeof(float)); + emb_sin = (float *)aligned_alloc(64, max_seq_len_cached * (inv_freq_size * 2) * sizeof(float)); + +#pragma omp parallel for + for (size_t i = 0; i < max_seq_len_cached; i++) { + float *pcos = emb_cos + i * inv_freq_size * 2; + float *psin = emb_sin + i * inv_freq_size * 2; + + for (size_t j = 0; j < inv_freq_size; j++) { + float tmp = i * inv_freq[j]; + float cos_tmp = std::cos(tmp); + float sin_tmp = std::sin(tmp); + + pcos[j] = cos_tmp; + pcos[j + inv_freq_size] = cos_tmp; + psin[j] = sin_tmp; + psin[j + inv_freq_size] = sin_tmp; + } + } +} + +// FOR PYTHON CODE LIKE BELOW: +// +// q1, q2 = query_layer.chunk(2, dim=(query_layer.ndim - 1)) +// k1, k2 = key_layer.chunk(2, dim=(key_layer.ndim - 1)) +// cos, sin = self.rotary_emb(q1, seq_len=position_ids.max() + 1) +// position_ids, block_position_ids = position_ids[:, 0, :].transpose(0, 1).contiguous(), \ +// position_ids[:, 1, :].transpose(0, 1).contiguous() +// q1, k1 = apply_rotary_pos_emb_index(q1, k1, cos, sin, position_ids) +// q2, k2 = apply_rotary_pos_emb_index(q2, k2, cos, sin, block_position_ids) +// query_layer = torch.concat([q1, q2], dim=(q1.ndim - 1)) +// key_layer = torch.concat([k1, k2], dim=(k1.ndim - 1)) +// +// def apply_rotary_pos_emb_index(q, k, cos, sin, position_id): +// # position_id: [sq, b], q, k: [sq, b, np, hn], cos: [sq, 1, hn] -> [sq, b, 1, hn] +// cos, sin = F.embedding(position_id, cos.squeeze(1)).unsqueeze(2), \ +// F.embedding(position_id, sin.squeeze(1)).unsqueeze(2) +// q, k = (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin) +// return q, k +// +// qk_shape: 4 values of [batch_size, seq_len, head_num, head_size] +// positions: position_ids + block_position_ids, with the size of 2*seq_len +// query and key is the matrix like below: +// +// |<------------------------------ head_size * head_num --------------------------------->| +// |_head_size|_____________________________________________________________________________ ____ +// | | | | | | | | | ^ +// | | | | | | | | | | +// | | | | | | | | | bs*seq_len +// | | | | | | | | | | +// | | | | | | | | | | +// |__________|__________|__________|__________|__________|__________|__________|__________| __v__ +void RotaryEmbedding2D::forward( + float *query, float *key, int qStride, int kStride, const int *qk_shape, const int *positions) { + int dim = inv_freq_size * 2; + REQUIRES(dim * 2 == qk_shape[3], "Incorrect shape, last dimention is not the head size."); + + const int batch_size = qk_shape[0]; + const int seq_len = qk_shape[1]; + const int head_num = qk_shape[2]; + const int head_size = qk_shape[3]; + const int half = inv_freq_size; + +#pragma omp parallel for + for (int head = 0; head < head_num; ++head) { + int off = head * head_size; + int row = 0; + + const int *position_ids = positions; + const int *block_position_ids = positions + seq_len; + + for (int bs = 0; bs < batch_size; ++bs) { + for (int seq = 0; seq < seq_len; ++seq) { + float *p1 = query + row * qStride + off; + float *p2 = key + row * kStride + off; + + int pos = position_ids[seq]; + if (unlikely(pos >= max_seq_len_cached)) { + printf("Unexpected position (%d), please expand the rotary table.\n", pos); + exit(-1); + } + + float *pcos = emb_cos + pos * dim; + float *psin = emb_sin + pos * dim; + +#pragma omp simd + for (int i = 0; i < half; ++i) { + auto t1 = p1[i]; + auto t2 = p2[i]; + + p1[i] = p1[i] * pcos[i] - p1[i + half] * psin[i]; + p2[i] = p2[i] * pcos[i] - p2[i + half] * psin[i]; + + p1[i + half] = p1[i + half] * pcos[i + half] + t1 * psin[i + half]; + p2[i + half] = p2[i + half] * pcos[i + half] + t2 * psin[i + half]; + } + + // Update params to compute for the second half + p1 += dim; + p2 += dim; + + pos = block_position_ids[seq]; + if (unlikely(pos >= max_seq_len_cached)) { + printf("Unexpected block position (%d), please expand the rotary table.\n", pos); + exit(-1); + } + + pcos = emb_cos + pos * dim; + psin = emb_sin + pos * dim; + +#pragma omp simd + for (int i = 0; i < half; ++i) { + auto t1 = p1[i]; + auto t2 = p2[i]; + + p1[i] = p1[i] * pcos[i] - p1[i + half] * psin[i]; + p2[i] = p2[i] * pcos[i] - p2[i + half] * psin[i]; + + p1[i + half] = p1[i + half] * pcos[i + half] + t1 * psin[i + half]; + p2[i + half] = p2[i + half] * pcos[i + half] + t2 * psin[i + half]; + } + + row += 1; + } // end seq + + // Update position_ids and block_position_ids + position_ids += seq_len * 2; + block_position_ids += seq_len * 2; + } // end bs + } // end head +} diff --git a/src/layers/rope_2d.h b/src/layers/rope_2d.h new file mode 100644 index 0000000..986e7a1 --- /dev/null +++ b/src/layers/rope_2d.h @@ -0,0 +1,21 @@ +#pragma once +#include +#include +#include +using namespace std; + +// 2D rotary embedding for ChatGLM +class RotaryEmbedding2D { +public: + RotaryEmbedding2D(const int dim, const int max_position_embeddings = 2048, const float base = 10000); + + ~RotaryEmbedding2D() {} + + void forward(float *query, float *key, int qStride, int kStride, const int *qk_shape, const int *positions); + +private: + void prepareEmbedding(); + +private: + static bool initialized; +}; diff --git a/src/layers/rotary_embedding.cpp b/src/layers/rotary_embedding.cpp new file mode 100644 index 0000000..e328fbd --- /dev/null +++ b/src/layers/rotary_embedding.cpp @@ -0,0 +1,127 @@ +#include "rotary_embedding.h" + +#include "compile_util.h" + +static int max_seq_len_cached = -1; +static int inv_freq_size = -1; +static float *inv_freq; +static float *emb_cos = nullptr; +static float *emb_sin = nullptr; + +bool LlamaRotaryEmbedding::initialized = false; + +// dim: equals to head size +LlamaRotaryEmbedding::LlamaRotaryEmbedding(const int dim, const int max_position_embeddings, const float base) { + if (!initialized) { + initialized = true; + + max_seq_len_cached = max_position_embeddings; + inv_freq_size = (dim + 1) / 2; + inv_freq = (float *)malloc(inv_freq_size * sizeof(float)); + for (size_t i = 0; i < inv_freq_size; i++) { + inv_freq[i] = 1.0 / pow(base, float(i * 2) / dim); + } + + llamaCalEmb(); + } else if (dim != inv_freq_size * 2) { + printf("Incorrect dim=%d, inv_freq_size=%d\n", dim, inv_freq_size); + exit(-1); + } +}; + +void LlamaRotaryEmbedding::llamaCalEmb() { + emb_cos = (float *)aligned_alloc(64, max_seq_len_cached * (inv_freq_size * 2) * sizeof(float)); + emb_sin = (float *)aligned_alloc(64, max_seq_len_cached * (inv_freq_size * 2) * sizeof(float)); + +#pragma omp parallel for + for (size_t i = 0; i < max_seq_len_cached; i++) { + float *pcos = emb_cos + i * inv_freq_size * 2; + float *psin = emb_sin + i * inv_freq_size * 2; + + for (size_t j = 0; j < inv_freq_size; j++) { + float tmp = i * inv_freq[j]; + float cos_tmp = std::cos(tmp); + float sin_tmp = std::sin(tmp); + + pcos[j] = cos_tmp; + pcos[j + inv_freq_size] = cos_tmp; + psin[j] = sin_tmp; + psin[j + inv_freq_size] = sin_tmp; + } + } +} + +// def rotate_half(x): +// """Rotates half the hidden dims of the input.""" +// x1 = x[..., : x.shape[-1] // 2] +// x2 = x[..., x.shape[-1] // 2 :] +// return torch.cat((-x2, x1), dim=-1) +// def apply_rotary_pos_emb(q, k, cos, sin, position_ids): +// # The first two dimensions of cos and sin are always 1, so we can `squeeze` them. +// cos = cos.squeeze(1).squeeze(0) # [seq_len, dim] +// sin = sin.squeeze(1).squeeze(0) # [seq_len, dim] +// cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] +// sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] +// q_embed = (q * cos) + (rotate_half(q) * sin) +// k_embed = (k * cos) + (rotate_half(k) * sin) +// return q_embed, k_embed +// +// qk_shape: 4 values of [batch_size, seq_len, head_num, head_size] +// position_ids: an array in the size of seq_len +// query and key is the matrix like below: +// +// |<------------------------------ head_size * head_num --------------------------------->| +// |_head_size|___________________________________________________________________________________ +// | | | | | | | | | ^ +// | | | | | | | | | | +// | | | | | | | | | bs*seq_len +// | | | | | | | | | | +// | | | | | | | | | | +// |__________|__________|__________|__________|__________|__________|__________|__________|____v__ +void LlamaRotaryEmbedding::forward( + float *query, float *key, int qStride, int kStride, const int *qk_shape, const int *position_ids) { + int dim = inv_freq_size * 2; + REQUIRES(dim == qk_shape[3], "Incorrect shape, last dimention is not the head size."); + + const int batch_size = qk_shape[0]; + const int seq_len = qk_shape[1]; + const int head_num = qk_shape[2]; + const int half = inv_freq_size; + + // for (size_t i = 0; i < emb_size; i++) { + // emb[i] = x[i] * emb_cos[position_ids[i % cached_size / dim]][i % dim]; + // int offset = (i % dim + inv_freq_size) % dim; + // float sign = ((offset < inv_freq_size) * 1) + ((offset >= inv_freq_size) * -1); + // emb[i] += x[(i - i % dim) + offset] * sign * emb_sin[position_ids[i % cached_size / dim]][i % dim]; + // } +#pragma omp parallel for + for (int head = 0; head < head_num; ++head) { + int off = head * dim; + int row = 0; + + for (int bs = 0; bs < batch_size; ++bs) { + for (int seq = 0; seq < seq_len; ++seq) { + float *p1 = query + row * qStride + off; + float *p2 = key + row * kStride + off; + + int pos = position_ids[seq]; + float *pcos = emb_cos + pos * dim; + float *psin = emb_sin + pos * dim; + +#pragma omp simd + for (int i = 0; i < half; ++i) { + auto t1 = p1[i]; + auto t2 = p2[i]; + + p1[i] = p1[i] * pcos[i] - p1[i + half] * psin[i]; + p2[i] = p2[i] * pcos[i] - p2[i + half] * psin[i]; + + p1[i + half] = p1[i + half] * pcos[i + half] + t1 * psin[i + half]; + p2[i + half] = p2[i + half] * pcos[i + half] + t2 * psin[i + half]; + } + + row += 1; + } + } + } +} diff --git a/src/layers/rotary_embedding.h b/src/layers/rotary_embedding.h new file mode 100644 index 0000000..e893b57 --- /dev/null +++ b/src/layers/rotary_embedding.h @@ -0,0 +1,31 @@ +#pragma once +#include +#include +#include + +/* Sample: + int bs = 2 headnum = 3 seq = 4 dim = 6; + int max_len = 10; + int pos_ids[4] = {2,0,1,3}; // seq = 4 , Each batch have same value + int pos_shape[2] = {bs, seq}; + float x[144] = {0, 1, 1,...}; // bs * h * seq * dim = 144 + int xshape[4] = {bs,headnum,seq,dim}; + Forward + LlamaRotaryEmbedding emb(dim, seq); + float *embd = emb.forward(x, x_shape, pos_ids, pos_shape); +*/ + +class LlamaRotaryEmbedding { +public: + LlamaRotaryEmbedding(const int dim, const int max_position_embeddings = 2048, const float base = 10000); + + ~LlamaRotaryEmbedding() {} + + void forward(float *query, float *key, int qStride, int kStride, const int *qk_shape, const int *position_ids); + +private: + void llamaCalEmb(); + +private: + static bool initialized; +}; diff --git a/src/layers/rotary_embedding_chatglm2.cpp b/src/layers/rotary_embedding_chatglm2.cpp new file mode 100644 index 0000000..ec1e72e --- /dev/null +++ b/src/layers/rotary_embedding_chatglm2.cpp @@ -0,0 +1,103 @@ +#include "rotary_embedding_chatglm2.h" + +#include "compile_util.h" + +static int max_seq_len_cached = -1; +static int inv_freq_size = -1; +static float *inv_freq; +static float *emb_cos = nullptr; +static float *emb_sin = nullptr; + +bool ChatGLM2RotaryEmbedding::initialized = false; + +// dim: equals to head size +ChatGLM2RotaryEmbedding::ChatGLM2RotaryEmbedding(const int dim, const int max_position_embeddings, const float base) { + if (!initialized) { + initialized = true; + + max_seq_len_cached = max_position_embeddings; + inv_freq_size = (dim + 1) / 2; + inv_freq = (float *)malloc(inv_freq_size * sizeof(float)); +#pragma omp parallel for + for (size_t i = 0; i < inv_freq_size; i++) { + inv_freq[i] = 1.0 / pow(base, float(i * 2) / dim); + } + + glm2CalEmb(); + } else if (dim != inv_freq_size * 2) { + printf("Incorrect dim=%d, inv_freq_size=%d\n", dim, inv_freq_size); + exit(-1); + } +}; + +void ChatGLM2RotaryEmbedding::glm2CalEmb() { + emb_cos = (float *)aligned_alloc(64, max_seq_len_cached * (inv_freq_size * 2) * sizeof(float)); + emb_sin = (float *)aligned_alloc(64, max_seq_len_cached * (inv_freq_size * 2) * sizeof(float)); + +#pragma omp parallel for + for (size_t i = 0; i < max_seq_len_cached; i++) { + float *pcos = emb_cos + i * inv_freq_size * 2; + float *psin = emb_sin + i * inv_freq_size * 2; + + for (size_t j = 0; j < inv_freq_size; j++) { + float tmp = i * inv_freq[j]; + float cos_tmp = std::cos(tmp); + float sin_tmp = std::sin(tmp); + + pcos[j] = cos_tmp; + pcos[j + inv_freq_size] = cos_tmp; + psin[j] = sin_tmp; + psin[j + inv_freq_size] = sin_tmp; + } + } +} + +// def apply_rotary_pos_emb(x: torch.Tensor, rope_cache: torch.Tensor) -> torch.Tensor: +// #x : [sq, b, np, hn] +// sq, b, np, hn = x.size(0), x.size(1), x.size(2), x.size(3) +// rot_dim = rope_cache.shape[-2] * 2 +// x, x_pass = x[..., :rot_dim], x[..., rot_dim:] +// #truncate to support variable sizes +// rope_cache = rope_cache[:sq] +// xshaped = x.reshape(sq, -1, np, rot_dim // 2, 2) +// rope_cache = rope_cache.view(sq, -1, 1, xshaped.size(3), 2) +// print('### rope_cache={}, x={}, sq={}, b={}, np={}, hn={}'.format(rope_cache.shape, xshaped.shape, sq, b, np, hn)) +// x_out2 = torch.stack( +// [ +// xshaped[..., 0] * rope_cache[..., 0] - xshaped[..., 1] * rope_cache[..., 1], +// xshaped[..., 1] * rope_cache[..., 0] + xshaped[..., 0] * rope_cache[..., 1], +// ], +// -1, +// ) +// x_out2 = x_out2.flatten(3) +// return torch.cat((x_out2, x_pass), dim=-1) + +void ChatGLM2RotaryEmbedding::forward(float *buf, int bufStride, int batch_size, int seq_len, int qk_size, + int hidden_size_per_attention_head, const int *position_ids) { + int dim = inv_freq_size * 2; + REQUIRES(dim == hidden_size_per_attention_head, "Incorrect shape, last dimention is not the head size."); + + const int half = inv_freq_size; + +#pragma omp parallel for + for (int head = 0; head < qk_size / hidden_size_per_attention_head; ++head) { + int off = head * dim; + for (int bs = 0; bs < batch_size; ++bs) { + for (int seq = 0; seq < seq_len; ++seq) { + float *p1 = buf + off; + + int pos = position_ids[seq]; + float *pcos = emb_cos + pos * dim; + float *psin = emb_sin + pos * dim; + +#pragma omp simd + for (int i = 0; i < half; i += 2) { + auto t1 = p1[i]; + p1[i] = p1[i] * pcos[i] - p1[i + 1] * psin[i]; + p1[i + 1] = p1[i + 1] * pcos[i] + t1 * psin[i]; + } + off += bufStride; + } + } + } +} diff --git a/src/layers/rotary_embedding_chatglm2.h b/src/layers/rotary_embedding_chatglm2.h new file mode 100644 index 0000000..490483d --- /dev/null +++ b/src/layers/rotary_embedding_chatglm2.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include +#include + +/* Sample: + int bs = 2 headnum = 3 seq = 4 dim = 6; + int max_len = 10; + int pos_ids[4] = {2,0,1,3}; // seq = 4 , Each batch have same value + int pos_shape[2] = {bs, seq}; + float x[144] = {0, 1, 1,...}; // bs * h * seq * dim = 144 + int xshape[4] = {bs,headnum,seq,dim}; + Forward + LlamaRotaryEmbedding emb(dim, seq); + float *embd = emb.forward(x, x_shape, pos_ids, pos_shape); +*/ + +class ChatGLM2RotaryEmbedding { +public: + ChatGLM2RotaryEmbedding(const int dim, const int max_position_embeddings = 32768, const float base = 10000.0); + + ~ChatGLM2RotaryEmbedding() {} + + void forward(float *buf, int bufStride, int batch_size, int seq_len, int qk_size, + int hidden_size_per_attention_head, const int *position_ids); + +private: + void glm2CalEmb(); + +private: + static bool initialized; +}; diff --git a/src/layers/token_embedding.h b/src/layers/token_embedding.h new file mode 100644 index 0000000..346cc7c --- /dev/null +++ b/src/layers/token_embedding.h @@ -0,0 +1,56 @@ +#pragma once +#include "float16.h" +#include "transformer_ctx.h" + +template +class TokenEmbedding { +public: + TokenEmbedding(DecoderContext *ctx) { + this->vocabSize = ctx->vocabSize; + this->hiddenSize = ctx->hiddenSize; + } + + void setWeights(float *tokenEmb) { + int size = vocabSize * hiddenSize; + embTable = (T *)aligned_alloc(64, size * sizeof(T)); + + if constexpr (std::is_same_v) { + memcpy(embTable, tokenEmb, size * sizeof(T)); + } else if constexpr (std::is_same_v) { + float16_t::cvt_float_to_float16(tokenEmb, embTable, size); + } else { + printf("Type %s not supported!\n", typeid(T).name()); + exit(-1); + } + } + + // tokenIds ia a 2-dimension array with batchSize rows, and seqLen cols + void forward(int *tokenIds, float *output, int batchSize, int seqLen) { + if constexpr (std::is_same_v) { + for (int i = 0; i < batchSize * seqLen; ++i) { + int id = tokenIds[i]; + memcpy(output + i * hiddenSize, embTable + id * hiddenSize, hiddenSize * sizeof(float)); + } + } else if constexpr (std::is_same_v) { + for (int i = 0; i < batchSize * seqLen; ++i) { + int id = tokenIds[i]; + float16_t::cvt_float16_to_float(embTable + id * hiddenSize, output + i * hiddenSize, hiddenSize); + } + } else { + printf("Type %s not supported!\n", typeid(T).name()); + exit(-1); + } + } + + int getVocabSize() { return vocabSize; } + + int getHiddenSize() { return hiddenSize; } + +private: + // Embedding like: + // self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + int vocabSize; + int hiddenSize; + + T *embTable; +}; diff --git a/src/models/CMakeLists.txt b/src/models/CMakeLists.txt new file mode 100644 index 0000000..30dfd00 --- /dev/null +++ b/src/models/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.15.1) + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} MODEL_SRCS) + +add_library(models OBJECT ${MODEL_SRCS}) +add_dependencies(models utils) diff --git a/src/models/baichuan.cpp b/src/models/baichuan.cpp new file mode 100644 index 0000000..6a986b3 --- /dev/null +++ b/src/models/baichuan.cpp @@ -0,0 +1,146 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#include + +#include "baichuan.h" + +template +Baichuan::Baichuan(const std::string &modelPath) + : CommonDecoder, LlamaMLP, float>(modelPath, "baichuan") { + // Context + DecoderContext *ctx = this->getContext(); + + // Embedding (no need position embed) + embedding = new TokenEmbedding(ctx); + setEmbeddingWeights(modelPath); + + // Final LN + setFinalLnWeight(modelPath); +} + +template +Baichuan::~Baichuan() { + delete embedding; +} + +template +void Baichuan::setEmbeddingWeights(const std::string &modelPath) { + int vocabSize = embedding->getVocabSize(); + int hiddenSize = embedding->getHiddenSize(); + + float *tokenEmb = (float *)malloc(vocabSize * hiddenSize * sizeof(float)); + + loadWeight(modelPath + "/model.wte.bin", tokenEmb, vocabSize * hiddenSize, this->getDataType()); + + embedding->setWeights(tokenEmb); + + free(tokenEmb); +} + +template +void Baichuan::setFinalLnWeight(const std::string &modelPath) { + int hiddenSize = embedding->getHiddenSize(); + + float *gamma = (float *)malloc(hiddenSize * sizeof(float)); + + loadWeight(modelPath + "/model.final_layernorm.weight.bin", gamma, hiddenSize, this->getDataType()); + + finalLN.setWeight(gamma, nullptr, hiddenSize); + + free(gamma); +} + +// Prepare attention_mask which is like: +//def _get_interleave(n): +// def _get_interleave_power_of_2(n): +// start = (2 ** (-2 ** -(math.log2(n) - 3))) +// ratio = start +// return [start * ratio ** i for i in range(n)] +// +// if math.log2(n).is_integer(): +// return _get_interleave_power_of_2(n) +// else: +// closest_power_of_2 = 2 ** math.floor(math.log2(n)) +// return _get_interleave_power_of_2(closest_power_of_2) + \ +// _get_interleave(2 * closest_power_of_2)[0::2][:n - closest_power_of_2] +//def _gen_alibi_mask(n_head, max_pos): +// """used in inference only""" +// slopes = torch.Tensor(_get_interleave(n_head)) +// alibi = slopes.unsqueeze(1).unsqueeze(1) * torch.arange(max_pos).unsqueeze(0).unsqueeze(0).expand( +// n_head, -1, -1) +// alibi = alibi.view(n_head, 1, max_pos) +// alibi_mask = torch.triu( +// _fill_with_neg_inf(torch.zeros([max_pos, max_pos])), 1 +// ) +// alibi_mask = alibi_mask.unsqueeze(0) + alibi +// return alibi_mask + +template +void Baichuan::prepareAttnMask(int *ids, int step) { + DecoderContext *ctx = this->getContext(); + int seqLen = ctx->inputSeqLen; + BaichuanAttention attn(0, ctx); + int responsibleHeads = attn.getResponsibleHeads(); + // alibi mask slope for each head + const float* alibiSlopes = attn.getAlibiSlopes(ctx->attHeadNum); + + if (step == 0) { + int sizeRequired = responsibleHeads * seqLen * seqLen; + float *mask = this->getAttnMask(sizeRequired); + for (int h = 0; h < responsibleHeads; ++h) { + float slope = 0; + if (ctx->maxPosEmbed <= 0) + slope = alibiSlopes[h]; + auto pmask = mask + h * seqLen * seqLen; + for (int i = 0; i < seqLen; ++i) { + memset(pmask + i * seqLen, 0, (i + 1) * sizeof(float)); // bottom left are 0 + // attention mask added with alibi mask + for (int j = 0; j < seqLen; ++j) { + pmask[i * seqLen + j] += j * slope; + } + std::fill_n(pmask + i * seqLen + i + 1, seqLen - i - 1, std::numeric_limits::lowest()); + } + } + } else { + int sizeRequired = responsibleHeads * this->accSeqLen; + float *mask = this->getAttnMask(sizeRequired); + memset(mask, 0, responsibleHeads * this->accSeqLen * sizeof(float)); // all elements are 0 + for (int h = 0; h < responsibleHeads; ++h) { + // alibi mask slope for each head + float slope = 0; + if (ctx->maxPosEmbed <= 0) + slope = alibiSlopes[h]; + auto pmask = mask + h * this->accSeqLen; + for (int j = 0; j < this->accSeqLen; ++j) { + pmask[j] += j * slope; + } + } + } +} + +template +void Baichuan::embeddingForward(int *ids, float *output, int batchSize, int seqLen) { + embedding->forward(ids, output, batchSize, seqLen); +} + +template +void Baichuan::lastLayerNormForward(float *input, float *output, int rows) { + finalLN.forward(input, output, rows); +} + +template class Baichuan; +template class Baichuan; +template class Baichuan; +template class Baichuan; diff --git a/src/models/baichuan.h b/src/models/baichuan.h new file mode 100644 index 0000000..1deae94 --- /dev/null +++ b/src/models/baichuan.h @@ -0,0 +1,41 @@ +// Copyright (c) 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once + +#include "attn_baichuan.h" +#include "common_decoder.h" +#include "mlp_llama.h" +#include "layers_norm.h" +#include "rotary_embedding.h" +#include "token_embedding.h" + +template +class Baichuan : public CommonDecoder, LlamaMLP, float> { +public: + Baichuan(const std::string &modelPath); + ~Baichuan(); + + void prepareAttnMask(int *ids, int step); + void embeddingForward(int *ids, float *output, int batchSize, int seqLen); + void lastLayerNormForward(float *input, float *output, int rows); + +private: + void setEmbeddingWeights(const std::string &modelPath); + void setFinalLnWeight(const std::string &modelPath); + +private: + TokenEmbedding *embedding; + RmsNorm finalLN; +}; diff --git a/src/models/chatglm.cpp b/src/models/chatglm.cpp new file mode 100644 index 0000000..7e48f43 --- /dev/null +++ b/src/models/chatglm.cpp @@ -0,0 +1,220 @@ +#include +#include + +#include "INIReader.h" +#include "chatglm.h" + +template +ChatGLM::ChatGLM(const std::string &modelPath) + : CommonDecoder, ChatGlmMLP>(modelPath, "chatglm") { + std::string configPath = modelPath + "/config.ini"; + INIReader reader = INIReader(configPath); + + this->maskTokenId = reader.GetInteger("chatglm", "mask_token_id", 130000); + this->gmaskTokenId = reader.GetInteger("chatglm", "gmask_token_id", 130001); + + this->positionIds = nullptr; + this->posBufSize = 0; + + // Context + DecoderContext *ctx = this->getContext(); + + // Embedding + embedding = new TokenEmbedding(ctx); + setEmbeddingWeights(modelPath); + + // Final LN + setFinalLnWeight(modelPath); +} + +template +ChatGLM::~ChatGLM() { + delete embedding; + + if (positionIds) { free(positionIds); } +} + +template +void ChatGLM::setEmbeddingWeights(const std::string &modelPath) { + int vocabSize = embedding->getVocabSize(); + int hiddenSize = embedding->getHiddenSize(); + + float *tokenEmb = (float *)malloc(vocabSize * hiddenSize * sizeof(float)); + + loadWeight(modelPath + "/model.wte.bin", tokenEmb, vocabSize * hiddenSize, this->getDataType()); + + embedding->setWeights(tokenEmb); + + free(tokenEmb); +} + +template +void ChatGLM::setFinalLnWeight(const std::string &modelPath) { + int hiddenSize = embedding->getHiddenSize(); + + float *gamma = (float *)malloc(hiddenSize * sizeof(float)); + float *beta = (float *)malloc(hiddenSize * sizeof(float)); + + loadWeight(modelPath + "/model.final_layernorm.weight.bin", gamma, hiddenSize, this->getDataType()); + loadWeight(modelPath + "/model.final_layernorm.bias.bin", beta, hiddenSize, this->getDataType()); + + finalLN.setWeight(gamma, beta, hiddenSize); + + free(gamma); + free(beta); +} + +// Prepare attention_mask +// Python code: +// def get_masks(self, input_ids, device): +// batch_size, seq_length = input_ids.shape +// context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids] +// attention_mask = torch.ones((batch_size, seq_length, seq_length), device=device) +// attention_mask.tril_() +// for i, context_length in enumerate(context_lengths): +// attention_mask[i, :, :context_length] = 1 +// attention_mask.unsqueeze_(1) +// attention_mask = (attention_mask < 0.5).bool() +// +// return attention_mask +template +void ChatGLM::prepareAttnMask(int *ids, int step) { + DecoderContext *ctx = this->getContext(); + int seqLen = ctx->inputSeqLen; + + if (step == 0) { + int sizeRequired = ctx->batchSize * seqLen * seqLen; + float *mask = this->getAttnMask(sizeRequired); + int startId = this->getStartId(); + + for (int b = 0; b < ctx->batchSize; ++b) { + int contextLen = -1; + auto it = std::find(ids + b * seqLen, ids + (b + 1) * seqLen, startId); + if (it != ids + (b + 1) * seqLen) { contextLen = std::distance(ids + b * seqLen, it); } + + auto pmask = mask + b * seqLen * seqLen; + for (int i = 0; i < seqLen; ++i) { + int zeroLen = contextLen > (i + 1) ? contextLen : (i + 1); + memset(pmask + i * seqLen, 0, zeroLen * sizeof(float)); // bottom left or 0:contextLen are 0 + std::fill_n(pmask + i * seqLen + zeroLen, seqLen - zeroLen, std::numeric_limits::lowest()); + } + } + } else { + int sizeRequired = ctx->batchSize * this->accSeqLen; + float *mask = this->getAttnMask(sizeRequired); + memset(mask, 0, ctx->batchSize * this->accSeqLen * sizeof(float)); // all elements are 0 + } +} + +template +void ChatGLM::embeddingForward(int *ids, float *output, int batchSize, int seqLen) { + embedding->forward(ids, output, batchSize, seqLen); +} + +template +void ChatGLM::lastLayerNormForward(float *input, float *output, int rows) { + finalLN.forward(input, output, rows); +} + +// Return the position_ids + block_position_ids +template +int *ChatGLM::getPositionIds(int *ids, int batchSize, int seqLen, int step) { + if (step == 0) { + maskPositions.clear(); + lastBlockPositions.clear(); + + for (int i = 0; i < batchSize; ++i) { + int *p = ids + i * seqLen; + + // Python code: + // mask_token = gMASK if gMASK in seq else MASK + // mask_positions.append(seq.index(mask_token)) + int maskPos = -1; + bool gMaskDetected = false; + for (int s = 0; s < seqLen; ++s) { + // Always use gMASK if gMASK detected + if (p[s] == gmaskTokenId) { + gMaskDetected = true; + maskPositions.emplace_back(s); + break; + } + // Detect MASK (not gMASK) + if (p[s] == maskTokenId && maskPos == -1) { maskPos == s; } + } + + if (!gMaskDetected) { maskPositions.emplace_back(maskPos); } + } + + // Prepare buffer + int sizeNeeded = 2 * batchSize * seqLen; // position_ids + block_position_ids + if (posBufSize < sizeNeeded) { + if (positionIds) { free(positionIds); } + posBufSize = sizeNeeded + 8; // whatever, a little bigger + positionIds = (int *)aligned_alloc(64, posBufSize * sizeof(int)); + } + + // position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1) + // context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids] + // position_ids[i, context_length:] = mask_positions[i] + // block_position_ids = [torch.cat(( + // torch.zeros(context_length, dtype=torch.long, device=device), + // torch.arange(seq_length - context_length, dtype=torch.long, device=device) + 1 + // )) + int bosId = this->getStartId(); + for (int i = 0; i < batchSize; ++i) { + int *pos = positionIds + i * seqLen * 2; + int *blockPos = pos + seqLen; + + int contextLength = -1; + int *pIds = ids + i * seqLen; + auto it = std::find(pIds, pIds + seqLen, bosId); + + if (unlikely(it == pIds + seqLen)) { + printf("WARNING: cannot find bos_token_id, unexpected!\n"); + continue; + } + + contextLength = std::distance(pIds, it); + + for (int j = 0; j < contextLength; ++j) { + pos[j] = j; + blockPos[j] = 0; + } + + for (int j = contextLength; j < seqLen; ++j) { + pos[j] = maskPositions[i]; + blockPos[j] = j - contextLength + 1; + } + + lastBlockPositions.emplace_back(seqLen - contextLength); + } + } else { + if (batchSize > maskPositions.size()) { + int userSideBS = maskPositions.size(); + int beamSize = batchSize / userSideBS; + std::vector tmpMaskP(maskPositions); + std::vector tmpLastBlockP(lastBlockPositions); + maskPositions.clear(); + lastBlockPositions.clear(); + + maskPositions.reserve(batchSize); + lastBlockPositions.reserve(batchSize); + for (int i = 0; i < userSideBS; ++i) { + maskPositions.insert(maskPositions.begin() + i * beamSize, beamSize, tmpMaskP[i]); + lastBlockPositions.insert(lastBlockPositions.begin() + i * beamSize, beamSize, tmpLastBlockP[i]); + } + } + for (int i = 0; i < batchSize; ++i) { + positionIds[i * 2] = maskPositions[i]; + positionIds[i * 2 + 1] = lastBlockPositions[i] + 1; + lastBlockPositions[i] += 1; + } + } + + return positionIds; +} + +template class ChatGLM; +template class ChatGLM; +template class ChatGLM; +template class ChatGLM; \ No newline at end of file diff --git a/src/models/chatglm.h b/src/models/chatglm.h new file mode 100644 index 0000000..087cf8f --- /dev/null +++ b/src/models/chatglm.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include "attn_chatglm.h" +#include "common_decoder.h" +#include "layer_norm.h" +#include "mlp_chatglm.h" +#include "rms_norm.h" +#include "rope_2d.h" +#include "token_embedding.h" + +template +class ChatGLM : public CommonDecoder, ChatGlmMLP> { +public: + ChatGLM(const std::string &modelPath); + ~ChatGLM(); + + void prepareAttnMask(int *ids, int step); + void embeddingForward(int *ids, float *output, int batchSize, int seqLen); + void lastLayerNormForward(float *input, float *output, int rows); + int *getPositionIds(int *ids, int batchSize, int seqLen, int step) override; + +private: + void setEmbeddingWeights(const std::string &modelPath); + void setFinalLnWeight(const std::string &modelPath); + +private: + TokenEmbedding *embedding; + LayerNorm finalLN; + + // Mask token ID from configuration + int maskTokenId; + int gmaskTokenId; + + // Mask position in current sequence, used to generate position_ids + // mask_positions, use_gmasks = [], [] + // for seq in seqs: + // mask_token = gMASK if gMASK in seq else MASK + // use_gmask = mask_token == gMASK + // mask_positions.append(seq.index(mask_token)) + // use_gmasks.append(use_gmask) + std::vector maskPositions; + + // Record last block positions + std::vector lastBlockPositions; + + // position_ids + block_position_ids + // For input_ids [ 74747, 83400, 66846, 130001, 130004], position_ids is: + // [0, 1, 2, 3, 3] + [0, 0, 0, 0, 1], as gmask_token_id = 130001 + // and next is [3] + [2], ... + int *positionIds; + int posBufSize; +}; \ No newline at end of file diff --git a/src/models/chatglm2.cpp b/src/models/chatglm2.cpp new file mode 100644 index 0000000..34a4454 --- /dev/null +++ b/src/models/chatglm2.cpp @@ -0,0 +1,166 @@ +#include +#include + +#include "INIReader.h" +#include "chatglm2.h" + +const char *model_type = "chatglm2"; +template +ChatGLM2::ChatGLM2(const std::string &modelPath) + : CommonDecoder, ChatGLM2MLP>( + modelPath, model_type) { + this->positionIds = nullptr; + this->posBufSize = 0; + + // Context + DecoderContext *ctx = this->getContext(); + + // Embedding + embedding = new TokenEmbedding(ctx); + setEmbeddingWeights(modelPath); + + // Final LN + setFinalLnWeight(modelPath); +} + +template +ChatGLM2::~ChatGLM2() { + delete embedding; + + if (positionIds) { free(positionIds); } +} + +template +void ChatGLM2::setEmbeddingWeights(const std::string &modelPath) { + int vocabSize = embedding->getVocabSize(); + int hiddenSize = embedding->getHiddenSize(); + + float *tokenEmb = (float *)malloc(vocabSize * hiddenSize * sizeof(float)); + + loadWeight(modelPath + "/model.wte.bin", tokenEmb, vocabSize * hiddenSize, this->getDataType()); + + embedding->setWeights(tokenEmb); + + free(tokenEmb); +} + +template +void ChatGLM2::setFinalLnWeight(const std::string &modelPath) { + int hiddenSize = embedding->getHiddenSize(); + + float *gamma = (float *)malloc(hiddenSize * sizeof(float)); + float *beta = (float *)malloc(hiddenSize * sizeof(float)); + + loadWeight(modelPath + "/model.final_layernorm.weight.bin", gamma, hiddenSize, this->getDataType()); + + finalLN.setWeight(gamma, beta, hiddenSize); + + free(gamma); + free(beta); +} + +// Prepare attention_mask +// Python code: +// def get_masks(self, input_ids, device): +// batch_size, seq_length = input_ids.shape +// context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids] +// attention_mask = torch.ones((batch_size, seq_length, seq_length), device=device) +// attention_mask.tril_() +// for i, context_length in enumerate(context_lengths): +// attention_mask[i, :, :context_length] = 1 +// attention_mask.unsqueeze_(1) +// attention_mask = (attention_mask < 0.5).bool() +// +// return attention_mask +template +void ChatGLM2::prepareAttnMask(int *ids, int step) { + DecoderContext *ctx = this->getContext(); + int seqLen = ctx->inputSeqLen; + int sizeRequired = ctx->batchSize * seqLen * seqLen; + + if (step == 0) { + float *mask = this->getAttnMask(sizeRequired); + int startId = this->getStartId(); + + for (int b = 0; b < ctx->batchSize; ++b) { + int contextLen = -1; + auto it = std::find(ids + b * seqLen, ids + (b + 1) * seqLen, startId); + if (it != ids + (b + 1) * seqLen) { contextLen = std::distance(ids + b * seqLen, it); } + + auto pmask = mask + b * seqLen * seqLen; + for (int i = 0; i < seqLen; ++i) { + int zeroLen = contextLen > (i + 1) ? contextLen : (i + 1); + memset(pmask + i * seqLen, 0, zeroLen * sizeof(float)); // bottom left or 0:contextLen are 0 + std::fill_n(pmask + i * seqLen + zeroLen, seqLen - zeroLen, std::numeric_limits::lowest()); + } + } + } else { + int sizeRequired = ctx->batchSize * this->accSeqLen; + float *mask = this->getAttnMask(sizeRequired); + memset(mask, 0, ctx->batchSize * this->accSeqLen * sizeof(float)); // all elements are 0 + } +} + +template +void ChatGLM2::embeddingForward(int *ids, float *output, int batchSize, int seqLen) { + embedding->forward(ids, output, batchSize, seqLen); +} + +template +void ChatGLM2::lastLayerNormForward(float *input, float *output, int rows) { + finalLN.forward(input, output, rows); +} + +// Return the position_ids + block_position_ids +// if position_ids is None: +// position_ids = self.get_position_ids(input_ids, device=input_ids.device) +// if not is_first_forward: +// position_ids = position_ids[..., -1:] +// input_ids = input_ids[:, -1:] +// def get_position_ids(self, input_ids, device): +// batch_size, seq_length = input_ids.shape +// position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1) +// return position_ids +template +int *ChatGLM2::getPositionIds(int *ids, int batchSize, int seqLen, int step) { + // printf("ChatGLM2 getPositionIds batchSize=%d, seqLen=%d, step=%d\n", batchSize, seqLen, step); + if (step == 0) { + // Prepare buffer + lastBlockPositions.clear(); + int sizeNeeded = (batchSize * seqLen + 63) / 64 * 64; // position_ids + block_position_ids + if (posBufSize < sizeNeeded) { + if (positionIds) { free(positionIds); } + posBufSize = sizeNeeded + 8; // whatever, a little bigger + positionIds = (int *)aligned_alloc(64, posBufSize * sizeof(int)); + } + for (int i = 0; i < batchSize; ++i) { + int *pos = positionIds + i * seqLen; + for (int j = 0; j < seqLen; ++j) { + pos[j] = j; + } + lastBlockPositions.emplace_back(seqLen); + } + } else { + if (batchSize > lastBlockPositions.size()) { + int userSideBS = lastBlockPositions.size(); + int beamSize = batchSize / userSideBS; + std::vector tmp(lastBlockPositions); + lastBlockPositions.clear(); + + lastBlockPositions.reserve(batchSize); + for (int i = 0; i < userSideBS; ++i) { + lastBlockPositions.insert(lastBlockPositions.begin() + i * beamSize, beamSize, tmp[i]); + } + } + for (int i = 0; i < batchSize; ++i) { + positionIds[i] = lastBlockPositions[i]; + lastBlockPositions[i] += 1; + } + } + return positionIds; +} + +template class ChatGLM2; +template class ChatGLM2; +template class ChatGLM2; +template class ChatGLM2; \ No newline at end of file diff --git a/src/models/chatglm2.h b/src/models/chatglm2.h new file mode 100644 index 0000000..a3d3606 --- /dev/null +++ b/src/models/chatglm2.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include "attn_chatglm2.h" +#include "common_decoder.h" +#include "layer_norm.h" +#include "mlp_chatglm2.h" +#include "rms_norm.h" +#include "rotary_embedding_chatglm2.h" +#include "token_embedding.h" + +template +class ChatGLM2 : public CommonDecoder, + ChatGLM2MLP> { +public: + ChatGLM2(const std::string &modelPath); + ~ChatGLM2(); + + virtual void prepareAttnMask(int *ids, int step); + virtual void embeddingForward(int *ids, float *output, int batchSize, int seqLen); + virtual void lastLayerNormForward(float *input, float *output, int rows); + virtual int *getPositionIds(int *ids, int batchSize, int seqLen, int step) override; + +private: + virtual void setEmbeddingWeights(const std::string &modelPath); + virtual void setFinalLnWeight(const std::string &modelPath); + +private: + TokenEmbedding *embedding; + NormT finalLN; + + // Record last block positions + std::vector lastBlockPositions; + + // position_ids + // For input_ids [ 74747, 83400, 66846, 130001, 130004], position_ids is: + // [0, 1, 2, 3, 4] + // and next is [5], ... + int *positionIds; + int posBufSize; +}; diff --git a/src/models/common_decoder.h b/src/models/common_decoder.h new file mode 100644 index 0000000..2afc6b4 --- /dev/null +++ b/src/models/common_decoder.h @@ -0,0 +1,537 @@ +#pragma once + +#include +#include +#include +#include + +#include "INIReader.h" +#include "abstract_decoder.h" +#include "attention.h" +#include "debugger.h" +#include "decoder_layer.h" +#include "dist_linear.h" +#include "kvcache_manager.h" +#include "messenger.h" +#include "timeline.h" +#include "transformer_ctx.h" +#include "transpose_util.h" +#include "weight_util.h" + +using namespace xft; + +struct QKPO_Dummy { + QKPO_Dummy(int dim) {} + void forward(float *query, float *key, int qStride, int kStride, const int *qk_shape, const int *position_ids) {} +}; + +// Template parameters: +// ATTN_CLS - class for attention impl. +// MLP_CLS - MLP implementation +// KVCacheT - data type of the cached keys/values +// ATTN_MLP_PARALLEL - true means attention and MLP are in parallel, using the same initial input +template +class CommonDecoder : public AbstractDecoder { +public: + CommonDecoder(const std::string &modelPath, const std::string &modelType) + : messenger(Messenger::getInstance()) +#ifdef DEBUG + , dbg("model_decoder.csv") +#endif + { + std::string configPath = modelPath + "/config.ini"; + INIReader reader = INIReader(configPath); + wType = getWeightType(configPath, modelType); + + const int attHeadNum = reader.GetInteger(modelType, "head_num"); + // Use the same head number for the default multi-head attention + const int kvHeadNum = reader.GetInteger(modelType, "kv_head_num", attHeadNum); + const int size_per_head = reader.GetInteger(modelType, "size_per_head"); + const int imSize = reader.GetInteger(modelType, "inter_size"); + const int layers = reader.GetInteger(modelType, "num_layer"); + const int vocabSize = reader.GetInteger(modelType, "vocab_size"); + // Use 2k as default value + const int maxPositions = reader.GetInteger(modelType, "max_pos_seq_len", 2048); + const int hiddenSize = attHeadNum * size_per_head; + const int embeddingSize = hiddenSize; + const int multi_query_group_num = reader.GetInteger(modelType, "multi_query_group_num", attHeadNum); + const float epsilon = reader.GetFloat(modelType, "layernorm_eps", 1e-6); + + std::string act = reader.Get(modelType, "activation_type"); + std::transform(act.begin(), act.end(), act.begin(), ::tolower); + + this->startId = reader.GetInteger(modelType, "start_id", 0); + this->endId = reader.GetInteger(modelType, "end_id", startId); + + this->initSeqLen = 0; + this->accSeqLen = 0; + + // Buffer related (not initialized) + this->inputTokens = nullptr; + this->maskSize = 0; + this->attnMask = nullptr; + embBuf.reset(new hpj::Matrix()); + outBuf.reset(new hpj::Matrix()); + + // Context + DecoderContext *ctx = getDecoderContext(layers, hiddenSize, attHeadNum, kvHeadNum, imSize, act, epsilon, + vocabSize, embeddingSize, maxPositions); + + // Decoder + for (int i = 0; i < layers; ++i) { + auto pdec = new DECODER(ctx, i); + this->setDecoderWeights(pdec, modelPath, i); + this->decoders.push_back(pdec); + } + + // Predictor + int workers = messenger.getSize(); + int rank = messenger.getRank(); + this->predictor = new DistLinear(hiddenSize, vocabSize, rank, workers); + this->setPredictorWeight(modelPath); + + // KVCache Manager + this->kvCacheMgr.reset(new KVCacheManager(layers)); + } + + virtual ~CommonDecoder() { + if (this->inputTokens) free(this->inputTokens); + if (this->attnMask) free(this->attnMask); + + delete this->predictor; + + for (auto dec : this->decoders) { + delete dec; + } + } + + std::tuple forward(int *ids, int64_t *dims, int step, bool logitsAll = false) { + // Assume input has been synced with master in higher level. + // Assume the 1st step input's shape is [userSideBS][1][seqLen]. + TimeLine t("Decoder.forward"); + TimeLine t1("Decoder.embedding"); + + int userSideBS = dims[0]; + int beamSize = dims[1]; + int batchSize = (step == 0 ? userSideBS : userSideBS * beamSize); // as samples are duplicated at step 0 + int seqLen = dims[2]; + + // Prepare context + DecoderContext *ctx = this->getContext(); + ctx->resize(batchSize, seqLen, (step == 0 ? 0 : this->accSeqLen)); + + if (step == 0) { + // Enlarge buffer if needed + prepareBuffers(ctx, userSideBS, beamSize, logitsAll); + + // Reset initial and accumulated sequence length at the first step + this->initSeqLen = seqLen; + this->accSeqLen = 0; + } + + // Embedding + this->embeddingForward(ids, this->embBuf->Data(), batchSize, seqLen); + this->accSeqLen += seqLen; + + // Prepare attention mask + this->prepareAttnMask(ids, step); + + // Token position ids, note: different models may have different impl. + int *positionIds = this->getPositionIds(ids, batchSize, seqLen, step); + t1.release(); + + // Decoder: forward + int hiddenSize = ctx->hiddenSize; + for (int i = 0; i < this->decoders.size(); ++i) { + int workers = this->messenger.getSize(); + KVCacheTensor &presentKey = this->kvCacheMgr->getKey(i); + KVCacheTensor &presentValue = this->kvCacheMgr->getValue(i); + + this->decoders[i]->forwardAttention(getContext(), this->embBuf->Data(), this->outBuf->Data(), attnMask, + presentKey, // presentKey, + presentValue, // presentValue, + seqLen, // inputSeqLen, + this->accSeqLen - seqLen, // pastSeqLen + step == 0, // useSelfAttn, + true, // doLnBefore, + false, // returnAttn, + false, // returnKVs + false, // forPT + positionIds); + + // Expand the KV cache as it only has values for beam 0 + if (step == 0 && beamSize > 1) { this->kvCacheMgr->expandCache(i, userSideBS, beamSize, seqLen); } + + auto &attnOut = this->getContext()->tmpBuf; + + // Merge the result of attention + // When attention and FFN/MLP are in parallel, do not need to reduce after attention + if constexpr (!ATTN_MLP_PARALLEL) { + if (this->messenger.getSize() > 1) { + this->messenger.reduceAdd(attnOut.Data(), attnOut.Data(), batchSize * seqLen * attnOut.Stride()); + } + } + + // When attention and FFN/MLP are in parallel, use the initial embedding as input + if constexpr (ATTN_MLP_PARALLEL) { + if (this->messenger.getSize() > 1) { + this->decoders[i]->forwardFFN( + getContext(), this->embBuf->Data(), this->outBuf->Data(), hiddenSize, hiddenSize, true); + this->messenger.reduceAdd( + this->outBuf->Data(), this->embBuf->Data(), batchSize * seqLen * hiddenSize); + } else { + this->decoders[i]->forwardFFN( + getContext(), this->embBuf->Data(), this->embBuf->Data(), hiddenSize, hiddenSize, true); + } + } else { + // FFN (for multiple workers, output into outBuf and then reduce add to embBuf) + if (this->messenger.getSize() > 1) { + this->decoders[i]->forwardFFN( + getContext(), attnOut.Data(), this->outBuf->Data(), attnOut.Stride(), hiddenSize, true); + this->messenger.reduceAdd( + this->outBuf->Data(), this->embBuf->Data(), batchSize * seqLen * hiddenSize); + } else { + this->decoders[i]->forwardFFN( + getContext(), attnOut.Data(), this->embBuf->Data(), attnOut.Stride(), hiddenSize, true); + } + } + } + + // Prepare input for final Layer Norm (only care about the last row of the result) + // Shape of embBuf: (bs, seqLen, hiddenSize) + float *lnIn = this->embBuf->Data(); + if (seqLen > 1 && !logitsAll) { // copy is not needed when seqLen = 1 or logitsAll is true + lnIn = this->outBuf->Data(); +#pragma omp parallel for + for (int b = 0; b < batchSize; ++b) { + memcpy(lnIn + b * hiddenSize, this->embBuf->Data() + ((b + 1) * seqLen - 1) * hiddenSize, + hiddenSize * sizeof(float)); + } + } + +#ifdef DEBUG + dbg.debugPrint("LayerNorm In:\n"); + dbg.dumpMatrix(lnIn, batchSize, hiddenSize, hiddenSize); +#endif + + // LN, as it supports inplace computing, input and output can be the same + float *lnOut = this->embBuf->Data(); + if (!logitsAll) + lastLayerNormForward(lnIn, lnOut, batchSize); + else + lastLayerNormForward(lnIn, lnOut, batchSize * seqLen); + +#ifdef DEBUG + dbg.debugPrint("LayerNorm Out:\n"); + dbg.dumpMatrix(lnOut, batchSize, hiddenSize, hiddenSize); +#endif + + // Predictor + if (!logitsAll) + this->predictor->forward(lnOut, this->outBuf->Data(), batchSize); + else + this->predictor->forward(lnOut, this->outBuf->Data(), batchSize * seqLen); + +#ifdef DEBUG + auto splitSize = this->predictor->getSplitSize(); + dbg.debugPrint("outBuf:\n"); + dbg.dumpMatrix(outBuf->Data(), batchSize, splitSize, splitSize); +#endif + + // Expand the result to make it cover multiple beams + if (step == 0 && beamSize > 1) { + const int splitSize = this->predictor->getSplitSize(); + for (int b = userSideBS - 1; b >= 0; --b) { + float *src = this->outBuf->Data() + b * splitSize; +#pragma omp parallel for + for (int idx = b * beamSize; idx < (b + 1) * beamSize; ++idx) { + if (idx == b) { continue; } + float *dst = this->outBuf->Data() + idx * splitSize; + memcpy(dst, src, splitSize * sizeof(float)); + } + } + } + + return std::tuple( + this->outBuf->Data(), this->predictor->getSplitOffset(), this->predictor->getSplitSize()); + } + + // Reorder cached keys and values, size=batchSize*beamSize + void reorderCache(int *idx, int size) { kvCacheMgr->reorderCache(idx, size, initSeqLen, accSeqLen); } + + // Get decoder context + DecoderContext *getContext() { return context.get(); } + + // How many layers + int getLayers() { return decoders.size(); } + + Messenger &getMessenger() { return messenger; } + + int getRank() { return messenger.getRank(); } + + WDataType getDataType() { return wType; } + + int getEndId() { return endId; } + + int getInitSeqLen() { return initSeqLen; } + + std::tuple, std::shared_ptr>, + std::shared_ptr>, std::shared_ptr>> + getSharedResources() { + return std::make_tuple(context, kvCacheMgr, embBuf, outBuf); + } + + void setSharedResources(const std::tuple, std::shared_ptr>, + std::shared_ptr>, std::shared_ptr>> &r) { + this->context = std::get<0>(r); + this->kvCacheMgr = std::get<1>(r); + this->embBuf = std::get<2>(r); + this->outBuf = std::get<3>(r); + } + + // When first step is skipped, call this function to make everything aligned + void skipFirstStep(int initSeqLen) { + // Reset initial and accumulated sequence length at the first step + this->initSeqLen = initSeqLen; + this->accSeqLen = initSeqLen; + } + +protected: + using DECODER = Decoder; + + static bool fileExists(const std::string &filename) { + std::ifstream file(filename); + return file.good(); + } + + DecoderContext *getDecoderContext(int layers, const int hiddenSize, const int attHeadNum, const int kvHeadNum, + const int imSize, const std::string &act, const float epsilon, int vocabSize, int embeddingSize, + int maxPositions) { + int splits = messenger.getSize(); + int splitIdx = messenger.getRank(); + + if (context != nullptr) { + if (context->hiddenSize == hiddenSize && context->attHeadNum == attHeadNum + && context->kvHeadNum == kvHeadNum && context->intermediateSize == imSize + && context->splitIdx == splitIdx) { + return context.get(); + } else { + printf("Different context size not unsupported!\n"); + exit(-1); + } + } else { + this->context.reset(new DecoderContext(layers, hiddenSize, attHeadNum, kvHeadNum, imSize, act, epsilon, + vocabSize, embeddingSize, maxPositions, splitIdx, splits)); + } + + return this->context.get(); + } + + void setDecoderWeights(DECODER *pdecoder, const std::string &modelPath, int layerIdx) { + const int hiddenSize = getContext()->hiddenSize; + const int imSize = getContext()->intermediateSize; + const int kvHeadNum = getContext()->kvHeadNum; + const int attHeadSize = getContext()->attHeadSize; + const int mlpFactor = (getContext()->actType == DecoderContext::SWIGLU) ? 2 : 1; + int qSize = hiddenSize; + int kvSize = attHeadSize * kvHeadNum; + int qkvSize = qSize + kvSize + kvSize; + +#define ALLOC(size, alignment) aligned_alloc((alignment), (size)) + float *qkvWeight = (float *)ALLOC(hiddenSize * qkvSize * sizeof(float), 64); + float *qkvBias = (float *)ALLOC(qkvSize * sizeof(float), 64); + float *attnOutWeight = (float *)ALLOC(hiddenSize * hiddenSize * sizeof(float), 64); + float *attnOutBias = (float *)ALLOC(hiddenSize * sizeof(float), 64); + float *fc1Weight = (float *)ALLOC(hiddenSize * imSize * mlpFactor * sizeof(float), 64); + float *fc1Bias = (float *)ALLOC(imSize * sizeof(float), 64); + float *fc2Weight = (float *)ALLOC(hiddenSize * imSize * sizeof(float), 64); + float *fc2Bias = (float *)ALLOC(hiddenSize * sizeof(float), 64); + float *ln1Gamma = (float *)ALLOC(hiddenSize * sizeof(float), 64); + float *ln1Beta = (float *)ALLOC(hiddenSize * sizeof(float), 64); + float *ln2Gamma = (float *)ALLOC(hiddenSize * sizeof(float), 64); + float *ln2Beta = (float *)ALLOC(hiddenSize * sizeof(float), 64); + float *fc3Weight = nullptr; + + // printf("hiddenSize=%d, qkvSize=%d\n", hiddenSize, qkvSize); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".attention.query_key_value.weight.0.bin", + qkvWeight, hiddenSize * qkvSize, getDataType()); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".attention.dense.weight.0.bin", + attnOutWeight, hiddenSize * hiddenSize, getDataType()); + + // Stardard 2 layer MLP + if (fileExists(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.dense_h_to_4h.weight.0.bin")) { + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.dense_h_to_4h.weight.0.bin", + fc1Weight, hiddenSize * imSize * mlpFactor, getDataType()); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.dense_4h_to_h.weight.0.bin", + fc2Weight, hiddenSize * imSize, getDataType()); + } + // gate, up, down weights for Llama like model + else { + fc3Weight = (float *)ALLOC(hiddenSize * imSize * sizeof(float), 64); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.gate_proj.weight.0.bin", + fc1Weight, hiddenSize * imSize * mlpFactor, getDataType()); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.up_proj.weight.0.bin", fc2Weight, + hiddenSize * imSize, getDataType()); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.down_proj.weight.0.bin", + fc3Weight, hiddenSize * imSize, getDataType()); + } + + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".input_layernorm.weight.bin", ln1Gamma, + hiddenSize, getDataType()); + loadWeight(modelPath + "/model.layers." + std::to_string(layerIdx) + ".post_attention_layernorm.weight.bin", + ln2Gamma, hiddenSize, getDataType()); + +#define READ_OPTIONAL(filename, addr, size, errmsg) \ + { \ + int ret = loadWeight((filename), (addr), (size), getDataType(), false); \ + if (ret == 0) { \ + free(addr); \ + addr = nullptr; \ + } else { \ + if (ret != (size)) { \ + printf("%s\n", (errmsg)); \ + exit(-1); \ + } \ + } \ + } + + // The bias is optional + READ_OPTIONAL(modelPath + "/model.layers." + std::to_string(layerIdx) + ".attention.query_key_value.bias.0.bin", + qkvBias, qkvSize, "read QKV bias error"); + READ_OPTIONAL(modelPath + "/model.layers." + std::to_string(layerIdx) + ".attention.dense.bias.bin", + attnOutBias, hiddenSize, "read attn dense bias error"); + READ_OPTIONAL(modelPath + "/model.layers." + std::to_string(layerIdx) + ".input_layernorm.bias.bin", ln1Beta, + hiddenSize, "read LN1 beta error"); + READ_OPTIONAL(modelPath + "/model.layers." + std::to_string(layerIdx) + ".post_attention_layernorm.bias.bin", + ln2Beta, hiddenSize, "read LN2 beta error"); + READ_OPTIONAL(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.dense_h_to_4h.bias.0.bin", + fc1Bias, imSize, "read FC1 bias error"); + READ_OPTIONAL(modelPath + "/model.layers." + std::to_string(layerIdx) + ".mlp.dense_4h_to_h.bias.bin", fc2Bias, + hiddenSize, "read FC2 bias error"); + +#define FREE(x) \ + if ((x)) free((x)) + // Need the tranposed weights in our interface + // ordering, trans, rows, cols, alpha, a, lda, b, ldb + + std::vector params = {qkvWeight, qkvBias, qkvWeight + qSize, qkvBias + qSize, + qkvWeight + qSize + kvSize, qkvBias + qSize + kvSize, attnOutWeight, attnOutBias, ln1Gamma, ln1Beta, + fc1Weight, fc1Bias, fc2Weight, fc2Bias, ln2Gamma, ln2Beta, fc3Weight}; + pdecoder->setWeights(getContext(), params, false); + FREE(qkvWeight); + FREE(attnOutWeight); + FREE(fc1Weight); + FREE(fc2Weight); + FREE(fc3Weight); + FREE(qkvBias); + FREE(attnOutBias); + FREE(fc1Bias); + FREE(fc2Bias); + FREE(ln1Gamma); + FREE(ln1Beta); + FREE(ln2Gamma); + FREE(ln2Beta); + } + + void setPredictorWeight(const std::string &modelPath) { + int inputSize = predictor->getInputSize(); + int outputSize = predictor->getOutputSize(); + + float *weight = (float *)malloc(inputSize * outputSize * sizeof(float)); + float *bias = nullptr; + + loadWeight(modelPath + "/model.lm_head.weight.bin", weight, inputSize * outputSize, this->getDataType()); + + predictor->setWeight(weight, bias); + + free(weight); + } + + virtual void prepareBuffers(DecoderContext *ctx, int userSideBS, int beamSize, bool logitsAll = false) { + int batchSize = ctx->batchSize; + int hiddenSize = ctx->hiddenSize; + int seqLen = ctx->inputSeqLen; + int vocabSize = ctx->vocabSize; + int maxPositions = ctx->maxPositions; + int layers = this->decoders.size(); + int workers = this->messenger.getSize(); + + // Prepare buffers (embBuf & outBuf), userSideBS * beamSize is the output rows really needed + int logitsLen = logitsAll ? batchSize * seqLen : userSideBS * beamSize; + int requiredRows = batchSize * seqLen; + + // The required output buffer size is bigger than the embedding size + if (logitsLen * vocabSize > batchSize * seqLen * hiddenSize) { + requiredRows = logitsLen * vocabSize / hiddenSize + 1; + } + if (requiredRows > this->embBuf->Rows()) { + this->embBuf->Resize(requiredRows, hiddenSize); + this->outBuf->Resize(requiredRows, hiddenSize); + } + + // Attention mask + int sizeRequired = batchSize * seqLen * seqLen; + getAttnMask(sizeRequired); + + // Cached keys/values + // The maximum sequence length is to be the same as maxPositions, at most + // And the cache always needs to account for beam size + int headsPerSplit = (ctx->kvHeadNum + workers - 1) / workers; + this->kvCacheMgr->resize(maxPositions, userSideBS * beamSize, headsPerSplit, ctx->attHeadSize); + } + + float *getAttnMask(int sizeRequired) { + if (this->maskSize < sizeRequired) { + if (this->attnMask) free(this->attnMask); + this->attnMask = (float *)aligned_alloc(64, sizeRequired * sizeof(float)); + this->maskSize = sizeRequired; + } + return this->attnMask; + } + + int getStartId() { return startId; } + + virtual void embeddingForward(int *ids, float *output, int batchSize, int seqLen) = 0; + virtual void lastLayerNormForward(float *input, float *output, int rows) = 0; + virtual void prepareAttnMask(int *ids, int step) = 0; + +public: + virtual int *getPositionIds(int *ids, int batchSize, int seqLen, int step) { return nullptr; } + +protected: + // For communication + Messenger &messenger; + + // Execution context + std::shared_ptr context; + + // The initial input sequence length, which is the prompt token size + int initSeqLen; + // Accumulated sequence length, = past_seq_len + current_seq_len + int accSeqLen; + + // If not the master, need to receive token IDs from the master + int *inputTokens; + + std::shared_ptr> kvCacheMgr; + + std::shared_ptr> embBuf; // used to store the embedding result + std::shared_ptr> outBuf; // output buffer for decoder layers, same size as embBuf + +protected: + // Components most LLMs may use + std::vector decoders; + DistLinear *predictor; + +private: + int maskSize; // size of allocated attnMask + float *attnMask; // attention mask, set as private as may need to enlarge + + int startId; + int endId; + + WDataType wType; +#ifdef DEBUG + Debugger dbg; +#endif +}; \ No newline at end of file diff --git a/src/models/hybrid_model.h b/src/models/hybrid_model.h new file mode 100644 index 0000000..a43dfe0 --- /dev/null +++ b/src/models/hybrid_model.h @@ -0,0 +1,75 @@ +#pragma once +#include +#include "abstract_decoder.h" +#include "numa_allocator.h" +#include + +template