From 0df0a4462e4666fe0799d9a0a91028fd7a911c21 Mon Sep 17 00:00:00 2001 From: "hongliang.yuan" Date: Mon, 24 Mar 2025 13:37:26 +0800 Subject: [PATCH] update megatron deepspeed code --- nlp/llm/aquila2-34b/pytorch/README.md | 1 - nlp/llm/llama2-13b/pytorch/README.md | 1 - nlp/llm/llama2-34b/pytorch/README.md | 1 - nlp/llm/llama2-7b/pytorch/README.md | 7 - nlp/llm/llama2-7b_rlhf/pytorch/README.md | 14 +- nlp/llm/llama2-7b_sft/pytorch/README.md | 7 - toolbox/Megatron-DeepSpeed/.coveragerc | 5 - toolbox/Megatron-DeepSpeed/.gitignore | 37 - toolbox/Megatron-DeepSpeed/.gitlab-ci.yml | 302 -- toolbox/Megatron-DeepSpeed/CODEOWNERS | 1 - toolbox/Megatron-DeepSpeed/ILUVATAR.md | 99 - toolbox/Megatron-DeepSpeed/LICENSE | 376 -- toolbox/Megatron-DeepSpeed/MANIFEST.in | 5 - toolbox/Megatron-DeepSpeed/README.md | 530 --- toolbox/Megatron-DeepSpeed/README_RLHF.md | 31 - toolbox/Megatron-DeepSpeed/SECURITY.md | 41 - .../build_megatron-deepspeed.sh | 25 - .../ci/run_ci_tests_multi_node.sh | 16 - .../ci/run_ci_tests_one_node.sh | 14 - .../clean_megatron-deepspeed.sh | 8 - toolbox/Megatron-DeepSpeed/dataset/README.md | 5 - .../dataset/convert_dahoas.sh | 27 - .../convert_llama2tokenizer_dataset.sh | 21 - .../download_RedPajama-Data-1T-Sample.sh | 10 - .../download_and_covert_llama3_dataset.sh | 25 - .../dataset/download_books.sh | 2 - .../dataset/download_ckpt.sh | 8 - .../dataset/download_vocab.sh | 2 - .../docs/distrib_optimizer.md | 54 - .../images/distrib_optimizer/data_flow.png | Bin 90014 -> 0 bytes .../distrib_optimizer/sharding_scheme.png | Bin 99135 -> 0 bytes toolbox/Megatron-DeepSpeed/examples/README.md | 3 - .../examples/detxoify_lm/README.md | 112 - .../annotations/filter-selfgeneration.py | 75 - .../annotations/perspective_api_annotate.py | 182 - .../detxoify_lm/annotations/preprocess.sh | 14 - .../examples/detxoify_lm/finetune_gpt.py | 149 - .../finetune_gpt_distributed-1.3b.sh | 64 - .../examples/detxoify_lm/generate-1.3b.sh | 41 - .../detxoify_lm/generate_samples_gpt.py | 202 -- .../examples/detxoify_lm/perspective_api.py | 170 - .../selfgenerate-1.3b-unconditional.sh | 42 - .../examples/evaluate_retriever_nq.sh | 38 - .../examples/evaluate_zeroshot_gpt.sh | 38 - .../examples/finetune_mnli_distributed.sh | 44 - .../examples/finetune_race_distributed.sh | 47 - .../finetune_retriever_distributed.sh | 56 - .../llama2/run_ixte_llama2_34b_node4.sh | 179 - .../llama2/run_load_weight_llama2_7b.sh | 138 - .../llama2/run_load_weight_tinyllama_1.1b.sh | 141 - .../llama2/run_te_llama2_34b_node4.sh | 181 - ...te_llama2_34b_tpoverlap_profiling_node1.sh | 165 - .../llama2/run_te_llama2_70b_node4.sh | 182 - ...te_llama2_70b_tpoverlap_profiling_node1.sh | 162 - .../examples/llama2/run_te_llama2_7b_node1.sh | 152 - ..._te_llama2_7b_tpoverlap_profiling_node1.sh | 159 - .../run_te_torch_pp_overlap_node1_card8.sh | 176 - .../run_te_torch_tp_overlap_node1_card2.sh | 171 - .../examples/llama3/run_te_llama3_8b_node1.sh | 144 - .../examples/merge_mp_bert.sh | 18 - .../examples/msdp/README.md | 5 - .../examples/msdp/data_processing.sh | 83 - .../examples/msdp/eval_knwl_generation.sh | 43 - .../examples/msdp/eval_resp_generation.sh | 64 - .../examples/msdp/prep_resp_gen.sh | 18 - .../examples/msdp/prompt_knwl_gen.sh | 46 - .../examples/msdp/prompt_resp_gen.sh | 46 - .../examples/pretrain_bert.sh | 47 - .../examples/pretrain_bert_distributed.sh | 64 - .../pretrain_bert_distributed_with_mp.sh | 66 - .../examples/pretrain_gpt.sh | 51 - .../examples/pretrain_gpt3_175B.sh | 65 - .../examples/pretrain_gpt_distributed.sh | 68 - .../pretrain_gpt_distributed_with_mp.sh | 72 - .../examples/pretrain_ict.sh | 44 - .../examples/pretrain_t5.sh | 51 - .../examples/pretrain_t5_distributed.sh | 68 - .../pretrain_t5_distributed_with_mp.sh | 69 - .../run_text_generation_server_345M.sh | 34 - ...eneration_server_345M_8_tensor_parallel.sh | 32 - .../examples/sc21/CONFIG.sh | 57 - .../examples/sc21/README.md | 45 - .../examples/sc21/SBATCH.sh | 13 - .../Megatron-DeepSpeed/examples/sc21/SRUN.sh | 18 - .../examples/sc21/run_figure_11.sh | 46 - .../examples/sc21/run_figure_12.sh | 54 - .../examples/sc21/run_figure_13.sh | 46 - .../examples/sc21/run_figure_14.sh | 47 - .../examples/sc21/run_figure_15.sh | 47 - .../examples/sc21/run_figure_16.sh | 43 - .../examples/sc21/run_figure_17.sh | 54 - .../examples/sc21/run_figure_18.sh | 54 - .../examples/sc21/run_table_1.sh | 145 - .../MoE/ds_config_gpt_TEMPLATE.json | 38 - .../MoE/ds_config_gpt_Zero2_TEMPLATE.json | 38 - .../examples_deepspeed/MoE/ds_evalharness.sh | 72 - .../MoE/ds_pretrain_gpt_1.3B_MoE128.sh | 348 -- .../MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128.sh | 340 -- .../ds_pretrain_gpt_1.3B_PR-MoE64or128_MoS.sh | 354 -- .../MoE/ds_pretrain_gpt_1.3B_dense.sh | 349 -- .../MoE/ds_pretrain_gpt_1.3B_dense_cl.sh | 285 -- .../MoE/ds_pretrain_gpt_125M_MoE64.sh | 372 -- .../MoE/ds_pretrain_gpt_125M_dense_cl.sh | 309 -- .../MoE/ds_pretrain_gpt_350M_MoE128.sh | 348 -- .../MoE/ds_pretrain_gpt_350M_PR-MoE32or64.sh | 341 -- .../ds_pretrain_gpt_350M_PR-MoE32or64_MoS.sh | 353 -- .../MoE/ds_pretrain_gpt_350M_dense.sh | 348 -- .../MoE/ds_pretrain_gpt_6.7B_dense.sh | 349 -- .../MoE/readme_evalharness.md | 168 - .../examples_deepspeed/README.md | 33 - .../examples_deepspeed/azure/README.md | 27 - .../examples_deepspeed/azure/run-175b.sh | 142 - .../examples_deepspeed/azure/run-1t.sh | 154 - .../azure/run-benchmark-model.sh | 142 - .../azureml/Dockerfile.dockerfile | 5 - .../examples_deepspeed/azureml/README.md | 16 - .../examples_deepspeed/azureml/aml_submit.py | 198 - .../azureml/prepare_dataset.py | 33 - .../bert_with_pile/README.md | 23 - .../ds_config_bert_TEMPLATE.json | 27 - .../bert_with_pile/ds_finetune_bert_mnli.sh | 150 - .../bert_with_pile/ds_finetune_bert_qqp.sh | 158 - .../bert_with_pile/ds_finetune_bert_race.sh | 172 - .../bert_with_pile/ds_pretrain_bert.sh | 267 -- .../bert_with_pile/prepare_pile_data.py | 128 - .../125M-Int8-test-64gpu-distilled-group48.sh | 253 -- ...M-L10-Int8-test-64gpu-distilled-group48.sh | 253 -- ...M-L12-Int8-test-64gpu-distilled-group48.sh | 253 -- .../compression/ds_config_gpt_TEMPLATE.json | 38 - .../ds_config_gpt_TEMPLATE_compression.json | 86 - .../compression/ds_evalharness.sh | 75 - .../ds_pretrain_gpt_1.3B_dense_cl_kd.sh | 322 -- .../ds_pretrain_gpt_125M_dense_cl_kd.sh | 323 -- .../ds_pretrain_gpt_125M_dense_kd.sh | 323 -- .../ds_pretrain_gpt_350M_dense_kd.sh | 348 -- .../curriculum_learning/README.md | 1 - .../ds_config_gpt_slw_TEMPLATE.json | 34 - .../curriculum_learning/ds_pretrain_gpt2.sh | 150 - .../ds_pretrain_gpt_1.3B_rope_slw.sh | 347 -- .../curriculum_learning/ds_train.sh | 37 - .../ds_zero_stage_1_config_baseline.json | 26 - ...tage_1_config_curriculum_fixed_linear.json | 37 - .../data_efficiency/README.md | 23 - .../data_efficiency/analyze_data.py | 239 -- .../bert/ds_analyze_bert_data_map.sh | 67 - .../bert/ds_analyze_bert_data_reduce.sh | 66 - .../finetune/ds_config_bert_TEMPLATE.json | 23 - .../bert/finetune/ds_finetune_bert_mnli.sh | 150 - .../bert/finetune/ds_finetune_bert_qqp.sh | 158 - .../bert/finetune/ds_finetune_bert_race.sh | 172 - .../finetune/ds_finetune_gather_result.py | 111 - .../ds_config_bert_TEMPLATE.json | 23 - .../finetune_glue/ds_finetune_bert_glue.sh | 156 - .../ds_finetune_bert_glue_run.sh | 44 - .../ds_finetune_gather_result.py | 118 - .../bert/pile_data_download_preprocess.py | 129 - .../ds_config_bert_1clmetric_TEMPLATE.json | 73 - .../ds_config_bert_2clmetrics_TEMPLATE.json | 87 - .../ds_pretrain_bert_336M_base_script.sh | 472 --- .../pretrain/ds_pretrain_bert_336M_run.sh | 363 -- .../gpt/ds_analyze_gpt_data_map.sh | 70 - .../gpt/ds_analyze_gpt_data_reduce.sh | 69 - .../gpt/eval/ds_config_eval_dummy.json | 27 - .../gpt/eval/ds_evalharness_1gpu.sh | 78 - .../gpt/eval/ds_evalharness_gather_result.py | 358 -- .../gpt/eval/ds_evalharness_parallel_run.sh | 67 - .../ds_evalharness_parallel_run_10shot.sh | 62 - .../ds_config_gpt_1clmetric_TEMPLATE.json | 73 - .../ds_config_gpt_2clmetrics_TEMPLATE.json | 87 - .../ds_pretrain_gpt_1.3B_dense_base_script.sh | 515 --- .../ds_pretrain_gpt_1.3B_dense_run.sh | 366 -- .../megatron_long_seq_support/README.md | 107 - .../ds_config_gpt_TEMPLATE.json | 32 - .../megatron_long_seq_support/host_file | 1 - .../pretrain_gpt_1.3B_seq_parallel.sh | 349 -- .../pretrain_gpt_30B_seq_parallel.sh | 360 -- .../finetune_hf_llama/README.md | 24 - .../finetune_hf_llama/ds_config.json | 11 - .../finetune_hf_llama/finetune_llama.sh | 110 - .../examples_deepspeed/generate_text.sh | 51 - .../examples_deepspeed/offload_pp/README.md | 81 - .../offload_pp/ds_config_gpt_TEMPLATE.json | 32 - .../offload_pp/ds_pretrain_gpt_350M.sh | 316 -- .../offload_pp/twin-offload.png | Bin 59949 -> 0 bytes .../pretrain_llama2_distributed.sh | 135 - .../pretrain_llama_distributed.sh | 132 - .../examples_deepspeed/rebase/README.md | 47 - .../rebase/ds_config_gpt_TEMPLATE.json | 23 - .../rebase/ds_config_gpt_slw_TEMPLATE.json | 34 - .../rebase/ds_pretrain_gpt_1.3B.sh | 332 -- ...retrain_gpt_1.3B_megatron_checkpointing.sh | 345 -- .../rebase/ds_pretrain_gpt_1.3B_rope.sh | 334 -- .../rebase/ds_pretrain_gpt_1.3B_rope_slw.sh | 347 -- .../rebase/ds_pretrain_gpt_125M.sh | 331 -- .../rebase/ds_pretrain_gpt_125M_flashattn.sh | 332 -- .../rebase/ds_pretrain_gpt_13B.sh | 332 -- .../run_deepspeed_example.sh | 84 - .../sequence_parallel/README.md | 36 - .../ds_config_gpt_TEMPLATE.json | 23 - .../ds_pretrain_gpt_1.3B_seq_parallel_32k.sh | 341 -- .../ds_pretrain_gpt_30B_seq_parallel_32k.sh | 351 -- .../universal_checkpointing/README.md | 119 - .../assets/image/uc_char_training_loss.png | Bin 54558 -> 0 bytes .../assets/image/uc_char_validation_loss.png | Bin 42352 -> 0 bytes .../universal_checkpointing/ds_config.json | 19 - .../universal_checkpointing/run_bf16.sh | 157 - .../universal_checkpointing/run_fp16.sh | 163 - .../run_tb_analysis.sh | 29 - .../run_universal_bf16.sh | 157 - .../run_universal_fp16.sh | 163 - .../tb_analysis/abstract_analysis.py | 31 - .../tb_analysis/arguments.py | 19 - .../tb_analysis/tb_analysis_script.py | 52 - .../tb_analysis/uc_analysis.py | 31 - .../tb_analysis/utils.py | 32 - toolbox/Megatron-DeepSpeed/finetune_llama.py | 351 -- .../images/Achieved_petaFLOPs.png | Bin 229267 -> 0 bytes .../images/cases_april2021.png | Bin 163078 -> 0 bytes .../install_megatron-deepspeed.sh | 38 - .../megatron_ds/__init__.py | 22 - .../megatron_ds/checkpointing.py | 759 ---- .../megatron_ds/core/__init__.py | 18 - .../megatron_ds/core/datasets/Makefile | 9 - .../core/datasets/blended_dataset.py | 190 - .../blended_megatron_dataset_builder.py | 335 -- .../blended_megatron_dataset_config.py | 119 - .../megatron_ds/core/datasets/gpt_dataset.py | 460 --- .../megatron_ds/core/datasets/helpers.cpp | 765 ---- .../core/datasets/indexed_dataset.py | 639 ---- .../core/datasets/megatron_dataset.py | 135 - .../megatron_ds/core/datasets/readme.md | 193 - .../megatron_ds/core/datasets/utils.py | 60 - .../core/dist_checkpointing/__init__.py | 11 - .../core/dist_checkpointing/core.py | 41 - .../core/dist_checkpointing/dict_utils.py | 219 -- .../core/dist_checkpointing/mapping.py | 308 -- .../core/dist_checkpointing/optimizer.py | 90 - .../core/dist_checkpointing/serialization.py | 385 -- .../dist_checkpointing/strategies/__init__.py | 16 - .../dist_checkpointing/strategies/base.py | 90 - .../strategies/tensorstore.py | 131 - .../strategies/two_stage.py | 256 -- .../dist_checkpointing/strategies/zarr.py | 285 -- .../core/dist_checkpointing/utils.py | 44 - .../megatron_ds/core/distributed/__init__.py | 2 - .../distributed/distributed_data_parallel.py | 248 -- .../core/distributed/finalize_model_grads.py | 158 - .../core/distributed/grad_buffer.py | 410 --- .../megatron_ds/core/enums.py | 10 - .../core/fusions/fused_bias_dropout.py | 71 - .../core/fusions/fused_bias_gelu.py | 48 - .../core/fusions/fused_layer_norm.py | 151 - .../megatron_ds/core/fusions/fused_softmax.py | 204 -- .../megatron_ds/core/inference_params.py | 27 - .../megatron_ds/core/model_parallel_config.py | 224 -- .../megatron_ds/core/models/T5/__init__.py | 1 - .../megatron_ds/core/models/T5/t5_model.py | 466 --- .../megatron_ds/core/models/T5/t5_spec.py | 212 -- .../core/models/bert/bert_layer_specs.py | 64 - .../core/models/bert/bert_lm_head.py | 72 - .../core/models/bert/bert_model.py | 234 -- .../megatron_ds/core/models/bert/pooler.py | 51 - .../embeddings/language_model_embedding.py | 163 - .../common/embeddings/rotary_pos_embedding.py | 167 - .../common/language_module/language_module.py | 98 - .../megatron_ds/core/models/gpt/__init__.py | 1 - .../core/models/gpt/gpt_embedding.py | 114 - .../core/models/gpt/gpt_layer_specs.py | 123 - .../megatron_ds/core/models/gpt/gpt_model.py | 241 -- .../megatron_ds/core/models/retro/__init__.py | 5 - .../core/models/retro/base_attention.py | 45 - .../megatron_ds/core/models/retro/config.py | 43 - .../core/models/retro/decoder_attention.py | 301 -- .../core/models/retro/decoder_spec.py | 152 - .../core/models/retro/encoder_attention.py | 223 -- .../core/models/retro/encoder_spec.py | 141 - .../megatron_ds/core/models/retro/model.py | 89 - .../megatron_ds/core/parallel_state.py | 1134 ------ .../core/pipeline_parallel/__init__.py | 1 - .../pipeline_parallel/p2p_communication.py | 598 --- .../core/pipeline_parallel/schedules.py | 1307 ------- .../core/tensor_parallel/__init__.py | 66 - .../core/tensor_parallel/cross_entropy.py | 142 - .../core/tensor_parallel/layers.py | 995 ----- .../core/tensor_parallel/mappings.py | 359 -- .../core/tensor_parallel/random.py | 288 -- .../megatron_ds/core/tensor_parallel/utils.py | 118 - .../megatron_ds/core/transformer/__init__.py | 6 - .../megatron_ds/core/transformer/attention.py | 443 --- .../custom_layers/transformer_engine.py | 431 --- .../core/transformer/dot_product_attention.py | 195 - .../megatron_ds/core/transformer/enums.py | 26 - .../core/transformer/identity_op.py | 28 - .../megatron_ds/core/transformer/mlp.py | 184 - .../megatron_ds/core/transformer/module.py | 157 - .../core/transformer/spec_utils.py | 109 - .../core/transformer/switch_mlp.py | 158 - .../core/transformer/transformer_block.py | 349 -- .../core/transformer/transformer_config.py | 288 -- .../core/transformer/transformer_layer.py | 245 -- .../megatron_ds/core/transformer/utils.py | 148 - .../megatron_ds/core/utils.py | 236 -- .../megatron_ds/dist_signal_handler.py | 81 - .../Megatron-DeepSpeed/megatron_ds/enums.py | 34 - .../fp16_deprecated/loss_scaler.py | 26 - .../megatron_ds/fused_kernels/__init__.py | 75 - .../megatron_ds/fused_kernels/compat.h | 17 - .../fused_kernels/tests/test_fused_kernels.py | 388 -- .../megatron_ds/fused_kernels/type_shim.h | 103 - .../megatron_ds/global_vars.py | 234 -- .../Megatron-DeepSpeed/megatron_ds/indexer.py | 129 - .../megatron_ds/microbatches.py | 144 - .../megatron_ds/model/__init__.py | 12 - .../megatron_ds/model/biencoder_model.py | 328 -- .../megatron_ds/model/enums.py | 21 - .../megatron_ds/model/fused_bias_gelu.py | 43 - .../megatron_ds/model/fused_layer_norm.py | 177 - .../megatron_ds/model/fused_softmax.py | 213 -- .../megatron_ds/model/language_model.py | 699 ---- .../megatron_ds/model/module.py | 199 - .../megatron_ds/model/realm_model.py | 204 -- .../megatron_ds/model/rms_norm.py | 56 - .../megatron_ds/model/transformer.py | 2091 ----------- .../model/vision/classification.py | 86 - .../megatron_ds/model/vision/dino.py | 291 -- .../model/vision/esvit_swin_backbone.py | 849 ----- .../megatron_ds/model/vision/inpainting.py | 152 - .../megatron_ds/model/vision/knn_monitor.py | 129 - .../megatron_ds/model/vision/mit_backbone.py | 415 --- .../megatron_ds/model/vision/swin_backbone.py | 625 ---- .../megatron_ds/model/vision/utils.py | 27 - .../megatron_ds/model/vision/vit_backbone.py | 248 -- .../megatron_ds/mpu/tests/__init__.py | 0 .../megatron_ds/mpu/tests/commons.py | 70 - .../mpu/tests/test_cross_entropy.py | 95 - .../megatron_ds/mpu/tests/test_data.py | 75 - .../megatron_ds/mpu/tests/test_initialize.py | 82 - .../megatron_ds/mpu/tests/test_layers.py | 517 --- .../megatron_ds/mpu/tests/test_random.py | 191 - .../megatron_ds/optimizer/__init__.py | 171 - .../megatron_ds/optimizer/clip_grads.py | 148 - .../optimizer/distrib_optimizer.py | 1162 ------ .../megatron_ds/optimizer/grad_scaler.py | 120 - .../megatron_ds/optimizer/optimizer.py | 644 ---- .../megatron_ds/optimizer/utils.py | 19 - .../megatron_ds/optimizer_param_scheduler.py | 235 -- .../megatron_ds/rlhf/__init__.py | 0 .../megatron_ds/rlhf/generation/__init__.py | 0 .../megatron_ds/static/index.html | 124 - .../megatron_ds/text_generation/__init__.py | 7 - .../megatron_ds/text_generation/api.py | 207 -- .../megatron_ds/text_generation/beam_utils.py | 64 - .../text_generation/communication.py | 185 - .../text_generation/forward_step.py | 177 - .../megatron_ds/text_generation/generation.py | 428 --- .../megatron_ds/text_generation/sampling.py | 93 - .../text_generation/tokenization.py | 125 - .../megatron_ds/text_generation_server.py | 241 -- .../Megatron-DeepSpeed/megatron_ds/timers.py | 309 -- .../datasets => megatronspeed}/__init__.py | 0 .../core/README.md | 0 .../core}/__init__.py | 0 .../megatronspeed/core/optimizer/__init__.py | 264 ++ .../core/package_info.py | 4 +- .../megatronspeed/core/parallel_state.py | 502 +++ .../core/pipeline_parallel}/__init__.py | 0 .../deepspeed_zbh1_engine.py | 110 + .../deepspeed_zbh1_schedule.py | 148 + .../core/pipeline_parallel/schedules.py | 486 +++ .../core/requirements.txt | 0 .../core/sequence_parallel/__init__.py | 0 .../core/sequence_parallel/cross_entropy.py | 2 +- .../core/tensor_parallel}/__init__.py | 0 .../core/tensor_parallel/data.py | 54 +- .../core/tensor_parallel/layers.py | 1540 ++++++++ .../core/tensor_parallel/random.py | 315 ++ .../core/tensor_parallel/weight_grad_store.py | 34 + .../core/transformer}/__init__.py | 0 .../megatronspeed/core/transformer/utils.py | 16 + .../megatronspeed/core/utils.py | 8 + .../legacy}/__init__.py | 0 .../legacy}/data/Makefile | 0 .../legacy/data}/__init__.py | 0 .../legacy}/data/autoaugment.py | 0 .../legacy}/data/bert_dataset.py | 4 +- .../legacy}/data/biencoder_dataset_utils.py | 10 +- .../legacy}/data/blendable_dataset.py | 9 +- .../legacy}/data/data_samplers.py | 15 +- .../legacy}/data/dataset_utils.py | 16 +- .../legacy}/data/gpt_dataset.py | 19 +- .../legacy}/data/helpers.cpp | 0 .../legacy}/data/ict_dataset.py | 8 +- .../legacy}/data/image_folder.py | 0 .../legacy}/data/indexed_dataset.py | 2 +- .../legacy}/data/multimodal_dataset.py | 0 .../legacy}/data/orqa_wiki_dataset.py | 6 +- .../legacy}/data/realm_dataset_utils.py | 12 +- .../legacy}/data/realm_index.py | 4 +- .../legacy}/data/t5_dataset.py | 4 +- .../legacy}/data/test/test_indexed_dataset.py | 4 +- .../legacy}/data/test/test_preprocess_data.sh | 0 .../legacy}/data/vit_dataset.py | 8 +- .../legacy/model}/__init__.py | 0 .../legacy}/model/bert_model.py | 112 +- .../legacy/model/biencoder_model.py | 103 + .../legacy}/model/classification.py | 25 +- .../legacy}/model/distributed.py | 6 +- .../legacy}/model/gpt_model.py | 679 ++-- .../legacy/model/language_model.py | 1205 +++++++ .../megatronspeed/legacy/model/module.py | 11 + .../legacy}/model/multiple_choice.py | 25 +- .../megatronspeed/legacy/model/realm_model.py | 72 + .../megatronspeed/legacy/model/rms_norm.py | 16 + .../legacy}/model/rotary_pos_embedding.py | 34 +- .../legacy}/model/t5_model.py | 58 +- .../megatronspeed/legacy/model/transformer.py | 3205 +++++++++++++++++ .../legacy}/model/utils.py | 70 +- .../log_handler.py | 0 .../megatronspeed/megatron_adaptor.py | 319 ++ .../p2p_communication.py | 6 +- .../megatronspeed/patch_utils.py | 118 + .../data => megatronspeed/rlhf}/__init__.py | 0 .../rlhf/checkpointing_rlhf.py | 6 +- .../rlhf/generation}/__init__.py | 0 .../rlhf/generation/communication_rlhf.py | 2 +- .../rlhf/generation/forward_rlhf.py | 6 +- .../rlhf/generation/generation_rlhf.py | 8 +- .../rlhf/initialize_rlhf.py | 23 +- .../rlhf/schedules_rlhf.py | 10 +- .../rlhf/training_rlhf.py | 484 ++- .../text_generation_utils.py | 16 +- .../theoretical_memory_usage.py | 0 .../megatronspeed/training/__init__.py | 1 + .../training}/arguments.py | 2743 ++++++-------- .../megatronspeed/training/checkpointing.py | 626 ++++ .../megatronspeed/training/global_vars.py | 16 + .../training}/initialize.py | 224 +- .../training}/memory.py | 1 - .../training}/tokenizer/__init__.py | 0 .../training}/tokenizer/bert_tokenization.py | 0 .../training}/tokenizer/gpt2_tokenization.py | 0 .../training}/tokenizer/tokenization_utils.py | 0 .../training}/tokenizer/tokenizer.py | 525 +-- .../training}/training.py | 859 ++--- .../training}/utils.py | 213 +- toolbox/Megatron-DeepSpeed/pretrain_bert.py | 192 - toolbox/Megatron-DeepSpeed/pretrain_ict.py | 166 - toolbox/Megatron-DeepSpeed/pretrain_retro.py | 244 -- toolbox/Megatron-DeepSpeed/pretrain_t5.py | 263 -- .../pretrain_vision_classify.py | 105 - .../pretrain_vision_dino.py | 105 - .../pretrain_vision_inpaint.py | 141 - .../report_theoretical_memory.py | 14 - .../Megatron-DeepSpeed/requirments_rlhf.txt | 3 - toolbox/Megatron-DeepSpeed/setup.py | 114 - .../Megatron-DeepSpeed/tasks/data_utils.py | 105 - .../tasks/ensemble_classifier.py | 149 - .../tasks/eval_harness/download.py | 26 - .../tasks/eval_harness/evaluate.py | 453 --- .../tasks/eval_harness/report-to-csv.py | 61 - .../Megatron-DeepSpeed/tasks/eval_utils.py | 247 -- .../tasks/finetune_utils.py | 351 -- toolbox/Megatron-DeepSpeed/tasks/glue/cola.py | 90 - toolbox/Megatron-DeepSpeed/tasks/glue/data.py | 56 - .../Megatron-DeepSpeed/tasks/glue/finetune.py | 134 - toolbox/Megatron-DeepSpeed/tasks/glue/mnli.py | 71 - toolbox/Megatron-DeepSpeed/tasks/glue/mrpc.py | 101 - toolbox/Megatron-DeepSpeed/tasks/glue/qnli.py | 101 - toolbox/Megatron-DeepSpeed/tasks/glue/qqp.py | 88 - toolbox/Megatron-DeepSpeed/tasks/glue/rte.py | 101 - toolbox/Megatron-DeepSpeed/tasks/glue/sst2.py | 95 - toolbox/Megatron-DeepSpeed/tasks/glue/stsb.py | 100 - toolbox/Megatron-DeepSpeed/tasks/main.py | 102 - .../Megatron-DeepSpeed/tasks/msdp/README.md | 19 - .../Megatron-DeepSpeed/tasks/msdp/evaluate.py | 45 - toolbox/Megatron-DeepSpeed/tasks/msdp/main.py | 66 - .../Megatron-DeepSpeed/tasks/msdp/metrics.py | 77 - .../tasks/msdp/preprocessing.py | 582 --- .../Megatron-DeepSpeed/tasks/msdp/prompt.py | 313 -- .../Megatron-DeepSpeed/tasks/orqa/README.md | 36 - .../tasks/orqa/evaluate_orqa.py | 39 - .../tasks/orqa/evaluate_utils.py | 176 - .../tasks/orqa/supervised/data.py | 287 -- .../tasks/orqa/supervised/eval_utils.py | 193 - .../tasks/orqa/supervised/finetune.py | 238 -- .../tasks/orqa/unsupervised/nq.py | 216 -- .../tasks/orqa/unsupervised/qa_utils.py | 177 - .../tasks/orqa/unsupervised/tokenizers.py | 243 -- toolbox/Megatron-DeepSpeed/tasks/race/data.py | 135 - .../Megatron-DeepSpeed/tasks/race/finetune.py | 55 - .../vision/classification/classification.py | 81 - .../tasks/vision/classification/eval_utils.py | 116 - .../tasks/vision/finetune_utils.py | 301 -- .../Megatron-DeepSpeed/tasks/vision/main.py | 53 - .../tasks/vision/segmentation/cityscapes.py | 207 -- .../tasks/vision/segmentation/data.py | 154 - .../vision/segmentation/finetune_segformer.py | 239 -- .../vision/segmentation/finetune_setr.py | 213 -- .../tasks/vision/segmentation/metrics.py | 594 --- .../tasks/vision/segmentation/seg_heads.py | 127 - .../tasks/vision/segmentation/seg_models.py | 79 - .../tasks/vision/segmentation/transforms.py | 433 --- .../tasks/vision/segmentation/utils.py | 85 - .../tasks/zeroshot_gpt/datasets.py | 148 - .../tasks/zeroshot_gpt/detokenizer.py | 67 - .../tasks/zeroshot_gpt/evaluate.py | 213 -- toolbox/Megatron-DeepSpeed/tests/__init__.py | 0 toolbox/Megatron-DeepSpeed/tests/conftest.py | 22 - .../tests/functional_tests/__init__.py | 0 .../python_test_utils/__init__.py | 0 .../check_slurm_job_completion.py | 19 - .../get_test_results_from_tensorboard_logs.py | 73 - .../python_test_utils/test_ci_pipeline.py | 87 - .../test_resume_checkpoint_pipeline.py | 55 - .../shell_test_utils/jobwait.sh | 25 - .../bert/bert_tp1_pp2_1nodes_50steps.json | 1 - .../bert/bert_tp1_pp4_1nodes_50steps.json | 1 - .../bert/bert_tp2_pp2_1nodes_50steps.json | 1 - .../bert/bert_tp4_pp1_1nodes_50steps.json | 1 - .../gpt3/gpt3_tp1_pp2_1nodes_50steps.json | 1 - .../gpt3/gpt3_tp1_pp4_1nodes_50steps.json | 1 - .../gpt3/gpt3_tp2_pp2_1nodes_50steps.json | 1 - .../gpt3/gpt3_tp4_pp1_1nodes_50steps.json | 1 - ...bert_distributed_resume_checkpoint_test.sh | 100 - .../bert/pretrain_bert_distributed_test.sh | 59 - ...bert_distributed_resume_checkpoint_test.sh | 16 - .../bert/sbatch_bert_distributed_test.sh | 16 - ...gpt3_distributed_resume_checkpoint_test.sh | 108 - .../gpt3/pretrain_gpt3_distributed_test.sh | 76 - ...gpt3_distributed_resume_checkpoint_test.sh | 16 - .../gpt3/sbatch_gpt3_distributed_test.sh | 22 - .../tests/models/__init__.py | 0 .../tests/models/test_gpt_embedding.py | 47 - .../tests/models/test_gpt_model.py | 69 - .../tests/pipeline_parallel/__init__.py | 0 .../tests/pipeline_parallel/test_schedules.py | 201 -- .../Megatron-DeepSpeed/tests/requirements.txt | 3 - .../Megatron-DeepSpeed/tests/run_megatron.py | 118 - .../tests/run_test_multi_node.sh | 68 - .../tests/run_test_one_node.sh | 16 - .../tests/tensor_parallel/__int__.py | 0 .../Megatron-DeepSpeed/tests/test_megatron.py | 61 - .../tests/test_megatron_adapter.py | 6 - toolbox/Megatron-DeepSpeed/tests/tests.py | 288 -- .../tests/transformer/__init__.py | 0 .../tests/transformer/test_core_attention.py | 63 - .../tests/transformer/test_module.py | 77 - .../transformer/test_parallel_attention.py | 78 - .../tests/transformer/test_parallel_mlp.py | 46 - .../test_parallel_transformer_block.py | 91 - .../test_parallel_transformer_layer.py | 40 - .../transformer/test_transformer_config.py | 10 - .../tests/unit_tests/__init__.py | 0 .../unit_tests/tensor_parallel/__init__.py | 0 .../tensor_parallel/test_cross_entropy.py | 15 - .../unit_tests/tensor_parallel/test_data.py | 22 - .../tensor_parallel/test_mappings.py | 136 - .../unit_tests/tensor_parallel/test_random.py | 45 - .../test_tensor_parallel_utils.py | 44 - .../tests/unit_tests/test_basic.py | 4 - .../tests/unit_tests/test_parallel_state.py | 109 - .../tests/unit_tests/test_utilities.py | 37 - .../tests/unit_tests/test_utils.py | 37 - .../tools/bert_embedding/__init__.py | 3 - .../tools/bert_embedding/dataset.py | 68 - .../tools/bert_embedding/embed.py | 321 -- .../tools/bert_embedding/external_libs.py | 14 - .../tools/bert_embedding/huggingface.py | 126 - .../tools/bert_embedding/utils.py | 193 - .../tools/convert_checkpoint/README.md | 78 - .../deepspeed_checkpoint.py | 196 - .../deepspeed_to_megatron.py | 150 - .../deepspeed_to_transformers.py | 83 - .../convert_checkpoint/inspect_checkpoint.py | 40 - .../inspect_deepspeed_checkpoint.py | 80 - .../Megatron-DeepSpeed/tools/convert_mg2hf.sh | 14 - .../tools/generate_samples_gpt.py | 176 - .../tools/hf2megads_weight_converter.py | 334 -- toolbox/Megatron-DeepSpeed/tools/linter.py | 36 - .../tools/merge_datasets.py | 66 - .../tools/openwebtext/README.md | 59 - .../tools/openwebtext/add_id.py | 54 - .../tools/openwebtext/blacklist_urls.py | 299 -- .../tools/openwebtext/cleanup_dataset.py | 102 - .../tools/openwebtext/cleanup_fix_dataset.py | 178 - .../tools/openwebtext/filter_ngrams.py | 479 --- .../tools/openwebtext/find_duplicates.py | 292 -- .../tools/openwebtext/group_duplicate_url.py | 77 - .../tools/openwebtext/merge_jsons.py | 42 - .../openwebtext/remove_group_duplicates.py | 56 - .../tools/preprocess_data.py | 431 --- .../tools/preprocess_data_nmt.py | 113 - .../Megatron-DeepSpeed/tools/retro/README.md | 226 -- .../tools/retro/__init__.py | 0 .../tools/retro/cli/__init__.py | 3 - .../tools/retro/cli/__main__.py | 9 - .../Megatron-DeepSpeed/tools/retro/cli/cli.py | 299 -- .../tools/retro/db/__init__.py | 3 - .../tools/retro/db/build.py | 497 --- .../tools/retro/db/dataset.py | 74 - .../tools/retro/db/utils.py | 143 - .../retro/examples/get_dataset_configs.sh | 43 - .../retro/examples/get_preprocess_cmd.sh | 137 - .../tools/retro/examples/preprocess_data.sh | 50 - .../tools/retro/examples/pretrain_model.sh | 105 - .../tools/retro/external_libs.py | 15 - .../tools/retro/index/__init__.py | 4 - .../tools/retro/index/build.py | 187 - .../tools/retro/index/factory.py | 23 - .../tools/retro/index/index.py | 67 - .../tools/retro/index/indexes/__init__.py | 4 - .../tools/retro/index/indexes/faiss_base.py | 137 - .../retro/index/indexes/faiss_par_add.py | 162 - .../tools/retro/index/utils.py | 72 - .../Megatron-DeepSpeed/tools/retro/main.py | 242 -- .../tools/retro/query/__init__.py | 3 - .../tools/retro/query/chunk_dataset.py | 138 - .../tools/retro/query/query.py | 252 -- .../tools/retro/query/retro_dataset.py | 169 - .../tools/retro/query/utils.py | 17 - .../Megatron-DeepSpeed/tools/retro/utils.py | 75 - .../tools/run_text_generation_server.py | 80 - .../tools/text_generation_cli.py | 23 - 623 files changed, 11671 insertions(+), 81390 deletions(-) delete mode 100644 toolbox/Megatron-DeepSpeed/.coveragerc delete mode 100644 toolbox/Megatron-DeepSpeed/.gitignore delete mode 100644 toolbox/Megatron-DeepSpeed/.gitlab-ci.yml delete mode 100644 toolbox/Megatron-DeepSpeed/CODEOWNERS delete mode 100644 toolbox/Megatron-DeepSpeed/ILUVATAR.md delete mode 100644 toolbox/Megatron-DeepSpeed/LICENSE delete mode 100644 toolbox/Megatron-DeepSpeed/MANIFEST.in delete mode 100644 toolbox/Megatron-DeepSpeed/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/README_RLHF.md delete mode 100644 toolbox/Megatron-DeepSpeed/SECURITY.md delete mode 100644 toolbox/Megatron-DeepSpeed/build_megatron-deepspeed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/ci/run_ci_tests_multi_node.sh delete mode 100644 toolbox/Megatron-DeepSpeed/ci/run_ci_tests_one_node.sh delete mode 100644 toolbox/Megatron-DeepSpeed/clean_megatron-deepspeed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/convert_dahoas.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/convert_llama2tokenizer_dataset.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/download_RedPajama-Data-1T-Sample.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/download_and_covert_llama3_dataset.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/download_books.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/download_ckpt.sh delete mode 100644 toolbox/Megatron-DeepSpeed/dataset/download_vocab.sh delete mode 100644 toolbox/Megatron-DeepSpeed/docs/distrib_optimizer.md delete mode 100644 toolbox/Megatron-DeepSpeed/docs/images/distrib_optimizer/data_flow.png delete mode 100644 toolbox/Megatron-DeepSpeed/docs/images/distrib_optimizer/sharding_scheme.png delete mode 100644 toolbox/Megatron-DeepSpeed/examples/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/filter-selfgeneration.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/perspective_api_annotate.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/preprocess.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt_distributed-1.3b.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate-1.3b.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate_samples_gpt.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/perspective_api.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/evaluate_retriever_nq.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/evaluate_zeroshot_gpt.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/finetune_mnli_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/finetune_race_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/finetune_retriever_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_ixte_llama2_34b_node4.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_llama2_7b.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_tinyllama_1.1b.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_node4.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_tpoverlap_profiling_node1.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_node4.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_tpoverlap_profiling_node1.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_node1.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_tpoverlap_profiling_node1.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_pp_overlap_node1_card8.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_tp_overlap_node1_card2.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/llama3/run_te_llama3_8b_node1.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/merge_mp_bert.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/data_processing.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/eval_knwl_generation.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/eval_resp_generation.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/prep_resp_gen.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/prompt_knwl_gen.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/msdp/prompt_resp_gen.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_bert.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed_with_mp.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_gpt.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_gpt3_175B.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed_with_mp.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_ict.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_t5.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed_with_mp.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M_8_tensor_parallel.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/CONFIG.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/SBATCH.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/SRUN.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_11.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_12.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_13.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_14.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_15.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_16.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_17.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_18.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples/sc21/run_table_1.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_Zero2_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_evalharness.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_MoE128.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128_MoS.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense_cl.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_MoE64.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_dense_cl.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_MoE128.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64_MoS.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_dense.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_6.7B_dense.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/readme_evalharness.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-175b.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-1t.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-benchmark-model.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/Dockerfile.dockerfile delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/aml_submit.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/prepare_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_config_bert_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_mnli.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_qqp.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_race.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_pretrain_bert.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/prepare_pile_data.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-Int8-test-64gpu-distilled-group48.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L10-Int8-test-64gpu-distilled-group48.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L12-Int8-test-64gpu-distilled-group48.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE_compression.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_evalharness.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_1.3B_dense_cl_kd.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_cl_kd.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_kd.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_350M_dense_kd.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_config_gpt_slw_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt2.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt_1.3B_rope_slw.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_train.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_baseline.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_curriculum_fixed_linear.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/analyze_data.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_map.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_reduce.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_config_bert_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_mnli.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_qqp.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_race.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_gather_result.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_config_bert_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue_run.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_gather_result.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pile_data_download_preprocess.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_1clmetric_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_2clmetrics_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_base_script.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_run.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_map.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_reduce.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_config_eval_dummy.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_1gpu.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_gather_result.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run_10shot.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_1clmetric_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_2clmetrics_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_base_script.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_run.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/ds_config_gpt_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/host_file delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_1.3B_seq_parallel.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_30B_seq_parallel.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/ds_config.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/finetune_llama.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/generate_text.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_config_gpt_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_pretrain_gpt_350M.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/twin-offload.png delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/pretrain_llama2_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/pretrain_llama_distributed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_slw_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_megatron_checkpointing.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope_slw.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M_flashattn.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_13B.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/run_deepspeed_example.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_config_gpt_TEMPLATE.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_1.3B_seq_parallel_32k.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_30B_seq_parallel_32k.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/assets/image/uc_char_training_loss.png delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/assets/image/uc_char_validation_loss.png delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/ds_config.json delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/run_bf16.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/run_fp16.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/run_tb_analysis.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/run_universal_bf16.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/run_universal_fp16.sh delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/tb_analysis/abstract_analysis.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/tb_analysis/arguments.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/tb_analysis/tb_analysis_script.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/tb_analysis/uc_analysis.py delete mode 100644 toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/tb_analysis/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/finetune_llama.py delete mode 100644 toolbox/Megatron-DeepSpeed/images/Achieved_petaFLOPs.png delete mode 100644 toolbox/Megatron-DeepSpeed/images/cases_april2021.png delete mode 100644 toolbox/Megatron-DeepSpeed/install_megatron-deepspeed.sh delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/checkpointing.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/Makefile delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/blended_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/blended_megatron_dataset_builder.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/blended_megatron_dataset_config.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/gpt_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/helpers.cpp delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/indexed_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/megatron_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/readme.md delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/datasets/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/core.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/dict_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/mapping.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/optimizer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/serialization.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/strategies/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/strategies/base.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/strategies/tensorstore.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/strategies/two_stage.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/strategies/zarr.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/dist_checkpointing/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/distributed/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/distributed/distributed_data_parallel.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/distributed/finalize_model_grads.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/distributed/grad_buffer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/enums.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/fusions/fused_bias_dropout.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/fusions/fused_bias_gelu.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/fusions/fused_layer_norm.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/fusions/fused_softmax.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/inference_params.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/model_parallel_config.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/T5/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/T5/t5_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/T5/t5_spec.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/bert/bert_layer_specs.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/bert/bert_lm_head.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/bert/bert_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/bert/pooler.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/common/embeddings/language_model_embedding.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/common/embeddings/rotary_pos_embedding.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/common/language_module/language_module.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/gpt/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/gpt/gpt_embedding.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/gpt/gpt_layer_specs.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/gpt/gpt_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/base_attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/config.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/decoder_attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/decoder_spec.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/encoder_attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/encoder_spec.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/models/retro/model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/parallel_state.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/pipeline_parallel/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/pipeline_parallel/p2p_communication.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/pipeline_parallel/schedules.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/tensor_parallel/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/tensor_parallel/cross_entropy.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/tensor_parallel/layers.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/tensor_parallel/mappings.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/tensor_parallel/random.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/tensor_parallel/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/custom_layers/transformer_engine.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/dot_product_attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/enums.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/identity_op.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/mlp.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/module.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/spec_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/switch_mlp.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/transformer_block.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/transformer_config.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/transformer_layer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/transformer/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/core/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/dist_signal_handler.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/enums.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/fp16_deprecated/loss_scaler.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/fused_kernels/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/fused_kernels/compat.h delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/fused_kernels/tests/test_fused_kernels.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/fused_kernels/type_shim.h delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/global_vars.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/indexer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/microbatches.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/biencoder_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/enums.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/fused_bias_gelu.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/fused_layer_norm.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/fused_softmax.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/language_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/module.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/realm_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/rms_norm.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/transformer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/classification.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/dino.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/esvit_swin_backbone.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/inpainting.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/knn_monitor.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/mit_backbone.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/swin_backbone.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/model/vision/vit_backbone.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/commons.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/test_cross_entropy.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/test_data.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/test_initialize.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/test_layers.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/mpu/tests/test_random.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer/clip_grads.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer/distrib_optimizer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer/grad_scaler.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer/optimizer.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/optimizer_param_scheduler.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/rlhf/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/rlhf/generation/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/static/index.html delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/api.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/beam_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/communication.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/forward_step.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/generation.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/sampling.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation/tokenization.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/text_generation_server.py delete mode 100644 toolbox/Megatron-DeepSpeed/megatron_ds/timers.py rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/datasets => megatronspeed}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/core/README.md (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/fusions => megatronspeed/core}/__init__.py (100%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/optimizer/__init__.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/core/package_info.py (98%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/parallel_state.py rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/models => megatronspeed/core/pipeline_parallel}/__init__.py (100%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/pipeline_parallel/deepspeed_zbh1_engine.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/pipeline_parallel/deepspeed_zbh1_schedule.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/pipeline_parallel/schedules.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/core/requirements.txt (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/core/sequence_parallel/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/core/sequence_parallel/cross_entropy.py (97%) rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/models/bert => megatronspeed/core/tensor_parallel}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/core/tensor_parallel/data.py (67%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/tensor_parallel/layers.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/tensor_parallel/random.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/tensor_parallel/weight_grad_store.py rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/models/common => megatronspeed/core/transformer}/__init__.py (100%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/transformer/utils.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/core/utils.py rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/models/common/embeddings => megatronspeed/legacy}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/Makefile (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/models/common/language_module => megatronspeed/legacy/data}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/autoaugment.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/bert_dataset.py (98%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/biencoder_dataset_utils.py (95%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/blendable_dataset.py (94%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/data_samplers.py (94%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/dataset_utils.py (98%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/gpt_dataset.py (97%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/helpers.cpp (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/ict_dataset.py (96%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/image_folder.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/indexed_dataset.py (99%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/multimodal_dataset.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/orqa_wiki_dataset.py (97%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/realm_dataset_utils.py (94%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/realm_index.py (99%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/t5_dataset.py (99%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/test/test_indexed_dataset.py (97%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/test/test_preprocess_data.sh (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/data/vit_dataset.py (97%) rename toolbox/Megatron-DeepSpeed/{megatron_ds/core/transformer/custom_layers => megatronspeed/legacy/model}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/bert_model.py (68%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/legacy/model/biencoder_model.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/classification.py (80%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/distributed.py (98%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/gpt_model.py (37%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/legacy/model/language_model.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/legacy/model/module.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/multiple_choice.py (84%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/legacy/model/realm_model.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/legacy/model/rms_norm.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/rotary_pos_embedding.py (43%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/t5_model.py (83%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/legacy/model/transformer.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/legacy}/model/utils.py (35%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/log_handler.py (100%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/megatron_adaptor.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/p2p_communication.py (98%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/patch_utils.py rename toolbox/Megatron-DeepSpeed/{megatron_ds/data => megatronspeed/rlhf}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/checkpointing_rlhf.py (98%) rename toolbox/Megatron-DeepSpeed/{megatron_ds/fused_kernels/tests => megatronspeed/rlhf/generation}/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/generation/communication_rlhf.py (99%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/generation/forward_rlhf.py (97%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/generation/generation_rlhf.py (95%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/initialize_rlhf.py (94%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/schedules_rlhf.py (99%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/rlhf/training_rlhf.py (82%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/text_generation_utils.py (98%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed}/theoretical_memory_usage.py (100%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/training/__init__.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/arguments.py (36%) create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/training/checkpointing.py create mode 100644 toolbox/Megatron-DeepSpeed/megatronspeed/training/global_vars.py rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/initialize.py (66%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/memory.py (99%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/tokenizer/__init__.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/tokenizer/bert_tokenization.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/tokenizer/gpt2_tokenization.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/tokenizer/tokenization_utils.py (100%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/tokenizer/tokenizer.py (50%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/training.py (71%) rename toolbox/Megatron-DeepSpeed/{megatron_ds => megatronspeed/training}/utils.py (54%) delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_bert.py delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_ict.py delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_retro.py delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_t5.py delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_vision_classify.py delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_vision_dino.py delete mode 100644 toolbox/Megatron-DeepSpeed/pretrain_vision_inpaint.py delete mode 100644 toolbox/Megatron-DeepSpeed/report_theoretical_memory.py delete mode 100644 toolbox/Megatron-DeepSpeed/requirments_rlhf.txt delete mode 100644 toolbox/Megatron-DeepSpeed/setup.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/data_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/ensemble_classifier.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/eval_harness/download.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/eval_harness/evaluate.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/eval_harness/report-to-csv.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/eval_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/finetune_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/cola.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/data.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/finetune.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/mnli.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/mrpc.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/qnli.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/qqp.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/rte.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/sst2.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/glue/stsb.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/main.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/msdp/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/msdp/evaluate.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/msdp/main.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/msdp/metrics.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/msdp/preprocessing.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/msdp/prompt.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/evaluate_orqa.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/evaluate_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/supervised/data.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/supervised/eval_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/supervised/finetune.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/unsupervised/nq.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/unsupervised/qa_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/orqa/unsupervised/tokenizers.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/race/data.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/race/finetune.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/classification/classification.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/classification/eval_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/finetune_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/main.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/cityscapes.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/data.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/finetune_segformer.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/finetune_setr.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/metrics.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/seg_heads.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/seg_models.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/transforms.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/vision/segmentation/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/zeroshot_gpt/datasets.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/zeroshot_gpt/detokenizer.py delete mode 100644 toolbox/Megatron-DeepSpeed/tasks/zeroshot_gpt/evaluate.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/conftest.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/python_test_utils/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/python_test_utils/check_slurm_job_completion.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/python_test_utils/get_test_results_from_tensorboard_logs.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/python_test_utils/test_ci_pipeline.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/shell_test_utils/jobwait.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/bert/bert_tp1_pp2_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/bert/bert_tp1_pp4_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/bert/bert_tp2_pp2_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/bert/bert_tp4_pp1_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/gpt3/gpt3_tp1_pp2_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/gpt3/gpt3_tp1_pp4_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/gpt3/gpt3_tp2_pp2_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_results/gpt3/gpt3_tp4_pp1_1nodes_50steps.json delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/bert/pretrain_bert_distributed_resume_checkpoint_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/bert/pretrain_bert_distributed_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/bert/sbatch_bert_distributed_resume_checkpoint_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/bert/sbatch_bert_distributed_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/gpt3/pretrain_gpt3_distributed_resume_checkpoint_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/gpt3/pretrain_gpt3_distributed_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/gpt3/sbatch_gpt3_distributed_resume_checkpoint_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/functional_tests/test_scripts/gpt3/sbatch_gpt3_distributed_test.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/models/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/models/test_gpt_embedding.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/models/test_gpt_model.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/pipeline_parallel/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/pipeline_parallel/test_schedules.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/requirements.txt delete mode 100644 toolbox/Megatron-DeepSpeed/tests/run_megatron.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/run_test_multi_node.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/run_test_one_node.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tests/tensor_parallel/__int__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/test_megatron.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/test_megatron_adapter.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/tests.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_core_attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_module.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_parallel_attention.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_parallel_mlp.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_parallel_transformer_block.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_parallel_transformer_layer.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/transformer/test_transformer_config.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/tensor_parallel/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/tensor_parallel/test_cross_entropy.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/tensor_parallel/test_data.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/tensor_parallel/test_mappings.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/tensor_parallel/test_random.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/tensor_parallel/test_tensor_parallel_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/test_basic.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/test_parallel_state.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/test_utilities.py delete mode 100644 toolbox/Megatron-DeepSpeed/tests/unit_tests/test_utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/bert_embedding/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/bert_embedding/dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/bert_embedding/embed.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/bert_embedding/external_libs.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/bert_embedding/huggingface.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/bert_embedding/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_checkpoint/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_checkpoint/deepspeed_checkpoint.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_checkpoint/deepspeed_to_megatron.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_checkpoint/deepspeed_to_transformers.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_checkpoint/inspect_checkpoint.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_checkpoint/inspect_deepspeed_checkpoint.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/convert_mg2hf.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tools/generate_samples_gpt.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/hf2megads_weight_converter.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/linter.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/merge_datasets.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/add_id.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/blacklist_urls.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/cleanup_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/cleanup_fix_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/filter_ngrams.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/find_duplicates.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/group_duplicate_url.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/merge_jsons.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/openwebtext/remove_group_duplicates.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/preprocess_data.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/preprocess_data_nmt.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/README.md delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/cli/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/cli/__main__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/cli/cli.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/db/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/db/build.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/db/dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/db/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/examples/get_dataset_configs.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/examples/get_preprocess_cmd.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/examples/preprocess_data.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/examples/pretrain_model.sh delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/external_libs.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/build.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/factory.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/index.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/indexes/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/indexes/faiss_base.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/indexes/faiss_par_add.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/index/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/main.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/query/__init__.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/query/chunk_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/query/query.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/query/retro_dataset.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/query/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/retro/utils.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/run_text_generation_server.py delete mode 100644 toolbox/Megatron-DeepSpeed/tools/text_generation_cli.py diff --git a/nlp/llm/aquila2-34b/pytorch/README.md b/nlp/llm/aquila2-34b/pytorch/README.md index 5fd7a50b2..9c95b4742 100644 --- a/nlp/llm/aquila2-34b/pytorch/README.md +++ b/nlp/llm/aquila2-34b/pytorch/README.md @@ -29,7 +29,6 @@ ssh-copy-id -i ~/.ssh/id_rsa.pub ${host_name} ## {host_name} can be a specified ```sh # install cd /toolbox/Megatron-DeepSpeed -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh ``` ### Preparing datasets on all nodes diff --git a/nlp/llm/llama2-13b/pytorch/README.md b/nlp/llm/llama2-13b/pytorch/README.md index 523498f8c..1e76ae223 100644 --- a/nlp/llm/llama2-13b/pytorch/README.md +++ b/nlp/llm/llama2-13b/pytorch/README.md @@ -26,7 +26,6 @@ ssh-copy-id -i ~/.ssh/id_rsa.pub ${host_name} ## {host_name} can be a specified ```sh # install cd /toolbox/Megatron-DeepSpeed -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh ``` ### Preparing datasets on all nodes diff --git a/nlp/llm/llama2-34b/pytorch/README.md b/nlp/llm/llama2-34b/pytorch/README.md index bb9c845f5..aa3c3bd5e 100644 --- a/nlp/llm/llama2-34b/pytorch/README.md +++ b/nlp/llm/llama2-34b/pytorch/README.md @@ -28,7 +28,6 @@ ssh-copy-id -i ~/.ssh/id_rsa.pub ${host_name} ## {host_name} can be a specified ```sh # install cd /toolbox/Megatron-DeepSpeed -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh ``` ### Preparing datasets on all nodes diff --git a/nlp/llm/llama2-7b/pytorch/README.md b/nlp/llm/llama2-7b/pytorch/README.md index 800c59b0e..fda863dc7 100644 --- a/nlp/llm/llama2-7b/pytorch/README.md +++ b/nlp/llm/llama2-7b/pytorch/README.md @@ -22,13 +22,6 @@ tar -xf gpt_small_117M.tar rm -f gpt_small_117M.tar ``` -### Install Dependencies - -```sh -# install -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh -``` - ## Model Training ```sh diff --git a/nlp/llm/llama2-7b_rlhf/pytorch/README.md b/nlp/llm/llama2-7b_rlhf/pytorch/README.md index bde048190..1b18409e7 100644 --- a/nlp/llm/llama2-7b_rlhf/pytorch/README.md +++ b/nlp/llm/llama2-7b_rlhf/pytorch/README.md @@ -21,26 +21,24 @@ Download dataset and convert it. cd /toolbox/Megatron-DeepSpeed/ pushd dataset/ - # get gpt_small_117M.tar wget http://files.deepspark.org.cn:880/deepspark/data/datasets/gpt_small_117M.tar tar -xf gpt_small_117M.tar rm -f gpt_small_117M.tar popd -``` - -### Install Dependencies -```sh -# install -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh +# Download checkpoints as above and put them to proper path, then convert checkpoints. +pushd checkpoints +bash download_rlhf_checkpoints.sh +bash convert_hf_2_meg.sh +popd ``` ## Model Training ```sh cd examples/llama2 -# Modify run_llama2_7b_1node.sh according to your machine: for example, HOST_NAME, ADDR_ARRAY, CONTAINER_NAME, NCCL_SOCKET_IFNAME +# Modify run_llama2_7b_rlhf_node1.sh according to your machine: for example, HOST_NAME, ADDR_ARRAY, CONTAINER_NAME, NCCL_SOCKET_IFNAME, DATA_PATH bash run_llama2_7b_rlhf_node1.sh ``` diff --git a/nlp/llm/llama2-7b_sft/pytorch/README.md b/nlp/llm/llama2-7b_sft/pytorch/README.md index a58ea7494..4020a1a61 100644 --- a/nlp/llm/llama2-7b_sft/pytorch/README.md +++ b/nlp/llm/llama2-7b_sft/pytorch/README.md @@ -24,13 +24,6 @@ rm -f gpt_small_117M.tar popd ``` -### Install Dependencies - -```sh -# install -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh -``` - ## Model Training ```sh diff --git a/toolbox/Megatron-DeepSpeed/.coveragerc b/toolbox/Megatron-DeepSpeed/.coveragerc deleted file mode 100644 index 29de6ff8a..000000000 --- a/toolbox/Megatron-DeepSpeed/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[html] -directory = coverage - -[run] -data_file = .coverage_$LOCAL_RANK diff --git a/toolbox/Megatron-DeepSpeed/.gitignore b/toolbox/Megatron-DeepSpeed/.gitignore deleted file mode 100644 index c37f055fa..000000000 --- a/toolbox/Megatron-DeepSpeed/.gitignore +++ /dev/null @@ -1,37 +0,0 @@ -__pycache__ - -# Distribution / packaging -build/ -build_pip/ -dist/ -*.egg-info/ - -# binaries -*.so - -# tmp files -*.swp - -# AML workspace config file -config.json - -# DeepSpeed config file -ds_config.json - -.coverage_* -*~ -slurm* -logs - -# Data folder -dataset/BookCorpusDataset/ -dataset/gpt_small_117M* -dataset/dahoas/ -dataset/dahoas_* - -tests/test_logs/ -tests/exit_* - -checkpoints/output*/ -checkpoints/rlhf*/ -checkpoints/TinyLlama*/ diff --git a/toolbox/Megatron-DeepSpeed/.gitlab-ci.yml b/toolbox/Megatron-DeepSpeed/.gitlab-ci.yml deleted file mode 100644 index 0abebc72a..000000000 --- a/toolbox/Megatron-DeepSpeed/.gitlab-ci.yml +++ /dev/null @@ -1,302 +0,0 @@ -image: nvcr.io/nvidia/pytorch:23.04-py3 - -stages: - - test - - cleanup - -variables: &VARS - SELENE_ADLR_CI_PATH: "/lustre/fsw/adlr/adlr-nlp/adlr_ci/megatron" - DATA_DIR: "/lustre/fsw/adlr/adlr-nlp/adlr_ci/megatron/data" - PYTORCH_IMAGE: gitlab-master.nvidia.com:5005/adlr/megatron-lm/ngc/pytorch:22.12-py3_pytest-cov - PYTHON_VIRTUAL_ENV: /lustre/fsw/adlr/adlr-nlp/adlr_ci/cienv/bin/activate - TESTS_TO_RUN_AFTER_MERGE_REQ_APPROVED: L0 # Can specify levels - TESTS_TO_RUN_AFTER_MERGING: L0 # Can specify levels - TESTS_TO_RUN_ON_THIS_COMMIT: unit_tests - TEST_REGEX_ON_THIS_COMMIT: NONE #https://github.com/google/re2/wiki/Syntax (Can define regex as in this spec) e.g /.*gpt3.*/ - DISPLAY_OUTPUT: "True" # Set to true for new tests to copy the logs for creating golden truth file - -unit_tests: - tags: - - docker_local_runner - stage: test - script: - - pip install pytest-cov - - torchrun --nproc_per_node=8 -m pytest --cov-report=term --cov-report=html --cov=megatron/core tests/unit_tests - coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' - artifacts: - paths: - - coverage - expire_in: 30 days - only: - - merge_requests - -.selene_test_resume_checkpoint_launcher: &selene-test-resume-checkpoint-launcher - tags: - - ssh_selene_runner - stage: test - script: &selene-test-resume-launcher-script - - echo "Running selene resume from checkpoint test. " - - pwd - - export BUILD_DIR=`pwd` - - export RUN_NAME=resume_${RUN_MODEL}_tp${TP_SIZE}_pp${PP_SIZE}_${NUM_NODES}nodes - - echo "In case of error check ${SELENE_ADLR_CI_PATH}/${CI_PIPELINE_ID}/${RUN_NAME}/results directory for result logs." - - export TP_SIZE PP_SIZE NUM_NODES MAX_STEPS - - export DATA_DIR=$DATA_DIR - - echo "Run name is $RUN_NAME" - - mkdir -p $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/checkpoints - - mkdir -p $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/logs - - mkdir -p $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/results - - rm -rf $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/checkpoints/* - - rm -rf $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/logs/* - - rm -rf $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/results/* - - export BASE_DIR=$SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME - - export LOGS_DIR=$BASE_DIR/logs - - export RESULTS_DIR=$BASE_DIR/results - - export CHECKPOINTS_DIR=$BASE_DIR/checkpoints - - echo "Submitting job" - - sbatch_submission=`sbatch $BUILD_DIR/tests/functional_tests/test_scripts/$RUN_MODEL/sbatch_${RUN_MODEL}_distributed_resume_checkpoint_test.sh --export=BASE_DIR,BUILD_DIR,DATA_DIR,TP_SIZE,PP_SIZE,NUM_NODES` - - export SLURM_JOBID=$(echo $sbatch_submission| grep 'Submitted batch job' | awk '{ print $4 }'); - - bash $BUILD_DIR/tests/functional_tests/shell_test_utils/jobwait.sh $SLURM_JOBID - - \[ ! -z ${SLURM_JOBID} \] && echo -e " --------------------------------------------------\n" - "----------WAITING FOR SLURM JOB TO BEGIN-----------\n" - "---------------------------------------------------\n" - "$(scontrol show job=${SLURM_JOBID})\n" - "---------------------------------------------------\n" - # Gitlab logs collapsible section markers - - echo -e "\e[0Ksection_end:`date +%s`:slurm_setup\r\e[0K" - # Follow output of the job - - echo "Finished job" - - export SLURM_STATE=$(sacct -j "${SLURM_JOBID}" --format State --parsable2 --noheader |& head -n 1) - - echo "Slurm job state $SLURM_STATE" - - if [[ "$SLURM_STATE" != "COMPLETED" ]]; then echo "Slurm job did not complete. See ${SELENE_ADLR_CI_PATH}/${CI_PIPELINE_ID}/${RUN_NAME}/results directory for result logs. Skipping pytest."; exit 1; fi - - source $PYTHON_VIRTUAL_ENV - - cmd="pytest $BUILD_DIR/tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py" - - if $cmd; then echo "Pytest succeded"; else echo "Pytest failed. See ${SELENE_ADLR_CI_PATH}/${CI_PIPELINE_ID}/${RUN_NAME}/results directory for result logs"; fi - - echo "Completed the job" - rules: - - if: $TEST_LEVEL =~ $TESTS_TO_RUN_ON_THIS_COMMIT || $CI_JOB_NAME =~ $TESTS_TO_RUN_ON_THIS_COMMIT || $CI_JOB_NAME =~ $TEST_REGEX_ON_THIS_COMMIT - when: always - - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $TEST_LEVEL =~ $TESTS_TO_RUN_AFTER_MERGING' - when: always - - if: $CI_MERGE_REQUEST_APPROVED && $TEST_LEVEL =~ $TESTS_TO_RUN_AFTER_MERGE_REQ_APPROVED - when: always - allow_failure: false - -.selene_test_launcher: &selene-test-launcher - tags: - - ssh_selene_runner - stage: test - script: &selene-test-launcher-script - - echo "Running selene test" - - echo "$CI_MERGE_REQUEST_APPROVED" - - pwd - - export BUILD_DIR=`pwd` - - RUN_NAME=${RUN_MODEL}_tp${TP_SIZE}_pp${PP_SIZE}_${NUM_NODES}nodes_${MAX_STEPS}steps - - if [[ $USE_TE == 1 ]]; then RUN_NAME=${RUN_NAME}_te_enabled; fi - - export $RUN_NAME - - echo "In case of error check ${SELENE_ADLR_CI_PATH}/${CI_PIPELINE_ID}/${RUN_NAME}/results directory for result logs." - - export USE_TE TP_SIZE PP_SIZE NUM_NODES MAX_STEPS VP_SIZE - - export MBS GBS - - export DATA_DIR=$DATA_DIR - - echo "Run name is $RUN_NAME" - - mkdir -p $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/checkpoints - - mkdir -p $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/logs - - mkdir -p $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/results - - rm -rf $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/checkpoints/* - - rm -rf $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/logs/* - - rm -rf $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/results/* - - export BASE_DIR=$SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME - - export LOGS_DIR=$BASE_DIR/logs - - export RESULTS_DIR=$BASE_DIR/results - - export CHECKPOINTS_DIR=$BASE_DIR/checkpoints - - echo "Submitting job" - - sbatch_submission=`sbatch $BUILD_DIR/tests/functional_tests/test_scripts/$RUN_MODEL/sbatch_${RUN_MODEL}_distributed_test.sh --export=BASE_DIR,BUILD_DIR,DATA_DIR,USE_TE,TP_SIZE,PP_SIZE,NUM_NODES,MAX_STEPS,VP_SIZE,MBS,GBS` - - export SLURM_JOBID=$(echo $sbatch_submission| grep 'Submitted batch job' | awk '{ print $4 }'); - - bash $BUILD_DIR/tests/functional_tests/shell_test_utils/jobwait.sh $SLURM_JOBID - - \[ ! -z ${SLURM_JOBID} \] && echo -e " --------------------------------------------------\n" - "----------WAITING FOR SLURM JOB TO BEGIN-----------\n" - "---------------------------------------------------\n" - "$(scontrol show job=${SLURM_JOBID})\n" - "---------------------------------------------------\n" - # Gitlab logs collapsible section markers - - echo -e "\e[0Ksection_end:`date +%s`:slurm_setup\r\e[0K" - # Follow output of the job - - echo "Finished job" - - echo "Slurm log dump start ------------------------------------------------------------" - - cat $SELENE_ADLR_CI_PATH/$CI_PIPELINE_ID/$RUN_NAME/results/* - - echo "Slurm log dump end --------------------------------------------------------------" - - python3 $BUILD_DIR/tests/functional_tests/python_test_utils/check_slurm_job_completion.py $SLURM_JOBID - - if [ $? -ne 0 ]; then echo "Slurm job did not complete. See ${SELENE_ADLR_CI_PATH}/${CI_PIPELINE_ID}/${RUN_NAME}/results directory for result logs. Skipping pytest."; exit 1; fi - - source $PYTHON_VIRTUAL_ENV - - | - if [[ "$DISPLAY_OUTPUT" == "True" ]]; then - python3 $BUILD_DIR/tests/functional_tests/python_test_utils/get_test_results_from_tensorboard_logs.py $LOGS_DIR $RUN_NAME - fi - - | - if [[ $USE_TE -ne 1 ]]; then - echo "Checking against ground truth file" - export EXPECTED_METRICS_FILE=$BUILD_DIR/tests/functional_tests/test_results/$RUN_MODEL/$RUN_NAME.json - cmd="pytest $BUILD_DIR/tests/functional_tests/python_test_utils/test_ci_pipeline.py" - if $cmd; then echo "Pytest succeded"; else echo "Pytest failed. See ${SELENE_ADLR_CI_PATH}/${CI_PIPELINE_ID}/${RUN_NAME}/results directory for result logs"; fi - fi - - echo "Completed the job" - rules: - - if: $TEST_LEVEL =~ $TESTS_TO_RUN_ON_THIS_COMMIT || $CI_JOB_NAME =~ $TESTS_TO_RUN_ON_THIS_COMMIT || $CI_JOB_NAME =~ $TEST_REGEX_ON_THIS_COMMIT - when: always - - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $TEST_LEVEL =~ $TESTS_TO_RUN_AFTER_MERGING' - when: always - - if: $CI_MERGE_REQUEST_APPROVED && $TEST_LEVEL =~ $TESTS_TO_RUN_AFTER_MERGE_REQ_APPROVED - when: always - allow_failure: false - -train.te_gpt3.345m_tp2_pp2_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: gpt3 - USE_TE: 1 - TP_SIZE: 2 - PP_SIZE: 2 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "50:00" - TEST_LEVEL: L0 - -train.gpt3.345m_tp4_pp1_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: gpt3 - USE_TE: 0 - TP_SIZE: 4 - PP_SIZE: 1 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -train.gpt3.345m_tp2_pp2_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: gpt3 - USE_TE: 0 - TP_SIZE: 2 - PP_SIZE: 2 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -train.gpt3.345m_tp1_pp2_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: gpt3 - USE_TE: 0 - TP_SIZE: 1 - PP_SIZE: 2 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -train.gpt3.345m_tp1_pp4_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: gpt3 - USE_TE: 0 - TP_SIZE: 1 - PP_SIZE: 4 - VP_SIZE: 1 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -resume.checkpoint.gpt3.345m_tp1_pp2_1node: - <<: *selene-test-resume-checkpoint-launcher - variables: - <<: [*VARS] - RUN_MODEL: gpt3 - TP_SIZE: 1 - PP_SIZE: 2 - NUM_NODES: 1 - TIME_LIMIT: "30:00" - TEST_LEVEL: L0 - -train.bert.345m_tp4_pp1_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: bert - TP_SIZE: 4 - PP_SIZE: 1 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -train.bert.345m_tp2_pp2_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: bert - TP_SIZE: 2 - PP_SIZE: 2 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -train.bert.345m_tp1_pp2_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: bert - TP_SIZE: 1 - PP_SIZE: 2 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -train.bert.345m_tp1_pp4_1node_50steps: - <<: *selene-test-launcher - variables: - <<: [*VARS] - RUN_MODEL: bert - TP_SIZE: 1 - PP_SIZE: 4 - VP_SIZE: 2 - NUM_NODES: 1 - MAX_STEPS: 50 - TIME_LIMIT: "20:00" - TEST_LEVEL: L0 - -resume.checkpoint.bert.345m_tp1_pp2_1node: - <<: *selene-test-resume-checkpoint-launcher - variables: - <<: [*VARS] - RUN_MODEL: bert - TP_SIZE: 1 - PP_SIZE: 2 - NUM_NODES: 1 - TIME_LIMIT: "30:00" - TEST_LEVEL: L0 - -cleanup.selene: - tags: - - ssh_selene_runner - stage: cleanup - variables: - <<: [*VARS] - script: - - set +e - - NUM_CLEANUP=`find ${SELENE_ADLR_CI_PATH}/* -type d -ctime +20 | grep -v data | wc -l` - - find ${SELENE_ADLR_CI_PATH}/* -type d -ctime +20 | grep -v data | xargs rm -rf - - echo "Finished cleaning $NUM_CLEANUP directories older than 20 days everything in Selene" - allow_failure: true - rules: - - when: always diff --git a/toolbox/Megatron-DeepSpeed/CODEOWNERS b/toolbox/Megatron-DeepSpeed/CODEOWNERS deleted file mode 100644 index 60a921d7f..000000000 --- a/toolbox/Megatron-DeepSpeed/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @jeffra @samyam @tjruwase @ShadenSmith @conglongli @awan-10 @cli99 @eltonzheng @minjiaz @RezaYazdaniAminabadi @duli2012 @mrwyattii @yaozhewei @arashb @xiaoxiawu-microsoft @guanhuawang diff --git a/toolbox/Megatron-DeepSpeed/ILUVATAR.md b/toolbox/Megatron-DeepSpeed/ILUVATAR.md deleted file mode 100644 index ac140aa22..000000000 --- a/toolbox/Megatron-DeepSpeed/ILUVATAR.md +++ /dev/null @@ -1,99 +0,0 @@ -### 1. Install - -``` -bash clean_megatron-deepspeed.sh -bash build_megatron-deepspeed.sh -bash install_megatron-deepspeed.sh -``` - -### 2. CI Test - -#### 2.1 Test node = 1 - -``` -cd ci && bash run_ci_tests_one_node.sh -``` - -#### 2.2 Test node >= 2 - -First, you should make sure something below. - -1. The CI Test in 1 node can pass in master node container. -2. Copy master node container environment to other node servers. -3. Make sure the account name, contrainer name is the same in different node servers. -4. Set up password free login between the master node container and other node servers. - -Second, set your node server info. You can set up like: - -``` -## The account in server -export HOST_NAME="username" - -## Severs IP, begin with the master node server IP, and split by "," -export ADDR_ARRAY="10.111.222.1,10.111.222.2" - -## Container name -export CONTAINER_NAME="megatron-deepspeed" -``` - -Third, run. - -``` -cd ci && bash run_ci_tests_multi_node.sh -``` - -### 3. Run Aquila-7b bf16 pretrain - -#### 3.1 Download Dataset - -``` -bash dataset/download_dataset.sh -bash dataset/download_vocab.sh -``` - -#### 3.2 Run node=1 - -``` -cd examples/aquila && bash run_aquila_7b_node1_bf16.sh -``` - -#### 3.3 Run node=2 - -First, you should make sure something below. - -1. The pretrain in 1 node run successfully in master node container. -2. Copy master node container environment to other node servers. -3. Make sure the account name, contrainer name is the same in different node servers. -4. Set up password free login between the master node container and other node servers. -5. Make megatron-deepspeed repo and dataset at same path in different node servers. - -Second, set your node server info. You can set up like: - -``` -## The account in server -export HOST_NAME="username" - -## Severs IP, begin with the master node server IP, and split by "," -export ADDR_ARRAY="10.111.222.1,10.111.222.2" - -## Container name -export CONTAINER_NAME="megatron-deepspeed" -``` - -Third, run. - -``` -cd examples/aquila && bash run_aquila_7b_node2_bf16.sh -``` - -### 4. Data preprocess - -If you have an your own jsonl text dataset, and you want to use it in Megatron-Deepspeed, you can do something to change it to proper format. - -For example, look file `megatron-deepspeed/dataset/convert_llama2tokenizer_dataset.sh` , you need to change some main parameter by your own, there contain: - -* tokenizer-type -* tokenizer-model or (vocab-file and merge-file） -* json-keys -* input -* output-prefix diff --git a/toolbox/Megatron-DeepSpeed/LICENSE b/toolbox/Megatron-DeepSpeed/LICENSE deleted file mode 100644 index 281fde95a..000000000 --- a/toolbox/Megatron-DeepSpeed/LICENSE +++ /dev/null @@ -1,376 +0,0 @@ -The following applies to all files unless otherwise noted: - -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --- - -This repository also contains code from Hugging Face Inc., Google Research, -Facebook (from their Fairseq and Dino projects), Microsoft(from their -Swin-Transformer project)and Philip Popien. Files from these -organizations have notices at the top of each file. Below are -licenses used in those files, as indicated. - - -------------- LICENSE FOR Facebook, huggingface and Google Research code -------------- - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -------------- LICENSE FOR Facebook Fairseq code -------------- - -MIT License - -Copyright (c) Facebook, Inc. and its affiliates. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -------------- LICENSE FOR Mircrosoft Swin transformer code -------------- - -MIT License - -Copyright (c) Microsoft Corporation. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE - - ---------------- NVIDIA Source Code License for SegFormer ----------------- -1. Definitions - -“Licensor” means any person or entity that distributes its Work. - -“Software” means the original work of authorship made available under this -License. - -“Work” means the Software and any additions to or derivative works of the -Software that are made available under this License. - -The terms “reproduce,” “reproduction,” “derivative works,” and -“distribution” have the meaning as provided under U.S. copyright law; -provided, however, that for the purposes of this License, derivative works -shall not include works that remain separable from, or merely link -(or bind by name) to the interfaces of, the Work. - -Works, including the Software, are “made available” under this License by -including in or with the Work either (a) a copyright notice referencing -the applicability of this License to the Work, or (b) a copy of this License. - -2. License Grant - -2.1 Copyright Grant. Subject to the terms and conditions of this License, -each Licensor grants to you a perpetual, worldwide, non-exclusive, -royalty-free, copyright license to reproduce, prepare derivative works of, -publicly display, publicly perform, sublicense and distribute its Work -and any resulting derivative works in any form. - -3. Limitations - -3.1 Redistribution. You may reproduce or distribute the Work only if -(a) you do so under this License, (b) you include a complete copy of this -License with your distribution, and (c) you retain without modification any -copyright, patent, trademark, or attribution notices that are present -in the Work. - -3.2 Derivative Works. You may specify that additional or different terms -apply to the use, reproduction, and distribution of your derivative works -of the Work (“Your Terms”) only if (a) Your Terms provide that the use -limitation in Section 3.3 applies to your derivative works, and (b) you -identify the specific derivative works that are subject to Your Terms. -Notwithstanding Your Terms, this License (including the redistribution -requirements in Section 3.1) will continue to apply to the Work itself. - -3.3 Use Limitation. The Work and any derivative works thereof only may -be used or intended for use non-commercially. Notwithstanding the -foregoing, NVIDIA and its affiliates may use the Work and any derivative -works commercially. As used herein, “non-commercially” means for research -or evaluation purposes only. - -3.4 Patent Claims. If you bring or threaten to bring a patent claim against -any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) -to enforce any patents that you allege are infringed by any Work, then -your rights under this License from such Licensor (including the grant -in Section 2.1) will terminate immediately. - -3.5 Trademarks. This License does not grant any rights to use any Licensor’s -or its affiliates’ names, logos, or trademarks, except as necessary to -reproduce the notices described in this License. - -3.6 Termination. If you violate any term of this License, then your rights -under this License (including the grant in Section 2.1) will terminate -immediately. - -4. Disclaimer of Warranty. - -THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. -YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. - -5. Limitation of Liability. - -EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL -THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE -SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, -INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT -OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK -(INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, -LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER -COMMERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN -ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - diff --git a/toolbox/Megatron-DeepSpeed/MANIFEST.in b/toolbox/Megatron-DeepSpeed/MANIFEST.in deleted file mode 100644 index b73769311..000000000 --- a/toolbox/Megatron-DeepSpeed/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include megatron_ds/data/Makefile -include megatron_ds/data/helpers.cpp -include megatron_ds/core/datasets/Makefile -include megatron_ds/core/datasets/helpers.cpp -recursive-include megatron_ds/fused_kernels *.cpp *.h *.cu *.tr *.cuh *.cc diff --git a/toolbox/Megatron-DeepSpeed/README.md b/toolbox/Megatron-DeepSpeed/README.md deleted file mode 100644 index 976f30b92..000000000 --- a/toolbox/Megatron-DeepSpeed/README.md +++ /dev/null @@ -1,530 +0,0 @@ -Megatron ([1](https://arxiv.org/pdf/1909.08053.pdf), [2](https://arxiv.org/pdf/2104.04473.pdf), and [3](https://arxiv.org/pdf/2205.05198)) is a large, powerful transformer developed by the Applied Deep Learning Research team at NVIDIA. This repository is for ongoing research related to training large transformer language models at scale. We developed efficient, model-parallel ([tensor](https://arxiv.org/pdf/1909.08053.pdf), [sequence](https://arxiv.org/pdf/2205.05198), and [pipeline](https://arxiv.org/pdf/2104.04473.pdf)), and multi-node pre-training of transformer based models such as [GPT](https://arxiv.org/abs/2005.14165), [BERT](https://arxiv.org/pdf/1810.04805.pdf), and [T5](https://arxiv.org/abs/1910.10683) using mixed precision. - -Below are some of the projects where we have directly used Megatron: -* [BERT and GPT Studies Using Megatron](https://arxiv.org/pdf/1909.08053.pdf) -* [BioMegatron: Larger Biomedical Domain Language Model](https://www.aclweb.org/anthology/2020.emnlp-main.379.pdf) -* [End-to-End Training of Neural Retrievers for Open-Domain Question Answering](https://arxiv.org/abs/2101.00408) -* [Large Scale Multi-Actor Generative Dialog Modeling](https://www.aclweb.org/anthology/2020.acl-main.8.pdf) -* [Local Knowledge Powered Conversational Agents](https://arxiv.org/abs/2010.10150) -* [MEGATRON-CNTRL: Controllable Story Generation with External Knowledge Using Large-Scale Language Models](https://www.aclweb.org/anthology/2020.emnlp-main.226.pdf) -* [RACE Reading Comprehension Dataset Leaderboard](http://www.qizhexie.com/data/RACE_leaderboard.html) -* [Training Question Answering Models From Synthetic Data](https://www.aclweb.org/anthology/2020.emnlp-main.468.pdf) -* [Few-shot Instruction Prompts for Pretrained Language Models to Detect Social Biases](https://arxiv.org/abs/2112.07868) -* [Exploring the Limits of Domain-Adaptive Training for Detoxifying Large-Scale Language Models](https://arxiv.org/abs/2202.04173) -* [Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model](https://arxiv.org/abs/2201.11990) -* [Multi-Stage Prompting for Knowledgeable Dialogue Generation](https://arxiv.org/abs/2203.08745) -* [Evaluating Parameter Efficient Learning for Generation](https://aclanthology.org/2022.emnlp-main.319.pdf) - -Megatron is also used in [NeMo Megatron](https://developer.nvidia.com/nvidia-nemo#nemo-megatron), a framework to help enterprises overcome the challenges of building and training sophisticated natural language processing models with billions and trillions of parameters. - -Our codebase is capable of efficiently training very large (hundreds of billions of parameters) language models with both model and data parallelism. To demonstrate how the code scales with multiple GPUs and model sizes, we consider GPT models from 1 billion all the way to 1 trillion parameters. All models use a vocabulary size of 51,200 and a sequence length of 2048. We vary hidden size, number of attention heads, and number of layers to arrive at a specific model size. As the model size increases, we also modestly increase the batch size. We leverage [NVIDIA's Selene supercomputer](https://www.top500.org/system/179842/) to perform scaling studies and use up to 3072 [A100](https://www.nvidia.com/en-us/data-center/a100/) GPUs for the largest model. Each cluster node has 8 NVIDIA 80GB A100 GPUs. The graph below shows that we scale nearly linear up to 1 trillion parameter models running on 3072 GPUs. Note that these results are from benchmark runs and these models were not trained to convergence; however, the FLOPs are measured for end-to-end training, i.e., includes all operations including data loading, optimization, and even logging. - -![Scaling Graph](images/Achieved_petaFLOPs.png) - -The following table shows both model (MFU) and hardware (HFU) FLOPs utilization for select configurations up to 1T parameters (see [our paper](https://arxiv.org/pdf/2205.05198) for a description of how these are calculated). As the model size increases, we achieve better GPU utilization. For the one trillion parameter model, we reach a MFU and HFU of 56.3% and 57.0%, respectively. Note that these numbers are also measured on benchmark runs and in this case are measured using a data parallel size of one. Data parallelism introduces some overhead due to the gradient all-reduce required between the data parallel groups. However, for large transformer models, this overhead is not large and can almost entirely eliminated by overlapping the gradient all-reduce with backpropagation. - -| Model Size | Model FLOPs Utilization | Hardware FLOPs Utilization | -| :---: | :---: | :---: | -| 22B | 41.5% | 43.7% | -| 175B | 51.4% | 52.8% | -| 530B | 56.0% | 57.0% | -| 1T | 56.3% | 57.0% | - -# Contents - * [Contents](#contents) - * [Setup](#setup) - * [Downloading Checkpoints](#downloading-checkpoints) - * [Usage](#usage) - * [Training](#training) - * [Data Preprocessing](#data-preprocessing) - * [BERT Pretraining](#bert-pretraining) - * [GPT Pretraining](#gpt-pretraining) - * [T5 Pretraining](#t5-pretraining) - * [Distributed Pretraining](#distributed-pretraining) - * [Activation Checkpointing and Recomputation](#activation-checkpointing-and-recomputation) - * [Distributed Optimizer](#distributed-optimizer) - * [FlashAttention](#flashattention) - * [GPT-3 Example](#gpt-3-example) - * [Retro](#retro) - * [Evaluation and Tasks](#evaluation-and-tasks) - * [GPT Text Generation](#gpt-text-generation) - * [GPT Evaluation](#gpt-evaluation) - * [WikiText Perplexity Evaluation](#wikitext-perplexity-evaluation) - * [LAMBADA Cloze Accuracy](#lambada-cloze-accuracy) - * [BERT Task Evaluation](#bert-task-evaluation) - * [RACE Evaluation](#race-evaluation) - * [MNLI Evaluation](#mnli-evaluation) - * [Llama-2 Inference and Finetuning](#llama-2-inference-and-finetuning) - * [Datasets](#datasets) - * [Collecting Wikipedia Training Data](#collecting-wikipedia-training-data) - * [Collecting GPT Webtext Data](#collecting-gpt-webtext-data) - * [Reproducibility](#reproducibility) - -# Setup -We strongly recommend using the latest release of [NGC's PyTorch container](https://ngc.nvidia.com/catalog/containers/nvidia:pytorch) with DGX nodes. If you can't use this for some reason, use the latest pytorch, cuda, nccl, and NVIDIA [APEX](https://github.com/NVIDIA/apex#quick-start) releases. Data preprocessing requires [NLTK](https://www.nltk.org/install.html), though this is not required for training, evaluation, or downstream tasks. - -You can launch an instance of the PyTorch container and mount Megatron, your dataset, and checkpoints with the following Docker commands: -``` -docker pull nvcr.io/nvidia/pytorch:xx.xx-py3 -docker run --gpus all -it --rm -v /path/to/megatron:/workspace/megatron -v /path/to/dataset:/workspace/dataset -v /path/to/checkpoints:/workspace/checkpoints nvcr.io/nvidia/pytorch:xx.xx-py3 -``` - -## Downloading Checkpoints -We have provided pretrained [BERT-345M](https://ngc.nvidia.com/catalog/models/nvidia:megatron_bert_345m) and [GPT-345M](https://ngc.nvidia.com/catalog/models/nvidia:megatron_lm_345m) checkpoints to evaluate or for finetuning downstream tasks. To access these checkpoints, first [sign up](https://ngc.nvidia.com/signup) for and [setup](https://ngc.nvidia.com/setup/installers/cli) the NVIDIA GPU Cloud (NGC) Registry CLI. Further documentation for downloading models can be found in the [NGC documentation](https://docs.nvidia.com/dgx/ngc-registry-cli-user-guide/index.html#topic_6_4_1). - -Alternatively, you can directly download the checkpoints using: - -
-BERT-345M-uncased: wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/megatron_bert_345m/versions/v0.1_uncased/zip -O megatron_bert_345m_v0.1_uncased.zip
-BERT-345M-cased: wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/megatron_bert_345m/versions/v0.1_cased/zip -O megatron_bert_345m_v0.1_cased.zip
-GPT-345M: wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/megatron_lm_345m/versions/v0.0/zip -O megatron_lm_345m_v0.0.zip
-
- -The models require vocabulary files to run. The BERT WordPiece vocab file can be extracted from Google's pretrained BERT models: [uncased](https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt), [cased](https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt). The GPT [vocab file](https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json) and [merge table](https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt) can be downloaded directly. - -# Usage - -After installation, there are several possible workflows. The most comprehensive is: -1. Data preprocessing -2. Pretraining -3. Finetuning (Optional for zero-shot tasks) -4. Downstream task evaluation or text generation - -However, steps 1 and 2 can be replaced by using one of the pretrained models mentioned above. - -We've provided several scripts for pretraining both BERT and GPT in the [`examples`](./examples) directory, as well as scripts for both zero-shot and fine-tuned downstream tasks including MNLI, RACE, WikiText103, and LAMBADA evaluation. There is also a script for GPT interactive text generation. - -# Training -## Data Preprocessing -The training data requires preprocessing. First, place your training data in a loose json format, with one json containing a text sample per line. For example: -
-{"src": "www.nvidia.com", "text": "The quick brown fox", "type": "Eng", "id": "0", "title": "First Part"}
-{"src": "The Internet", "text": "jumps over the lazy dog", "type": "Eng", "id": "42", "title": "Second Part"}
-
- -The name of the `text` field of the json can be changed by using the `--json-key` flag in [`preprocess_data.py`](./tools/preprocess_data.py) The other metadata are optional and are not used in training. - -The loose json is then processed into a binary format for training. To convert the json into mmap format use `preprocess_data.py`. An example script to prepare data for BERT training is: -
-python tools/preprocess_data.py \
-       --input my-corpus.json \
-       --output-prefix my-bert \
-       --vocab-file bert-vocab.txt \
-       --tokenizer-type BertWordPieceLowerCase \
-       --split-sentences \
-       --workers 5
-
- -The output will be two files named, in this case, `my-bert_text_sentence.bin` and `my-bert_text_sentence.idx`. The `--data-path` specified in later BERT training is the full path and new filename, but without the file extension. - -For T5 use the same preprocessing as BERT, perhaps renaming it to: -
-       --output-prefix my-t5 \
-
- -Some minor modifications are required for GPT data preprocessing, namely, the addition of a merge table, an end-of-document token, removal of sentence splitting, and a change to the tokenizer type: -
-python tools/preprocess_data.py \
-       --input my-corpus.json \
-       --output-prefix my-gpt2 \
-       --vocab-file gpt2-vocab.json \
-       --tokenizer-type GPT2BPETokenizer \
-       --merge-file gpt2-merges.txt \
-       --append-eod \
-       --workers 5
-
- -Here the output files are named `my-gpt2_text_document.bin` and `my-gpt2_text_document.idx`. As before, in GPT training, use the longer name without the extension as `--data-path`. - -Further command line arguments are described in the source file [`preprocess_data.py`](./tools/preprocess_data.py). - -## BERT Pretraining - - -The [`examples/pretrain_bert.sh`](./examples/pretrain_bert.sh) script runs single GPU 345M parameter BERT pretraining. Debugging is the primary use for single GPU training, as the code base and command line arguments are optimized for highly distributed training. Most of the arguments are fairly self-explanatory. By default, the learning rate decays linearly over the training iterations starting at `--lr` to a minimum set by `--min-lr` over `--lr-decay-iters` iterations. The fraction of training iterations used for warmup is set by `--lr-warmup-fraction`. While this is single GPU training, the batch size specified by `--micro-batch-size` is a single forward-backward path batch-size and the code will perform gradient accumulation steps until it reaches `global-batch-size` which is the batch size per iteration. The data is partitioned into a 949:50:1 ratio for training/validation/test sets (default is 969:30:1). This partitioning happens on the fly, but is consistent across runs with the same random seed (1234 by default, or specified manually with `--seed`). We use `train-iters` as the training iterations requested. Alternatively, one can provide `--train-samples` which is total number of samples to train on. If this option is present, then instead of providing `--lr-decay-iters`, one will need to provide `--lr-decay-samples`. - -The logging, checkpoint-saving, and evaluation interval options are specified. Note that the `--data-path` now includes the additional `_text_sentence` suffix added in preprocessing, but does not include the file extensions. - -Further command line arguments are described in the source file [`arguments.py`](./megatron/arguments.py). - -To run `examples/pretrain_bert.sh`, make any desired modifications including setting the environment variables for `CHECKPOINT_PATH`, `VOCAB_FILE`, and `DATA_PATH`. Make sure to set these variables to their paths in the container. Then launch the container with Megatron and necessary paths mounted (as explained in [Setup](#setup)) and run the example script. - -## GPT Pretraining - -The `examples/pretrain_gpt.sh` script runs single GPU 345M parameter GPT pretraining. As mentioned above, single GPU training is primarily intended for debugging purposes, as the code is optimized for distributed training. - -It follows largely the same format as the previous BERT script with a few notable differences: the tokenization scheme used is BPE (which requires a merge table and a `json` vocabulary file) instead of WordPiece, the model architecture allows for longer sequences (note that the max position embedding must be greater than or equal to the maximum sequence length), and the `--lr-decay-style` has been set to cosine decay. Note that the `--data-path` now includes the additional `_text_document` suffix added in preprocessing, but does not include the file extensions. - -Further command line arguments are described in the source file [`arguments.py`](./megatron/arguments.py). - -`examples/pretrain_gpt.sh` can be launched the same way as described for BERT. Set the env vars and make any other modifications, launch the container with appropriate mounts, and run the script. - -## T5 Pretraining - -Very similar to BERT and GPT, the `examples/pretrain_t5.sh` script runs single GPU "base" (~220M parameter) T5 pretraining. The primary difference from BERT and GPT is the addition of the following arguments to accommodate the T5 architecture: - -* `--kv-channels` sets the inner dimension of the "key" and "value" matrices of all attention mechanisms in the model. For BERT and GPT this defaults to the hidden size divided by the number of attention heads, but can be configured for T5. - -* `--ffn-hidden-size` sets the hidden size in the feed-forward networks within a transformer layer. For BERT and GPT this defaults to 4 times the transformer hidden size, but can be configured for T5. - -* `--encoder-seq-length` and `--decoder-seq-length` set the sequence length for the encoder and decoder separately. - -All of the other arguments remain as they were for BERT and GPT pretraining. Run this example with the same steps described above for the other scripts. - -## Distributed Pretraining - -The `examples/pretrain_{bert,gpt,t5}_distributed.sh` scripts use the PyTorch distributed launcher for distributed training. As such, multi-node training can be achieved by properly setting environment variables. See the official PyTorch [documentation](https://pytorch.org/docs/stable/elastic/run.html#launcher-api) for further description of these [environment variables](https://pytorch.org/docs/stable/distributed.html#environment-variable-initialization). By default, multi-node training uses the [nccl](https://developer.nvidia.com/nccl) distributed backend. A simple set of additional arguments and the use of the PyTorch distributed module with the `torchrun` elastic launcher (equivalent to `python -m torch.distributed.run`) are the only additional requirements to adopt distributed training. See any of `examples/pretrain_{bert,gpt,t5}_distributed.sh` for more details. - -We use two types of parallelism: data and model parallelism. We facilitate two distributed data parallel implementations: a simple one of our own that performs gradient all-reduce at the end of back propagation step, and Torch's distributed data parallel wrapper that overlaps gradient reduction with back propagation computation. To switch between these two options use `--DDP-impl local` or `--DDP-impl torch`, respectively. As expected, Torch distributed data parallelism is more efficient at larger model sizes. For example, for the 8.3 billion parameters model running on 512 GPUs, the scaling increases from 60% to 76% when Torch's distributed data parallel is used. However, the overlapping method requires more memory and for some configurations (e.g., 2.5 billion parameters using 2-way model parallel and 1.2 billion parameters with no model parallel) can make the overall training slower as a result. We empirically found that using a smaller model in those cases improves the training time. - -Second, we developed a simple and efficient two-dimensional model-parallel approach. To use the first dimension, tensor model parallelism (splitting execution of a single transformer module over multiple GPUs, see Section 3 of [our paper](https://arxiv.org/pdf/1909.08053.pdf)), add the `--tensor-model-parallel-size` flag to specify the number of GPUs among which to split the model, along with the arguments passed to the distributed launcher as mentioned above. To use the second dimension, sequence parallelism, specify `--sequence-parallel`, which also requires tensor model parallelism to be enabled because it splits across the same GPUs (more details in Section 4.2.2 of [our paper](https://arxiv.org/pdf/2205.05198.pdf)). - -To use pipeline model parallelism (sharding the transformer modules into stages with an equal number of transformer modules on each stage, and then pipelining execution by breaking the batch into smaller microbatches, see Section 2.2 of [our paper](https://arxiv.org/pdf/2104.04473.pdf)), use the `--pipeline-model-parallel-size` flag to specify the number of stages to split the model into (e.g., splitting a model with 24 transformer layers across 4 stages would mean each stage gets 6 transformer layers each). - - - -We have examples of how to use these two different forms of model parallelism the example scripts ending in `distributed_with_mp.sh`: - -Other than these minor changes, the distributed training is identical to the training on a single GPU. - -The interleaved pipelining schedule (more details in Section 2.2.2 of [our paper](https://arxiv.org/pdf/2104.04473.pdf)) can be enabled using the `--num-layers-per-virtual-pipeline-stage` argument, which controls the number of transformer layers in a virtual stage (by default with the non-interleaved schedule, each GPU will execute a single virtual stage with `NUM_LAYERS / PIPELINE_MP_SIZE` transformer layers). The total number of layers in the transformer model should be divisible by this argument value. Additionally, the number of microbatches in the pipeline (computed as `GLOBAL_BATCH_SIZE / (DATA_PARALLEL_SIZE * MICRO_BATCH_SIZE)`) should be divisible by the `PIPELINE_MP_SIZE` when using this schedule (this condition is checked in an assertion in the code). The interleaved schedule is not supported for pipelines with 2 stages (`PIPELINE_MP_SIZE=2`). - -## Activation Checkpointing and Recomputation - -To reduce GPU memory usage when training a large model, we support various forms of activation checkpointing and recomputation. Instead of all activations being stored in memory to be used during backprop, as was traditionally the case in deep learning models, only activations at certain "checkpoints" in the model are retained (or stored) in memory, and the other activations are recomputed on-the-fly when needed for backprop. Note that this kind of checkpointing, *activation* checkpointing, is very different from the checkpointing of model parameters and optimizer state, which is mentioned elsewhere. - -We support two levels of recompute granularity: `selective` and `full`. Selective recomputation is the default and is recommended in almost all cases. This mode retains in memory the activations that take less memory storage space and are more expensive to recompute and recomputes the activations that take more memory storage space but are relatively inexpensive to recompute. See [our paper](https://arxiv.org/pdf/2205.05198) for details. You should find that this mode maximizes performance while minimizing the memory required to store activations. To enable selective activation recompute simply use `--recompute-activations`. - -For cases where memory is very limited, `full` recompute saves just the inputs to a transformer layer, or a group, or block, of transformer layers, and recomputes everything else. To enable full activation recompute use `--recompute-granularity full`. When using `full` activation recompute, there are two methods: `uniform` and `block`, chosen using the `--recompute-method` argument. - -* The `uniform` method uniformly divides the transformer layers into groups of layers (each group of size `--recompute-num-layers`) and stores the input activations of each group in memory. The baseline group size is 1 and, in this case, the input activation of each transformer layer is stored. When the GPU memory is insufficient, increasing the number of layers per group reduces the memory usage, enabling a bigger model to be trained. For example, when `--recompute-num-layers` is set to 4, only the input activation of each group of 4 transformer layers is stored. - -* The `block` method recomputes the input activations of a specific number (given by `--recompute-num-layers`) of individual transformer layers per pipeline stage and stores the input activations of the remaining layers in the pipeline stage. Reducing `--recompute-num-layers` results in storing the input activations to more transformer layers, which reduces the activation recomputation required in the backprop, thus improving training performance while increasing memory usage. For example, when we specify 5 layers to recompute of 8 layers per pipeline stage, the input activations of only the first 5 transformer layers are recomputed in the backprop step while the input activations for the final 3 layers are stored. `--recompute-num-layers` can be incrementally increased until the amount of memory storage space required is just small enough to fit in the available memory, thereby both maximally utilizing memory and maximizing performance. - - -## Distributed Optimizer - -Usage: `--use-distributed-optimizer`. Compatible with all model and data types. - -The distributed optimizer is a memory savings technique, whereby the optimizer state is evenly distributed across data parallel ranks (versus the traditional method of replicating the optimizer state across data parallel ranks). As described in [ZeRO: Memory Optimizations Toward Training Trillion Parameter Models](https://arxiv.org/abs/1910.02054), our implementation distributes all optimizer state that does not overlap with the model state. For example, when using fp16 model params, the distributed optimizer maintains its own separate copy of fp32 main params & grads, which are distributed across DP ranks. When using bf16 model params, however, the distributed optimizer's fp32 main grads are the same as the model's fp32 grads, and so the grads in this case are not distributed (although the fp32 main params are still distributed, as they are separate from the bf16 model params). - -Theoretical memory savings vary depending on the combination of the model's param dtype and grad dtype. In our implementation, the theoretical number of bytes per parameter is (where 'd' is the data parallel size): - -| | Non-distributed optim | Distributed optim | -|-|-|-| -| fp16 param, fp16 grads | 20 | 4 + 16/d | -| bf16 param, fp32 grads | 18 | 6 + 12/d | -| fp32 param, fp32 grads | 16 | 8 + 8/d | - -## FlashAttention - -Usage: `--use-flash-attn`. Support attention head dimensions at most 128. - -[FlashAttention](https://github.com/HazyResearch/flash-attention) is a fast and -memory-efficient algorithm to compute exact attention. It speeds up model -training and reduces memory requirement. - -To install FlashAttention: -```sh -pip install flash-attn -``` - -## GPT-3 Example - -In `examples/pretrain_gpt3_175B.sh` we have provided an example of how to configure Megatron to train [GPT-3](https://arxiv.org/abs/2005.14165) with 175 billion parameters on 1024 GPUs. The script is designed for [slurm](https://slurm.schedmd.com/documentation.html) with [pyxis](https://github.com/NVIDIA/pyxis) plugin but can be easily adopted to any other scheduler. It uses 8-way tensor parallelism and 16-way pipeline parallelism. With options `global-batch-size 1536` and `rampup-batch-size 16 16 5859375`, the training will start with global batch size 16 and linearly increase the global batch size to 1536 over 5,859,375 samples with incremental steps 16. The training dataset can be either a single set or a multiple datasets combined with a set of weights. - -With full global batch size of 1536 on 1024 A100 GPUs, each iteration takes around 32 seconds resulting in 138 teraFLOPs per GPU which is 44% of the theoretical peak FLOPs. - - -## Retro - -See: - -- `tools/retro/README.md` for an overview. -- `tools/retro/examples/get_preprocess_cmd.sh` for an example of common preprocessing arguments. -- `tools/retro/examples/preprocess_data.sh` for an example of how to preprocess data. -- `tools/retro/examples/pretrain_model.sh` for an example of how to pretrain a model. - -Retro is a retrieval-enhanced model that is based on GPT. As described in [Improving language models by retrieving from trillions of tokens](https://arxiv.org/abs/2112.04426), Retro retrieves from a database of document chunks by performing locality search using a sample's tokens. The retrieval database can be large -- often billions or even trillions of tokens -- and provides a more efficient storage mechanism of factual knowledge, when compared to storing factual knowledge implicitly within the network's parameters. - -Using Retro requires two steps: 1) preprocessing the retrieval database and pretraining neighbors, and 2) pretraining a model using this data. Please see `tools/retro/README.md` for a detailed overview. - - - -# Evaluation and Tasks - -We provide several command line arguments, detailed in the scripts listed below, to handle various zero-shot and fine-tuned downstream tasks. However, you can also finetune your model from a pretrained checkpoint on other corpora as desired. To do so, simply add the `--finetune` flag and adjust the input files and training parameters within the original training script. The iteration count will be reset to zero, and the optimizer and internal state will be reinitialized. If the fine-tuning is interrupted for any reason, be sure to remove the `--finetune` flag before continuing, otherwise the training will start again from the beginning. - -Because evaluation requires substantially less memory than training, it may be advantageous to merge a model trained in parallel for use on fewer GPUs in downstream tasks. The following script accomplishes this. This example reads in a GPT model with 4-way tensor and 4-way pipeline model parallelism and writes out a model with 2-way tensor and 2-way pipeline model parallelism. - -
-python tools/checkpoint/util.py \
-        --model-type GPT \
-        --load-dir checkpoints/gpt3_tp4_pp4 \
-        --save-dir checkpoints/gpt3_tp2_pp2 \
-        --target-tensor-parallel-size 2 \
-        --target-pipeline-parallel-size 2
-
-
- -Several downstream tasks are described for both GPT and BERT models below. They can be run in distributed and model parallel modes with the same changes used in the training scripts. - -## GPT Text Generation - -We have included a simple REST server to use for text generation in `tools/run_text_generation_server.py`. You run it much like you would start a pretraining job, specifying an appropriate pretrained checkpoint. There are also few optional parameters: `temperature`, `top-k`and `top-p`. See `--help` or the source file for more information. See [examples/run_text_generation_server_345M.sh](examples/run_text_generation_server_345M.sh) for an example of how to run the server. - -Once the server is running you can use `tools/text_generation_cli.py` to query it, it takes one argument which is the host the server is running on. - -
-tools/text_generation_cli.py localhost:5000
-
- -You can also use CURL or any other tools to query the server directly: - -
-curl 'http://localhost:5000/api' -X 'PUT' -H 'Content-Type: application/json; charset=UTF-8'  -d '{"prompts":["Hello world"], "tokens_to_generate":1}'
-
- -See [megatron/text_generation_server.py](megatron/text_generation_server.py) for more API options. - -### Detoxify GPT via Self-generation -We include an example in `examples/detxoify_lm/` to detoxify language models by leveraging the generative power of language models. - -See [examples/detxoify_lm/README.md](examples/detxoify_lm/README.md) for step-by-step tutorials on how to perform domain-adaptive training and detoxify LM using self-generated corpus. - - -## GPT Evaluation -We include example scripts for GPT evaluation on WikiText perplexity evaluation and LAMBADA Cloze accuracy. - -### WikiText Perplexity Evaluation -For even comparison with prior works, we evaluate perplexity on the word-level [WikiText-103 test dataset](https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip), and appropriately compute perplexity given the change in tokens when using our subword tokenizer. - -We use the following command to run WikiText-103 evaluation on a 345M parameter model. -
-TASK="WIKITEXT103"
-
-VALID_DATA=<wikitext path>.txt
-VOCAB_FILE=gpt2-vocab.json
-MERGE_FILE=gpt2-merges.txt
-CHECKPOINT_PATH=checkpoints/gpt2_345m
-
-COMMON_TASK_ARGS="--num-layers 24 \
-                  --hidden-size 1024 \
-                  --num-attention-heads 16 \
-                  --seq-length 1024 \
-                  --max-position-embeddings 1024 \
-                  --fp16 \
-                  --vocab-file $VOCAB_FILE"
-
-python tasks/main.py \
-       --task $TASK \
-       $COMMON_TASK_ARGS \
-       --valid-data $VALID_DATA \
-       --tokenizer-type GPT2BPETokenizer \
-       --merge-file $MERGE_FILE \
-       --load $CHECKPOINT_PATH \
-       --micro-batch-size 8 \
-       --log-interval 10 \
-       --no-load-optim \
-       --no-load-rng
-
- - -### LAMBADA Cloze Accuracy -To compute LAMBADA cloze accuracy (the accuracy of predicting the last token given the preceding tokens) we utilize a detokenized, processed version of the [LAMBADA dataset](https://github.com/cybertronai/bflm/blob/master/lambada_test.jsonl). - -We use the following command to run LAMBADA evaluation on a 345M parameter model. Note that the `--strict-lambada` flag should be used to require whole word matching. Ensure that `lambada` is part of the file path. - -
-TASK="LAMBADA"
-
-VALID_DATA=<lambada path>.json
-VOCAB_FILE=gpt2-vocab.json
-MERGE_FILE=gpt2-merges.txt
-CHECKPOINT_PATH=checkpoints/gpt2_345m
-COMMON_TASK_ARGS=<same as those in WikiText Perplexity Evaluation above>
-
-python tasks/main.py \
-       --task $TASK \
-       $COMMON_TASK_ARGS \
-       --valid-data $VALID_DATA \
-       --tokenizer-type GPT2BPETokenizer \
-       --strict-lambada \
-       --merge-file $MERGE_FILE \
-       --load $CHECKPOINT_PATH \
-       --micro-batch-size 8 \
-       --log-interval 10 \
-       --no-load-optim \
-       --no-load-rng
-
- -Further command line arguments are described in the source file [`main.py`](./tasks/main.py) - -## BERT Task Evaluation -### RACE Evaluation -The following script finetunes the BERT model for evaluation on the [RACE dataset](http://www.cs.cmu.edu/~glai1/data/race/). The `TRAIN_DATA` and `VALID_DATA` directory contain the RACE dataset as separate `.txt` files. Note that for RACE, the batch size is the number of RACE query's to evaluate. Since each RACE query has four samples, the effective batch size passed through the model will be four times the batch size specified on the command line. - -
-TRAIN_DATA="data/RACE/train/middle"
-VALID_DATA="data/RACE/dev/middle \
-            data/RACE/dev/high"
-VOCAB_FILE=bert-vocab.txt
-PRETRAINED_CHECKPOINT=checkpoints/bert_345m
-CHECKPOINT_PATH=checkpoints/bert_345m_race
-COMMON_TASK_ARGS="--num-layers 24 \
-                  --hidden-size 1024 \
-                  --num-attention-heads 16 \
-                  --seq-length 512 \
-                  --max-position-embeddings 512 \
-                  --fp16 \
-                  --vocab-file $VOCAB_FILE"
-
-COMMON_TASK_ARGS_EXT="--train-data $TRAIN_DATA \
-                      --valid-data $VALID_DATA \
-                      --pretrained-checkpoint $PRETRAINED_CHECKPOINT \
-                      --save-interval 10000 \
-                      --save $CHECKPOINT_PATH \
-                      --log-interval 100 \
-                      --eval-interval 1000 \
-                      --eval-iters 10 \
-                      --weight-decay 1.0e-1"
-
-python tasks/main.py \
-       --task RACE \
-       $COMMON_TASK_ARGS \
-       $COMMON_TASK_ARGS_EXT \
-       --tokenizer-type BertWordPieceLowerCase \
-       --epochs 3 \
-       --micro-batch-size 4 \
-       --lr 1.0e-5 \
-       --lr-warmup-fraction 0.06
-
- -### MNLI Evaluation -The following script finetunes the BERT model for evaluation with the [MultiNLI sentence pair corpus](https://www.nyu.edu/projects/bowman/multinli/). Because the matching tasks are quite similar, the script can be quickly tweaked to work with the [Quora Question Pairs](https://www.kaggle.com/quora/question-pairs-dataset) (QQP) dataset as well. - -
-
-TRAIN_DATA="data/glue_data/MNLI/train.tsv"
-VALID_DATA="data/glue_data/MNLI/dev_matched.tsv \
-            data/glue_data/MNLI/dev_mismatched.tsv"
-PRETRAINED_CHECKPOINT=checkpoints/bert_345m
-VOCAB_FILE=bert-vocab.txt
-CHECKPOINT_PATH=checkpoints/bert_345m_mnli
-COMMON_TASK_ARGS=<same as those in RACE Evaluation above>
-COMMON_TASK_ARGS_EXT=<same as those in RACE Evaluation above>
-
-python tasks/main.py \
-       --task MNLI \
-       $COMMON_TASK_ARGS \
-       $COMMON_TASK_ARGS_EXT \
-       --tokenizer-type BertWordPieceLowerCase \
-       --epochs 5 \
-       --micro-batch-size 8 \
-       --lr 5.0e-5 \
-       --lr-warmup-fraction 0.065
-
- -## Llama-2 Inference and Finetuning - -The Llama-2 [family of models](https://ai.meta.com/llama/) are an open-source set of pretrained & finetuned (for chat) models that have achieved strong results across a wide set of benchmarks. At the time of release, Llama-2 models achieved among the best results for open-source models, and were competitive with the closed-source GPT-3.5 model (see https://arxiv.org/pdf/2307.09288.pdf). - -The Llama-2 checkpoints can be loaded into Megatron for inference and finetuning. See documentation [here](docs/llama2.md). - -# Datasets -We do not host any datasets for GPT or BERT training, however, we detail their collection so that our results may be reproduced. - -## Collecting Wikipedia Training Data -We recommend following the Wikipedia data extraction process specified by Google research: "the recommended pre-processing is to download [the latest dump](https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2), extract the text with [WikiExtractor.py](https://github.com/attardi/wikiextractor), and then apply any necessary cleanup to convert it into plain text." - -We recommend using the `--json` argument when using WikiExtractor, which will dump the Wikipedia data into loose json format (one json object per line), making it more manageable on the file system and also readily consumable by our codebase. We recommend further preprocessing this json dataset with nltk punctuation standardization. For BERT training, use the `--split-sentences` flag to `preprocess_data.py` as described [above](#data-preprocessing) to include sentence breaks in the produced index. If you'd like to use Wikipedia data for GPT training you should still clean it with nltk/spacy/ftfy, but do not use the `--split-sentences` flag. - -## Collecting GPT Webtext Data -We utilize the publicly available [OpenWebText](https://github.com/eukaryote31/openwebtext) library from [jcpeterson](https://github.com/jcpeterson/openwebtext) and [eukaryote31's](https://github.com/eukaryote31/openwebtext) work to download urls. We then filter, clean, and deduplicate all downloaded content according to the procedure described in our [openwebtext](./tools/openwebtext) directory. For reddit URLs corresponding to content up to October 2018 we arrived at approximately 37GB of content. - -# Reproducibility -Megatron training is intended to be bitwise reproducible. This means that the same training config run twice in the same HW and SW environment should produce identical model checkpoints, losses and accuracy metric values (iteration time metrics may vary). - -There are currently two known Megatron optimizations that break reproducibility whilst still producing almost identical training runs. The following workarounds should be applied in cases where reproducibility is required: -1. When training using `--bf16`, reproducbility is only obtained when the checkpointing and resume schedule of training is identical. If the checkpointing schedule will change, i.e. checkpointing and resume will occur at different iterations, the option `--no-bias-gelu-fusion` should be used. -2. Flash attention is nondeterministic. If reproducibility is required do not use `--use-flash-attn`. - -These sources of nondeterminism are under active investigation. If you observe nondeterminism in Megatron training under other circumstances please open an issue. diff --git a/toolbox/Megatron-DeepSpeed/README_RLHF.md b/toolbox/Megatron-DeepSpeed/README_RLHF.md deleted file mode 100644 index 8acb68e03..000000000 --- a/toolbox/Megatron-DeepSpeed/README_RLHF.md +++ /dev/null @@ -1,31 +0,0 @@ -# RLHF - -## Step 1: Install - -``` -bash build_megatron-deepspeed.sh && bash install_megatron-deepspeed.sh -``` - -## Step 2: Dataset - -Download dataset and convert it. - -``` -cd dataset && bash convert_dahoas.sh -``` - -## Step 3: Checkpoint - -Download and convert checkpoints. - -``` -cd checkpoints && bash download_rlhf_checkpoints.sh -bash convert_hf_2_meg.sh -``` - -## Step 4: Train - -``` -cd examples/llama2 -bash run_llama2_7b_rlhf_node1.sh -``` diff --git a/toolbox/Megatron-DeepSpeed/SECURITY.md b/toolbox/Megatron-DeepSpeed/SECURITY.md deleted file mode 100644 index 869fdfe2b..000000000 --- a/toolbox/Megatron-DeepSpeed/SECURITY.md +++ /dev/null @@ -1,41 +0,0 @@ - - -## Security - -Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). - -If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. - -## Reporting Security Issues - -**Please do not report security vulnerabilities through public GitHub issues.** - -Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). - -If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). - -You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). - -Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: - - * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) - * Full paths of source file(s) related to the manifestation of the issue - * The location of the affected source code (tag/branch/commit or direct URL) - * Any special configuration required to reproduce the issue - * Step-by-step instructions to reproduce the issue - * Proof-of-concept or exploit code (if possible) - * Impact of the issue, including how an attacker might exploit the issue - -This information will help us triage your report more quickly. - -If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. - -## Preferred Languages - -We prefer all communications to be in English. - -## Policy - -Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). - - diff --git a/toolbox/Megatron-DeepSpeed/build_megatron-deepspeed.sh b/toolbox/Megatron-DeepSpeed/build_megatron-deepspeed.sh deleted file mode 100644 index e566fec93..000000000 --- a/toolbox/Megatron-DeepSpeed/build_megatron-deepspeed.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -PYTHON_PATH=$(which python3) - -echo "build megatronspeed" -COREX_VERSION=${COREX_VERSION:-latest} -if [[ "${COREX_VERSION}" == "latest" || -z "${COREX_VERSION}" ]]; then - COREX_VERSION=`date --utc +%Y%m%d%H%M%S` -fi -MEGATRONSPEED_VERSION_IDENTIFIER="corex.${COREX_VERSION}" -export MEGATRONSPEED_VERSION_IDENTIFIER=${MEGATRONSPEED_VERSION_IDENTIFIER} - -${PYTHON_PATH} setup.py build -${PYTHON_PATH} setup.py bdist_wheel - -PKG_DIR="./dist" -rm -rf build_pip -if [[ ! -d "build_pip" ]]; then - mkdir build_pip -fi - -pip_pkg="$(ls -t ${PKG_DIR} | grep "megatron" | head -1)" -cp ${PKG_DIR}/${pip_pkg} build_pip - -exit 0 \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/ci/run_ci_tests_multi_node.sh b/toolbox/Megatron-DeepSpeed/ci/run_ci_tests_multi_node.sh deleted file mode 100644 index 28ab706a4..000000000 --- a/toolbox/Megatron-DeepSpeed/ci/run_ci_tests_multi_node.sh +++ /dev/null @@ -1,16 +0,0 @@ -#! /bin/bash - -ROOT=$(cd ..; pwd) -cd ${ROOT} - - -cd tests -bash run_test_multi_node.sh -## 获取退出码 -status=$(cat exit_code.txt) - -if [[ $status == 255 ]]; then - exit -1 -else - exit $status -fi \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/ci/run_ci_tests_one_node.sh b/toolbox/Megatron-DeepSpeed/ci/run_ci_tests_one_node.sh deleted file mode 100644 index 3da0b9af4..000000000 --- a/toolbox/Megatron-DeepSpeed/ci/run_ci_tests_one_node.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -ROOT=$(cd ..; pwd) -cd ${ROOT} - - -cd tests -bash run_test_one_node.sh -status=$? -if [ $status == 255 ]; then - exit -1 -else - exit $status -fi \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/clean_megatron-deepspeed.sh b/toolbox/Megatron-DeepSpeed/clean_megatron-deepspeed.sh deleted file mode 100644 index 4c9753ba9..000000000 --- a/toolbox/Megatron-DeepSpeed/clean_megatron-deepspeed.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -PYTHON_PATH=$(which python3) - -${PYTHON_PATH} setup.py clean || true -rm -rf build build_pip dist megatronspeed.egg-info - -exit 0 \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/dataset/README.md b/toolbox/Megatron-DeepSpeed/dataset/README.md deleted file mode 100644 index 1f0aa31d9..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Run the scripts below to setup dataset - -bash download_books.sh - -bash download_vocab.sh diff --git a/toolbox/Megatron-DeepSpeed/dataset/convert_dahoas.sh b/toolbox/Megatron-DeepSpeed/dataset/convert_dahoas.sh deleted file mode 100644 index 07a3fe50b..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/convert_dahoas.sh +++ /dev/null @@ -1,27 +0,0 @@ -#/bin/bash -CUR_DIR=$(cd "$(dirname "$0")";pwd) - -if [[ ! -e ${CUR_DIR}/dahoas_train.jsonl ]]; then - wget http://sw.iluvatar.ai/download/apps/datasets/nlp/RLHF/dahoas_train.jsonl -fi - -PROJ_HOME=$(dirname "$PWD") -SAVE_PATH=./dahoas -mkdir -p $SAVE_PATH - -MAX_PROMPT_LENGTH=16000 -PAD_ID=0 - -TOKENIZER=Llama2Tokenizer -TOKENIZER_PATH=$PROJ_HOME/examples/llama2/tokenizer/tokenizer.model - -python3 $PROJ_HOME/tools/preprocess_data.py \ - --input ./dahoas_train.jsonl \ - --json-keys prompt \ - --tokenizer-type $TOKENIZER \ - --tokenizer-model $TOKENIZER_PATH \ - --output-prefix $SAVE_PATH/dahoas_train \ - --workers 32 \ - --pad-2-maxlen $MAX_PROMPT_LENGTH \ - --pad-direction left \ - --pad-id $PAD_ID \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/dataset/convert_llama2tokenizer_dataset.sh b/toolbox/Megatron-DeepSpeed/dataset/convert_llama2tokenizer_dataset.sh deleted file mode 100644 index 8098ab7d2..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/convert_llama2tokenizer_dataset.sh +++ /dev/null @@ -1,21 +0,0 @@ -#/bin/bash - -PROJ_HOME=$(dirname "$PWD") -SAVE_PATH=./gpt_small_117M -mkdir -p $SAVE_PATH - -TOKENIZER=Llama2Tokenizer -TOKENIZER_PATH=$PROJ_HOME/examples/llama2/tokenizer/tokenizer.model - -python3 $PROJ_HOME/tools/preprocess_data.py \ - --input ./gpt_small-117M.train.jsonl \ - --json-keys text \ - --tokenizer-type $TOKENIZER \ - --tokenizer-model $TOKENIZER_PATH \ - --output-prefix $SAVE_PATH/gpt_small_117M \ - --append-eod \ - --workers 32 - - - - diff --git a/toolbox/Megatron-DeepSpeed/dataset/download_RedPajama-Data-1T-Sample.sh b/toolbox/Megatron-DeepSpeed/dataset/download_RedPajama-Data-1T-Sample.sh deleted file mode 100644 index 494e7386a..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/download_RedPajama-Data-1T-Sample.sh +++ /dev/null @@ -1,10 +0,0 @@ -set -euox pipefail - -CUR_DIR=$(cd "$(dirname "$0")";pwd) -cd ${CUR_DIR} - -if [[ ! -d ${CUR_DIR}/RedPajama-Data-1T-Sample ]]; then - echo "RedPajama-Data-1T-Sample dataset not exist, downloading..." - wget http://sw.iluvatar.ai/download/apps/datasets/nlp/RedPajama-Data-1T-Sample/RedPajama-Data-1T-Sample.tar - tar -xf RedPajama-Data-1T-Sample.tar && rm -f RedPajama-Data-1T-Sample.tar -fi \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/dataset/download_and_covert_llama3_dataset.sh b/toolbox/Megatron-DeepSpeed/dataset/download_and_covert_llama3_dataset.sh deleted file mode 100644 index 432d6d9b0..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/download_and_covert_llama3_dataset.sh +++ /dev/null @@ -1,25 +0,0 @@ -#/bin/bash -set -euox pipefail - -CUR_DIR=$(pwd) -if [[ ! -f $CUR_DIR/small-117M.train.jsonl ]]; then - wget http://10.150.9.95/swapp/datasets/nlp/gpt-2-output-dataset/small-117M.train.jsonl -fi - -PROJ_HOME=$(dirname "$PWD") -SAVE_PATH=./gpt_small_117M_llama3 -mkdir -p $SAVE_PATH - -TOKENIZER=Llama3Tokenizer -TOKENIZER_PATH=$PROJ_HOME/examples/llama2/tokenizer/tokenizer_llama3.model - -python3 $PROJ_HOME/tools/preprocess_data.py \ - --input ./small-117M.train.jsonl \ - --json-keys text \ - --tokenizer-type $TOKENIZER \ - --tokenizer-model $TOKENIZER_PATH \ - --output-prefix $SAVE_PATH/gpt_small_117M \ - --append-eod \ - --workers 32 - -rm -f small-117M.train.jsonl \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/dataset/download_books.sh b/toolbox/Megatron-DeepSpeed/dataset/download_books.sh deleted file mode 100644 index cb93c2b21..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/download_books.sh +++ /dev/null @@ -1,2 +0,0 @@ -wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/dataset/download_ckpt.sh b/toolbox/Megatron-DeepSpeed/dataset/download_ckpt.sh deleted file mode 100644 index ac10274b1..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/download_ckpt.sh +++ /dev/null @@ -1,8 +0,0 @@ -mkdir -p checkpoints/gpt2_345m - -cd checkpoints/gpt2_345m -wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/megatron_lm_345m/versions/v0.0/zip -O megatron_lm_345m_v0.0.zip -unzip megatron_lm_345m_v0.0.zip -rm megatron_lm_345m_v0.0.zip -cd ../.. - diff --git a/toolbox/Megatron-DeepSpeed/dataset/download_vocab.sh b/toolbox/Megatron-DeepSpeed/dataset/download_vocab.sh deleted file mode 100644 index 0b7637104..000000000 --- a/toolbox/Megatron-DeepSpeed/dataset/download_vocab.sh +++ /dev/null @@ -1,2 +0,0 @@ -wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/docs/distrib_optimizer.md b/toolbox/Megatron-DeepSpeed/docs/distrib_optimizer.md deleted file mode 100644 index def23b20e..000000000 --- a/toolbox/Megatron-DeepSpeed/docs/distrib_optimizer.md +++ /dev/null @@ -1,54 +0,0 @@ -# Distributed Optimizer - -The motivation for the distributed optimizer is to save memory by distributing the optimizer state evenly across data parallel ranks, versus the current method of replicating the optimizer state across data parallel ranks. As described in https://arxiv.org/abs/1910.02054, this branch specifically implements the following: - -- [yes] distribute all 'non-overlapping' optimizer state (i.e., model params already in fp32 are NOT distributed) -- [no] distribute model gradients -- [no] distribute model parameters - -Theoretical memory savings vary depending on the combination of the model's param dtype and grad dtype. In the current implementation, the theoretical number of bytes per parameter is (where 'd' is the data parallel size): - -| | Non-distributed optim | Distributed optim | -| ------ | ------ | ------ | -| float16 param, float16 grads | 20 | 4 + 16/d | -| float16 param, fp32 grads | 18 | 6 + 12/d | -| fp32 param, fp32 grads | 16 | 8 + 8/d | - -The implementation of the distributed optimizer is centered on using the contiguous grad buffer for communicating grads & params between the model state and the optimizer state. The grad buffer at any given moment either holds: - -1. all model grads -2. a 1/d size _copy_ of the main grads (before copying to the optimizer state) -3. a 1/d size _copy_ of the main params (after copying from the optimizer state) -4. all model params -5. zeros (or None), between iterations - -The grad buffer is used for performing reduce-scatter and all-gather operations, for passing grads & params between the model state and optimizer state. With this implementation, no dynamic buffers are allocated. - -The figures below illustrate the grad buffer's sharding scheme, and the key steps of the distributed optimizer's param update: - -## Data flow - -![Data flow](images/distrib_optimizer/data_flow.png) - -## Sharding scheme - -![Sharding scheme](images/distrib_optimizer/sharding_scheme.png) - -## Key steps - -_(note: using illustrations above, and assuming fp16 grads)_ - -- Backward pass finishes (grad buffer holds 16 fp16 grad elements) -- Call reduce-scatter on each DP rank -- Each DP rank now has 4 elements within the grad buffer that are fully reduced (remaining 12 elements are garbage) -- Each DP rank copies its relevant 4 fp16 grad elements from the grad buffer into 4 fp32 main grad elements (separate buffer, owned by the optimizer); i.e. - - DP rank 0 copies elements [0:4] - - DP rank 1 copies elements [4:8] - - DP rank 2 copies elements [8:12] - - DP rank 3 copies elements [12:16] -- Optimizer.step() -- Each DP rank copies its 4 fp32 main (/optimizer) param elements into the corresponding 4 fp16 elements in the grad buffer -- Call all-gather on each DP rank -- Grad buffer now contains all 16, fully updated, fp16 model param elements -- Copy updated model params from grad buffer into their respective param tensors -- (At this point, grad buffer is ready to be zero'd for the next iteration) diff --git a/toolbox/Megatron-DeepSpeed/docs/images/distrib_optimizer/data_flow.png b/toolbox/Megatron-DeepSpeed/docs/images/distrib_optimizer/data_flow.png deleted file mode 100644 index d48fc134c40d6d0aae335bf765971b1181237d48..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90014 zcmd431ydf|7A*>d1PhQra1B9%yK8WFcXxM}5Fog_2X}Xe;4UBT?(X`Uvvbb9H~SB~ zsxByKYIV=KWR5ZB_z0Ai6oQAvgarcwgBKCzmjwfZ+yw)J=!Av|!p=;?C_Zv6R8!5{9XzCtjlvmyw4L;iJ9NO=1fQxzS{?|( zzbX*L>{19}xituN3yLQSiaVkTcdy_h$IE{(3ax z(7&F}(-o`tuNZ{eCxDSu@2{j;|D6-eQtUT>$G`&-N6ga&nkWkxi2PTk#;mu{|CL#! zat3hB(l}2}`LMq;DD3S|-utgnBD`~8#;j{D4>E24{d8a|-u*in|38=t9w}&jzAzsj za3z+R=y}7kC{K{vHMB}zj;DvxQ(>LM@?)_{RCxUvqamW9VM+HgLrnNp-g3*HhZm@m zc3SHscs|Oz8ybjv7A}*Ga7+nu;7*@#)q(Y}7Cd?Q`0|!F(7d3`mB--0hbJcj*5$Cg zz`T{PSBbm1qIIGoMNUi4GH=OW->IJCS(-zw_)-Bczc>d*9(K>UZ)ZGDG=2t8(mF)t zK2?S_7+9UA<7j*}!tM8vK?U9mRnI=D55Y`%Mm4=h`I|SEZ3lvN)M-0bTnBBfp5x^y zQjm+s3w3ztduX8rMtC3dH<7t!nx%DFihhmq7P^Gt;=`D|93HeY|<=D(|}oPew}c-C|mkW1RW{la(^#EEJ|(cPoK(zW~Hq zi;%%w5~UQXm>;%mro5}|#l8WZ?W&J<$cmoF6&q!NIV}e4F$)J<^?n-XTk%*{Mg_@` z3hC;VFf!{nVDVFB#1EHV)A|H$)za9V-9CR*sbxxg-*CiO$&|+(`JOU<+_ztO z-WK!b>>wQlRKncD4_m-rd$?Rqo-X!osaB<d zbcDXDk>+-3q9H`g`_yvq$70#1$f|tm#{x_qR}$P_Pn<9BDjUEW8hHO6dNV;Vcp@$JPnGHcGh)<>kt@J^j%Y#ezu2<=$8+m}6e zseciSWVPS7FNNyNTS-u_rJga7^A>V!MRRP6qJ~bqc}Vmq&>Rcu{UhDGPdR_}@Zm?9 zK5)v4+<{JEvdOp1hXPD2+bvn%uJof1`lsxxF~}eZo4tIK-(alwSkrGkS~$@TVSNld zY~@VkA5(!E+AH3cPkJ<(0ap!*Xy+v=cg}Q7c_Q{9MfytoChe*zd|>s|)C-K;a=_30 z_d$QaQuKjuY$2WK+}G{4N)q%SNQ+orrr5=nr&4p`_yMA)dk8OIV%0TRgqycnf!`-L zH1zDIJ(W?AxK1>zEwC^(x6a&fYGD@_Dx2y`S5ij~2XZdiNK8-P{vxu%M~9y!?>48a zu-j>Qxajt9L9c}$HpNH?dh#iOGH|w_i$1N9u8bK)6L_bQ)?-1l3E#h#b?5%g-9cKl zy6=z;Vkx8|QkDz%ClTDSH#buIc!4@wgI#aS1ARk)?Vf+SsJ~+_XU`Pxdrt?2n99gz zcJvU)!(T}&6%RgYcHIFN1w6qWzOtzyv^m9tc(yjqLts9##Jg8rUN+WqWLoV<<*~tY z`76GzoA8`aGO;%lgx6i2RSANw+2nrbeJF zM>j?1-Yd!^Pn8;&_|d|V_?{Urm*$Epmw}9M!fA;@W<_+Y2sfZ_yPhLTQlv>S1wOn_BBxzv^+nhFJ}=OSFI#S5t$ltd(K4L5D4%qZjto1jHWCUYE6R? zb388hdgh~SprKWK@i$${tapc;4nU-_j+&E+0^Pt{qUdb_=6AHvzB z0^bPJ5`HpQj?Iao%30tyLfTR<;gN$-y7*|mWAK$Vkf&Eq+|9|^V{qn(_Ob5iymVij z!KKtpzc@5vZEdUQc+Tjt^}soVvq46}z}J`M`+c^~fSXP5ePiU-&F+_sc)i2T!H3$C zlCZJ<7kGy2a8{tQJ3#*62TC~`iS^@}S|PD+%bd$&rl}`#mxHrjsvnz-z=2^nw>qP` z!Pf8M0+@97vwluDN6zN5k$nEtedeb^%7C7OVvsfiUSYVl6uAJ&dyt4=HxRSpuU`hy zL+81srrjaA=RamM3oW==;=pQ07A z@;hz|LK{YUm2vPeODW%p-^`)ogSP7K!EJ3v*_LaX7${Uq6VANZ6Ot;uAjk(|)fk*qBZA8Vi zLK@b}D`0(wm$L=ZO0SN!bIF$sQim?-figktna#AQD^_oCzdMa6ba4jFyWuiv;HHgk z_LK@I&+ogjMUp6AruXyEwg+Xm>UK$mC_q2dG zuecN~napaWFlhXdCdtj(L~`N-?GZL!E&kXMI+p!IOY{?lg=M&z^+RN&d+FC_rserb zT)i$_DGg-D01xRfc^WnWcrm?w(~z3gu0s>#uiK!tmTz@4a?IAJ7L4_JP8&1I;k^m) zLZ-cK^QWAOg?`Xutv%W54ik*Ir9H}@F-}+9GAiBq*h0UQYEaFF?q{hdj)g6BEc5*KV{bMn+E5*aX~e98 zunN%%>r;GPK74qps)9xv|9Q|kzn@F`T?{1ZR{)p7U3OP3$Pq}B#)^L3<;l)zJXclR zpymoF`dlpTz#B7e`%!^)cnaO#xC4uL5PwmCAx_u`5!b@Jq_y^2Q%iUDthW4mwk|sN zjQ=!8Oc>}XrKC*3V>+yraY`1YM5UPVp+?1IyM}n|3cYKr=taZtvwv~cys^d7r!;Y| zkh!C)51L#D$#k9qY1x1SOhoJX6iIg-Y!lOG;ZBJ`Y-Qc#Oji5A|A8kAK!}R@e&J>tIc$fVEUByDm&~-NFp#Q z=yECw^qJnqB@)i-i=v{K7i`e_e^$IHVSKLej7lCsx9f$~qeTgWiC@oNAD^9Wa?G3P z;aj?byiJ4@&hP<)pLs_(gzpmvX&QNL2L72iSiv`8DjwUMpwLQqdi3X&ALD_wtfdP_ z_Q2`B!_93A8KLo&3l*+&M!%;OUuX%PHOaxfi~)4GK61Z1{=_wj+Ew1B%wcE-@|7X1 zVkBsEow$ujjdAo520|mef|#$^knL9?ahVa_Vb8=;8102t404GBXAcXRVGxf4T0!mo zlqHLNI&&COY+jnUNyyr{eA$hUFB#p`yc+};N_e9x#`?u#aJ^G#K^HAk6vA7fq7}V6 zehw^g`R8F>PTRyT-mg!JcnmjlN;34lT?gJr9K$Y`Su~&w9_MAOzhYQs` z$|dm0U4dfZ~!@f~E=w~h9SnUk%B%6>vH=kwdZ$jYi!z>3mQqo}{) zVfb)8rLdwrvS*7huqImZQ*%b5p^rkB91$tBJ7F@8XGX8MV8O{@rs>YAg9%gFD5ORv zCOl6Ztj;4}p_cdn6BL`@GqpggC@Q!PCG@eD!{DcHDdw<2SEz+kG(Fi!4WhtOxz~*q zqJJ}Mbo+F4k$|(ZmUXYrnxYwV{jh!$j^3~$&k*X_(0mU)Tv)}s(4%^BOXuF}FAd_Q zNSDmC3s|BwzL3H^Km5sseEguB&j-lMTJH;U$B5$9bz3Oe7nz{RkAw?;s?m|8+m>LO zNyt`YjHF5?MX$EQAR}`nX_QsOUCecU<<)rC<)8-R$Gc$0p1f<=hp8Fg31YquDoVIm zb^~j>b9dk-JqTV{(u_akOc>oJ%WNcNj#v2h9Ph$0a?h)H`#2#+9U3y_*hAfPSa~tZ zy#lL>VZZAd5?fRR6@8s8nzQe0t(ZE|B!-Hqw?u0BdGCu*SPF{`uiS2lmB&E_s?8!V zEL#dlu1^7NNQ4hkuV6}iG%c(m=o+3jBv^~MTarhW5sUmQ;2AXETCfp++q_V_OW zs(3Rh1)T*#ZT#||48rYfkhmU~<5J;ydFh_>Y(aZT^-EO0-(V|T&l4U6C018XV@?V) zX^^N67{Eo?H7;qWq13$HP0~tTRwlxd!X8siFILE6E3qwqDx$zkW{(P5ql*2)?%Tm% z5o^u3a&)l<5fmv-$9Pv~$Qmua5wcj$EsfSTP+dFcR&O>#05Xd|{1Rxz$l z%>FzkID$vEt>W zUHU@`R}Y}m)Ue`8OC+5*hJu~=?)8Hvq=Aj|;NlYtH%2#pWr4-j1HG1i1AfjjnknNi^Z zx~x6|3kmPU3WV)>S^&*8xITs|wD2WlJl%BZKKBe+qrqA@$p+Oh&U>kuVIqJ7*h z+Wb|);CZTZqz!x0E~__Wdt#XORsZ6h`)wzB|LOwZBpJnTc=^NH{>9FUx>FUXXv?ZgPQFCXw z#GE*L2+`p3g4pUsKYx_ag|UZzuGXMF3-aqp7{-EK?@_D?Lh2WH7E}mgiNNvgeOxWn z9veR6g&-WG1*#xU)-QY*Z|^VY!Rujdk#BBXH7v`xKb*R0NJ?dY`uR51S-q{1dl$j~ zP~_)5M5grDU8g1hDh#fkhX(e^ZF9WuZbIaw_cQ6X@hdytp^c=iO6G=f+g~SfgiTp6$g9E&YqO4J}6tvE0=Sy^12&1k0TXBairxIQivAt2zd)^Lr5csW0w zxz|z3wTfm8(lTwr-NAmVn>;|&E&M&SJy-3;A6F9C4Y9OPLI9jVOi~OeTJ#(&l`lTY z)J`n43W_@egH}tP+WOYFsPE+@;~uU2Asg{|x@zE*KM{d{u=WF3+KEcj0IA>Nu|2Vt zwFY)_E%+ZojcP5FeQPHdkPE~$ z7b7_v#AlCAd)LjBsJf^G+4N>tQrF?C9@y@Np~#ox`$$oXGNn9Ue0aMn_Txekx6TW^ z?Sd$)lw*0E^oEY5w&`RLd-%y3zR2aXD3vW#mJMSVAde|x>eVx?XQWjJ%L#_8X+zk|!{F2= zJ27mH*|QAMneI2#<5Htd7PagJ$Lig$Np{+r$Rt&Dn8wiChnD;rJ<8eavXrE7ElS(w z#Vr`?0ZMX6#v9oh?h z`t~GO#|#Co?bhpMU%rFunCMo7-=uVEkjM2KVf02L88t*=afh=NdbLRg?1W?fU{2}_ zVC#DSQf7S`h0X&51}_2f2sWO0zKh0gUN!*WQ4AqdOcVXS){ zd=SAL)BhHCizNYh9JKP`>Th}6=PhvO00Ix}kn3+9Tsalc28-H0@Bd4_hD!l(=vywZ zS&zR%E@uN5M5(>UQkA~fup4Id_4QSaFRNbXTergyT_}VWKdJV zhiuk7Sy;YiDyib*++S^p@fUw}#U zX|Z;U|9iOmP#{=OH@&HU-2om1aF#>As;0juam)&s`0E9%tk%Epl(z%2vW7Nw$`a9E zEe0P?;1z0CmeH?L#qrmpe0=t~1N5(i3<9BkGvSy1{MJaQsU|VI9es`+MG-toT-%Oq z+MfIWM4yokjD3_>C!mpxgsx8XfY6_r;#?RWY;x8}n1rmBTFn>b!g^U;tTA1UY4}$L zlw*L&4sJy+1ib}Mu2)~TRip%chb)iAh;dg<@x5|j`MixP)FPlp-81?pB${v-{qGwi zqHmD;mDp9IoP`$?L@acjf~YC(YTbW;J@HFfvBGgF!chU9r97utI{9D&4hrW3g!UNQG zI|f>+NZ&-z2xVe2_*NZLLPQBDU1BN%Cn-CA^1CC$##AB&3bRq8B-eW&1xqf&Q;GeW z*9GquK)LZF#QgLi@78Wh4C#0@fAx4VY6ela;`tD(q-jDhBt%H_7J}XJ@U{cdGXrI@ zT&JrWO^d@%RQU&_I5fxYs&8*N-VEgP;pp>Wiron9N6R~7>DtYy29y9fl@jpqLyE!q z2>wOc5it&`P^x;t<4b44>g-=CN)j#1{M@IQfYRIx=s2gSk^ z6xZH_8tVm^@iuMd1O4*}{7bszVr0ZO_c<|<^>wQ6lG{D|SqbBi!uf(!V4!m7*fMoH zshzeA_{7(#B7-)gku<0UBJHht{k^V&+Qq0_vcoIVUmkZ_3gfOKyd^V~DR6;9^Fg^l zUf3VfRdSVm6Gg_v&L&HMRpH5pjKFnr$Y@_)1B#6)C^n3!U*b zD2+7aJ+yxT zdIs}tCt*o9X*{3FzYut?@DEFhFFSOkyd!~uu9cmP3PZ2ePhJ20L#^|;K$LYTYw-6; zV*cb-8KrFzMH35-EkjoB>3Kag0-E1@Hu0EEtv_3B%nZV`mW~Zo;h=YsL3Q_X7GBXt z$ZkD6h?>gp*JjlBjq(I>yN$Td>e16AzcJ$M1?fc26l2U}S=g766*MblMA$*=R3SsNOq-K=Qa*B zLg_BO3TrRXU@*$@U>bfU50?3Jhsa-bT%4t`1&#DlL>&eYHY#c3euY9PFLx8zh)bk= zEgc@u%F54;lTQunhf>5@M^CwB1Z%4M6yatS=LThvBXl(P?#+7I;_t&O?cjf6g!=)O z%i0k*{oPr8gOiGM0Z~pBrYfoO;{s71whVg?i-GUvsTdu>u^UmEjHIC83}Fc!<;o5Dp#M}3?d6ZB{veq6ue!{e z+3In+lBpK`4hzFu>ysg!rr%iJbU}9i$d_dY0w6{FF1j1)sQjBw8O=xnd6=*%AM%xN<7b>a!{pl$VJIP@ z_cO%i(d&&kk9Oj!9o-^W#n|CeF`z2~#I$d5U#;#h|wh-7gX2!T-SXdL?PPld#- zCPC_=oNT zG;a9n3yOfBl(Nu>HkCj>9Fh#SFHO-PsNX#nn5&U+m+U9n+(uas9apHg`yNB+(?HTr zUuyS)kpB%4X7!p?J-!f$iB;&Wp$?tO({6-!*%K=c_8<(ufsj1!Pt)B(Tuq6eMh%A` z^D)C9S9iKaltml}_t1o45lU3Ny!SduecnEk??&y@!N4` zJe|d+D)b<_V~Vx3o}8%H`TNFOYQkB(+4@!L6#N`TvJHI2CO)6t5r(;`(_2{Pb~L9o z3i(l8wuyC<3^fr&M+hs&#S?}1k@uPhgyurq=k8Vy0sVq=kL?fiN4fABidf3zh zTs^}ySeW4RvzvfFq>VpbCxnI4vFw$?IsO03H`=QK@S2Lsg_>puQNi1r$A=SjMEK5E zgBQj``7ZS$O!mKUd6&pXR}+@WM*c*RQKTL%57LO}hnx1RjmU?U=yd}JrH2C5>W>|# z{SX;FR$^u#2<7{g)-`}XAHagtgI_?uOosJus|>ym0>Gzg_@qDm zO`5=w60iWldB>k!?QawhbTk3~&&bsisI4YnTNx#o|i0I$2ib_fxTS-M)9;?2DYybQ{iowsqjGS~w!6 z>((6)4AzpEG>F%5Tz1(>3_AV%zgUZr`gY#F{2EuWhzSMi#SkmNPwbvy^Lb_1aaM|awMS=5J83L7 zzZCv=000R?hL|fhMP)DT-%P0<6$Dn#wm*=khP4sJ|HEOlr=bs5+FlOkdc~92J?`xb z>@O$8y!!cbFKzcE_^jA&7VRYTm!E6RESrv)kbU;JJHyB)!f=off)_ZS?v>xbMDTch z05BNdNucWg;WzCuFxUXVf8c&RjuMlqDO|f8MNM_FI78g>Q-tU%d%Qf1HVUWUx)CQ& zobhs>{)L9RL`-j7Pe3=>CtUI*k*vScs@O;aP-^W~fIhOVEPs9A`#*-RP)O8tW^LO6 z!B(f~M7`i)NbP!DF1Iu46xv8u6=U<-Ue7m_CkXpppZQ~QC<*=T8 z`fnkqTP`R1>Z37A+1XwmFG?~{91^4;t+nEDXO3RA_SMx@_kElP-M0`cuPh_A(B+Zp zA9=j;yJA(YFKVKQ6@nuLFN00g5#h6T}aK76w$XV8j!FnIg^7>sUs1 zz8n|2g0nhlz8FciY%I937*Y(SQU7=a{3@I;n8Zk8OPjCvf~v6L6~o3FG!eTSo>iBC6@I&f4UXFoDc zahV=&I2zG<-qK2A&61>}R;knvXnoIJ$5!Eh)GiA9`Tw!od0)Yxf1%%-$5aX`S$S&; zXdyHP5O(Dou9v&xBIS{iK<>?QR%N%l(Xf`3&Ka)(=)@Gmrae zkGb_M6z^$QNcma4*_qa~>3+Yctm(Kht^PP>aLdZ`A{pO*U=8gOuI&^KP1AV|kUDCM zN*hg0=PRd*KWZ2MRN));|3|(+h4$$=QY_g}t1B$BIlO-R-Dio==dGx=;CrRnoD)iGn%>>cn=X$Dbb)aYQhtrgQfT^Za zA2e{_Ai>r8DL!k2q|5@T`dpcg0C5!#>0tlug8s0;!?%OvtB%{7S}62M9xbnuTv-q@+tZX1+4e)K3fsI>$=ru^w_3P zzbk6HFXFz>;->pJjoa~S8WD^ER!<1+s{LF%Z?e2c2c3DKdSI@}=>!jWOxL^>k@q9s z^J&0Bt;NGb+l!m>JR-*_n4_!2kDZk)M2}rQYHYeD%{ei>Wp$jcq~%TD^sU3xO~#z)hK(*W-GRj2V@ckyZ> zy1hv3(LWW`NW81IXFvIs*)F}KcrSSvL!gNlGNI@|fTwuybhDyG;@xg`x*98E zv8(@X#`9>V+-k8wC-5sA-d&&2zN3l^y`!Z}zm(@W@>E$}aO!f~U2@yE%{s5gUaubu zRe_xshmoy~I%l5n@Dl1#2n(@P;;3F)^5OpptbH9)^{C#|RBsaHqkkf>n77G&FknU? ztH%1^95-;B7Qxw(>yq8~<>79=6_ z6NCI)>Z^>QrT{my#a4vkIrws4_TqZJDS|C1Fmnp!DB-o^SZke z55bSO5ftR1Di5Kgk3PH0HKcus#_$Xr1D(!NLus6D+U8EPkllvS-n7~!ck1lPsuh&~ z0d78iBHEX;vp|Kby>~iqsPf4H$x(WV_mBmQGDUVwg70+>78jw7Gtfm>y!+9{I2UzP zND2E&o+H{r4(q2~dX}A5JnGqPwgzHZmS87k^1Po!_yz7mS8n0W#rE@*D_rt%4*A`` z^l90T3p8?E3s>O5-QBdkJT=0daX(FR6CnjwjVi9xI+#4&U;S*)HtJ;aYdZhn*zq2w zj%8U^Vs`(>cs}>tY%kZlyB_3hW3=!PMs43uc}5kag+{tv7@`Dg>+mVbPopo7Qs!&Z zx=Xi%!`X$T9Q*n(=t>(j zEbE_UE1_X7-}+$Cs)s6Ke{FB=Tqb4uoa~9!3F`-6>vm%qiY6{WIGgXFSR{$ui(u7q zHSP7h<27wtZgAfTN6nzw^D}hd<-zNvOcOTJx$PcQhUsK>pou8{4V%2XL!J_kJx8t9 zJdhdtLa2rXwkY23bBn{lbmQ~BR3-^o2kn$1iCZX}Hoc3!KL6l1zn4?Q7wI}g^G|}- z#EVX@K=s&v=%4lPzAkR4QJmR&eu6G?=nhg+EM}Q6gbU~Pyqlf<1uANr5q;It{C3is zHu>;eOJl*#ffuQhs*0V=y9hv>d9M_pmtkUOp~h-&JeSEg8oX%ui|oIM%$pS)o7B{- zz7~bXN;fy%QMX>V7v;scY2>m=`-zZ-1k&)gwZ$O}f*(_jDr5i4^~!UyWxXkD1hYzo zZ?k1RFRofrs6TnFPN*sV_sBp{p-89w;|h?FgXC(h`-ynz!sBKtrXQi8PIZR1>vv7{ zh+MbYnPY5VmNjhk;CctjWf(g?62*!Vhf!iwKAk%JUS-7Q`<_n+T5Wr|$(N$rH+bc$ zR!#8R>KB)taA3n2(wZNxH*CnyMhkyEB%yD02bNeQ>9iVB!!;O>x--Y7Y>~$M(nrNO_j$`Wn&gDY6!{Izs>o-gDeNCB% zL>Y4YZ_m{?c*b*&w{P6h9-ch;$)@Q(j zXu+Yp@(3S4VeOaaCu>gnJ(qaXlHr|+J>9RoXmXP?R{r zRs>S}Ce@g^Tf~-(T%mgEj@pha}nww_9?{Qq3IC4;!|4dbuLR>P-Gqm*EoIgHU?8&jQ8CAx;H=1ycGl zpoD7))WemBmIa%`oPsxzimzumH%A)RUB8f;0^OEkImY8@*HN8XjpGZjw1&0rbMjR8 z+TDRfa1m+Onok!|wLBeJF_#sN1l}llZYecJS#f*|I+g6fENy#sy@0*Sn^Ct5l{rr| zN~87sAys*%7eMv(gP9vn*!cG0{*OQz9Tvhb&(;B)IkSf!+JnlLrP+?sQw8T>|1h5+ zdWgH`rVCsE2O9(+t}Hfi5WUjV@9pM?^#l`9tifzeBkoNo3ZVXjkAYW;`y(*`0A5rF zf7(M)QU~ABeBw=NG;UvcSkBhwj@R;ZAp!BK39lohDR#0re$VFaZngZx9O)4EAwd-F z$GkN911J?GXUhCT{q1@nTD98^Wz{(0!haP77oiODE|eA_YyRa7t`k9U_UbVhU|NI7 zjT$=_OWgIgRL#LSSDc}wvdU?pn8(&}F*&)SakrJCJH}PAtkrzg(U_RKZLW$c=68zv z)khVj4nA2D^*pM4DYxH?T+>l^$Zqzt=|M>@y+lo8yfy#%M|AwqZj)SkZ+m~~@>45O z+)?3YCorFJ#57p`2ZwL+t5zB-e|)6YWL7_*P*{xAI}8?)Z)v1TCYLrcl!8)f!zw*> zpYCGOyK!UtS}cbew?AEi#pn^$j}I_3560J$dlg$e5?{ToXjHqb1vKr-aKDuwZ=XKz zL)X+$fG?hvroS(2dT6g?lhUd{a8eKO7hF7UUedf@zCnfZZR31$L^Fi1=h3v|Hg}t~ z&^gyB+X-raWKcf{bDcM@9l68w({ejv)az&rPvtk92ugl2(khT=QyNvTq?L?qXa=RN zv)C^nk1I|#lyI$Ct23*wQL49?Wz5!odTaO-?7dPpMqvsl3JiL+(#96w2ZP9)CyHLR z?Jarl6e}IwRQ~KQ0juUSK}z+(ryH;JEIbeC;N@_&Gp?t-O>1 z2Hj!$C~0w?jAaI^jXNLJ5dAbYCa}KDx7A|)xLb}ADL@=+T#~|ZHxF=!-*v55mL((B zEVV8d>6nbnCva)C`@|Q2II#BT5ABXcwXv#85|ntXbh=e7wH6DG8xyB;Qcq+jDQeKZ zJ|60QklT4{s!kU*4NJ;5_o1B*nwk!d#pAyj@%S9LdK@|(N~Z2el4qJ}(gDUX(Tmpx z@uKRW)kJ*!{9t8Dl(Cfzr;X#3R3Z=hP<@`uyf*MFX|2)iX1=0=WQ1J`u@{t!lJEL- z6&=Sk#NM6v2d&eLLa)Zauk-JwcIL1vv-AkvGYnUu4If7UEDlK5vpboEs!lJFNF|;xV@X0@p3M z|HDHKjk&(`({+H%1)|q2ALYx!%CjR0h)N%${`(E!J&FwPyT7W5uXU^A#IAK~fxE-0 zJozURRT8UxkryXzcy6)7lTx0xRJaMy#@)DHkG9eDbV6aXQ19M}lkK^X$E%;ChsBvS ze*X1a#mECWG7<1m9d4)UQ@s`}OSAxjGGT(3JdcT(!s_Sqyg(23%E-yjm zwH`HJu2ih7L%P9)9^TU?rgKqMcJdqUO++nu=JZ7=(xGYCuV2k*p_#bk zxR0u-RPY{NN7^=^Htqp+{echfe!gL60G?TWR#apg0I!#%uao@h1lvh-E5(JpjTFKi zN8qgW5Yc*=p9!Hmah+)j!F#2qhgW+6+B%U0TyhJ-moMkkFB7#(H$d%qc^rKy4(0^P z#JF*_1t~N(r|T!IS__BW>_V<6fNb0)Pv%j(KV1c@?ln7~RTPzZuC+dtS`yKa+|j68 zHCIgAa#1^XrM7Ic%EX2K109CQA!yVoYtk0@>Q4hEJ(AAvVw)UG?AhFuIg0>Yr5x5r z^}S6|{^S=<>#qIz8ndf`SJIHBGF$jDg@38aDE#@8nxg}I>8a=4EBe*|gVJ?6Szan+ zqQrwzY&(xFOBh_;d*#bmWovg@F9%)@&J70^bxZzVD(e<)2d|G7ye_lmEBb4Phu`?{ zu6gWTlBhm)O;*qjULm!1V42-CkS!fINmZ&0l8X5KS-Kf*(Cv99&d$6BRIoQQ2U)d* zW5E7t*a&(K;LQxcbrpsJYW5kL*>rJ1aH)L29?*0i4Pjn<1gNjg7oo+zX21=|`7)XQ z%n%|N4L=(0aj2L2o5BAKM$D}@3`cbs_yQNu)vsSE1FkpX3qZM0EB807vtI>yPdra< zJV%?M(?=-5hjRC-cYSiKB`^_07f2w8MoJvbQ~%TvBp&E5#Wu-KR(lh0FV|MwW>k8^ zc?p_V5=xH=okEt4suvP6Ve)_O#IRvqR97CH(9aUcU5t zJqmFmFO&d|9-}WRi+zXIESHbK^08cMbdWgd24sdjulu}L2#ViqK3CaOxNm#h_Tsrg z$wj04vMKeP4N|B$b$=c*;;I@i_k!{ZO&+pH&T3Ej;2jFvr69Fcc^Y)W>hjSXXefC@ zqGEy<2c~nltl5ovHIHCeegyL^8-BE~c&DzBL!twCyKi3lqZg z_>{-a^)NTOuM=?!eO%-9a^n?FUjcA|aUvTp7|F!WmHLlFBlVq~4BID*4R!!u1B3{V z=er#(k1GZFpJU%Dln;*p{I|n(!z#RH)GzA19_paA z)ZrcQzrw*`ljqK8IN@9eTouMqs6g=D00vc4`2<$QFRPBhI>n?`YczZ}YDYvec&|(| zAsmt{HaN}exmNK-)4c*UxlXFd(q67Bmt0qb=2hIK2O?5HKY^($e?{ZH1@cdr3d*sw zIsD&Ef{r9ozH~DtJ6L@}s7s=p-)Mf-s65drpd;@r!ws<0q*2lzM= zaDUtZCJwN}3ohAKI|W*0?XvxytU5~+pll(bHo`fY|~#`A2xO%yqwg zY^VFhwP!wCzP27rttq`^WMt$6HeC~rCv>Y7TOPDzpXz}+Rj!H*O<3VtbGV{42{-<5 zb5xZS=8-kJJ)B%aiTv0X!cX9l#f*}A0=Zqa7UUElW}pq-u?KA4mL4S-Hx!*{Os!+! za0`#Fn5D3A&)%U>t=7KdUx~ywrIzc{y85kqjkT_BM9I zo=vC8^Tch1KBz;}!c`F!=Lg+Im#Ki4Sk6QyUqE#j;4eO50tcOhm(zrhPCK(Bm@7T% zC7HZRuu)t+p3fKQqxN>iUf@VyQG@Fz#1HE4(4HZFbWz>>0E-)3yD3u0Q9k&{=}GZn!#r$-8?liQN~1kAe4ZHg{S2#dnnV;29K^+(g7U$++VXjvIB}KLO6w~d zFZoWdLywGAwP%m1*}AP*q)75i-@`>#=mESk9D8l#ksqKk0z%y5j@L6uO2paauV7*- z%Z`(T9t%x9cxVEJub;R8$kMjMdE(=;sLX1c(f+XV;<@Whyd4Ufug<$*L=Mo7jqpUd`5YKnyk|P6M0e6D2MXDr2M6bPg&w}wRP`0G= zkt2~8ut%JT!gE_dkjzVf{H!JvhTWkvmHvF0Hw*3O7uc2aa}#n$2N^z@>$_jz4~0=c zphZ&i%SU!3Zteugf#+MUQbAwT&tZ{CiW3aTSMK#mfKr)`!(t6KbN6`L%iVMQ;r-#O z96M_ufLs!C+6b$o!EEt8(M}TVp{QKrYE`^kKT1}(`SZjDIqXs15Ump#qDR~Rwf zNdZ$&_>eE6h13&9eyw5BG;-$#IGX?SfpQN~u)0IR9#13;U3+77oU%yk|qIYFr0FTdK-Ysa}iP zp;f1GX5*~v!c6^;w!+aN4-<`$iqVvAR*BcyfBbmWR;ki2^5Ait0&DnNMMkB7J3N2d?jM+f)x zETf|*7?SVIn|s}OJxh_qwmS;o^NSa$Qt)FBj+@U&4#)y5`agESA(%zS1q&rDkD7*% zvuiFpkppf}Y0r50nsKT4v{EuAfFKfpMv*B*KsbPu6c6A5X?-6}%RFZVt+eGjR3P}f z6!((oYI{yRmn%pC&+D)P=oGzNN-)=07?_X$awt(_t+d}mD{C6b4r~;fwecG*_oRRt z>AFq!Nn(Q%7Wro$ks&PLm7*)3ZlRM)8`pwxlvohE{1N~;IKvD0O#RV{Q(XpQ+wSkc zA}yC3l<#&@>A36%6v#fhj0~-T-ij9tY9={pB3#?@;-x)25-g*Gfzv?lv?QA*gf<~NWdp)!pPdnj8xgHJ`2@G!GU`o?vqRl6#M99DHUfhiUZH!z- z`?SCfrh{n-@q$_VE%$j=PSGgux38b^Ulq#~g-hbm$RJ(}^G#Lu=Tjfk`=0vR!uK&j zzMZaXt=UO`?q_A2FU-ZGR=pfpT@8G>4F?j#Z9l@=M#JWMhGkV9MVjVujisljl)|+n zn$e{=!g;fpY}MMJK$N-zD8#+i)$}ag1k4)jBtjxc9=#9zh6xZ@|DEd1rwv|mWA;q^qKy;CFE=Jaq>^m5Q<>Gpi5 zNRaOO^00Px87PEjK{TdA4j4#{75J3yX6?H*SZ2*YNX!~dzl3y<$}xQe6y<*2180Kv zFA$XzNZL3+BU#r+D(+ZmQQkZ8&w?+HfN8hJ!7K!C&n+s@X^mj5I3XN^dR>3@Xgsk7_FINu>@c+Q&Pn{%Jqki-LKIC&*q^JDBCYB?U~h z+KdqPD7EugDr5JLo{W^nkp3eZ`Unt=7$;HzJ5x?=MRXO0Lkx@%cqO?7u?D0_yFc1V zLd_&oKV`g!u@})MwzF?OYd>UVG;kwZpk1&6nSllvO1S5qE?lR$E1peQUI+~ zngifXM%ZkbSS%pWTJMI#gE>6AQMMVEye1_kKPodY`4i{P4Y2z9ZnI5s`mxK^fg?86 zg*8q5N8@;%F&aGRcG>PLTh-27a`>6MQ`xzIw}P*>X1BGAUrp&AryVZi;l!z@t_n0d zPdyF9NboAk0*3!3BWJy6-eA-R9&79EdN zkJ*&J{Iftf39L82!yRi?QrQgqJ%(8L9F!PNrQoeyXC0M68~8Knr3T3yMjturG0+ugiz z-=CbSJ9BM*pOH2lvRFi0+uVF8DG9*r%6*X9P0OxkLSVJ#Yu{4Ccx3))M?T2y9=7pX zntI;wvm3o=EQo-8{5}M%tZT0dP0#@i8oU_RYH?OZ%cZC0i}x)HI${G$AAC*cg+db_{9@>89$^Kgqqn zF#G@LddsM|vS@1*NFhaVw?J@rhv4q+?(VL^o!}0^-Q5Wm65KUda1Rii0Pm!`)3@)t zHDmvCz~41Qc$2 z2EvhT8i1(1VU*#xilA6BPX!8})Bc_NNjgRM-;98~*~DaY%tu89gv||WQ2-qA#$qD7 zZ}vA2ptrbHmdp?lxNFTrK6zMVQ$Y~MgzL^S(H8Tzcs1s(cYoj>2mtzLbB}ZwLi|IRWwFlLwx`8K!>rtB#E{3dhf23jfFJ$-)_OYbvy1 zg#UIi2EVc3X`Etw*gS!bnlbA@vbTNGIR#n&j|!7bK|_+lj!`Hx7*ridZW+T7uyjL8h4fi+a0W0k!EyNU*c>+$xB692PQOS#^* zZSNgP8*YALApQQ0jmPpKRt*Qqf(<*Kb|@*yVj63$xcSNc-WViW8KolK!u~mc0}3$v z?SD=x^Nq=-vIW^_Vut4k5Pu_#etYx0S5QM%$?N@ZAPV8YnnAxu>U6=-2tnzgZW=At zI4Y(8v-=oOOg)>vdIDY2AO|cz~7ONB6(dp0kDdPWI!&Lrt-NxycKf)`I;$AV4Hg1 z3z)!~5fHyZ4ry%jZ-vzU4nqEZHID(-f-5o_)k(SO$faJTnV+K5;c9h@SqB0)5uc~F zc8L+0?B~As$G76A2s{IG*XAiY8Cy_g|a)=C7EMR!%lH zuE4CAft3LJ6G(Qx5Xq#{5d(5W`i*H~KDHE*`|m0w4aLEm59&`a8aVZ?2AN7LRoNhK zF+f9WB^wC;=-MB_02Nf?-m-qypp=w<^!1Nr0xtAb6tExt1XI5Mbxwbs7OOB|)nW|I zivRCh-6R0FL4!;S`HxNb_Z$DC{QtV1|FIi@4r59!{0DCH&sQ^%{&pE~>gv+}5ysyl z1A;M1P0XZ`qAg%w`wtBbp)dsyK4Ie-sea_l{8;fh*_fOYAKj8lA*c6e> zknDUMDA>UY+y$ooT)*vE{O%Z!3m@B|v)N#Nhl*64NqCbIBgGwPq*KKkfz+x#_0;5(0*E8UV0)W-8oE0Na@YV~@t zOxld&Dc=^f0V%>FNS{xfw=sqS*!}VpvL#|M3etJSt?MA7@)qWGyRK9{KxVD z+7MX&8~3H=O!t2nKx9m`zXbqq&cVV8ViCf{DSL|Ev9U8n-*Smwqu|H_=o?Z}#SdhA zgxD=I=H^$2bqj40FxiR6iX9o63?%O1gRjRwj>~?`nDBB!jG*MG4x1%*S~2UTAo3Pm z1riM#2RrSQ2B!aLORq0+@22d}T{(s&e}s|L>^s{nM;g#!0LVS+Cst(WXZ-|OWx_VS zOS9n+vSh6HVhu*^HarxC&|GQCTXE5h2+>y3vBY0k1bPRMn2$8&WMy+IB zeC0VQ|5cVNKET}FJkgrMX4^Zp&tcJWFc0v4u9tVV0z;FdB$|pZAdarLt*kfkrlb$7 z@Apar0JJ-D=G=nWv|_TYp@LwARV1e{vb}^;;$dr_UBBm{vcJexno)w>*npIc0oq~Bx;VoHU6 zj;(de>LvV8RE(lwa8d8uJd}MIc}-4^jztMF6jWDXCKak>iDcgBeRmUT{qyP0B=RnW zsLEOIM!LZ06DK~l!+fjOOS__FwH*Tk?_yaz7woawSX#cFoE7}{AU63lN0w#oA^ z6O@>CuITn-J#`1&RD;P$EoYD)yvg?z%Et^iuPWrTJ1^GAF>AF?{a8pZ@^+Y+??EcLh%Y{pNT&(A-L_psM-`G} zZ?(dUw!*-zL>La`CSu%$Z+eHz^%4=Jn5ff3UJ3mYH6{t{AqKR)$4gDMi5{}*4UAEV zy!?GZgp6*>!>^Lu?@GRpNLtN8?F8GM1RRcE8482bJ1V!hIA3SclB14UNHtT$s!l}Q znwM9lYD#N0l6!WGf7VI%70>oWkoxpaG1-k7++M63?DqcN zO+k-oSF0ARn$wMhMV6YNDEQ0%x?U6G7B8}t>;Ro{tU6-wi6ybReWSNZjv!f0sk!6u zqKrq$YxgVYr>(en8bFKkuYIUSiIKt!*XX7fM&Bj(5nb)6kg6&!Kh(wgp)EzBS~$fW zW436-9j_dHENSPqfY15tR8&?M^2$1a5K~-ZG$FZ2M$U}VZ0iepOIw01l2hmWLR0EF ze9_yCEixnjzwK2zD%5<67GIwSjMU>s%Q*EH+)`6`v>WPu((!Jpk6Ee{W8)Um>#{=O zOHj&V(!DB(Uo@|;sEw)BZ=akh7*vPe7_3UfY zDsGe{V>P?AMIK9RA3=EPi##xA31bQkbhDj~%^We|r=liU?c&Sd@EA-n94v-l!b zw5inc$_Dw;uVomO96H36*@i|0N5f%T1L#CKRGP__x>38@C8Lbr^Mt~|TGW_xRWavc z-kF&bfgz^KFeZfjWAeK_mE}|F7Qy1^?~(2YIqh9Y7WY+98isH?tm+ps?BftVNZaGF zH`Qb*{C#7Gz)R!=L7jElE~>0bV&l5!G} zbvyrDCNqn37F$nCh|p2a8YIH6RlYSJsdWEd7E|QKYPPwUsVs&@+;=a_NH-rT%#w+n z?a~*cPF1Etxm=Cb3xfYr9E0&1&29KY{0e)udMQ?#(WlU@L@(tTt|P6~&Mx}kpV+gB zxTx)GG78YfusUw@T1KdpqiIkE`?7wZtNh|d;Uks69R?2UYgYzP@m6h4yXPGHm zeaQLv*J#7bt4CJlEau%?$;h(b{Lrs_3Gb{maAt_qnC~6Bqj*26k8|8BwYHkn;H?;D zpJ0)jvdZNv`U;na1RP*a0#GafQ}Y+aLI+Co^Wc5#o2_L!tL+4hS?>|EX0~I+#&J<) z2IXZbZa4$mI-Plf+z+$@w0C3!V2R>s@IBhtqL9);IdOU1*IEj$5qkz1*CgE@sW6R- zQ)vROi~MPGa2V|ohc%a2&Ufh840hVkL{-IMztjJ#To_JZS9cJ{8zPDiTKIZ3yMwvToo9ZLA1F`orxGC5V=a`!NtVCv}Ghw^D9 z#&Z-4H&FFe-Bl93QrdlRearX=zg4b|_CeCN*3{7-)^3N)M^&$>r0Uo@NF@&c#^O}v z>BVf8g@=rPQj}kQA%-qVwW39|GCO6pVP12Bq7ZiQsj9@5`vVbKq{dPY3KE1@#C2KW!OA6hzjn$=z?(21J*=S$= zbH?U;J%^?+HtJ4M7G-ZGcRO?<1`oNfnZ^0*nn^LNAOTJxVeW^i9m>MtiN zYfKoXXv}hVjVM~o3S`mDsWy0I9H@`ioayCM{f+v}`yfc1FTBMK)nJqR?!M?56WWjGGX z89gorK)>E76I3aW97xtXoobN^w${*6u19Lfss?`gkS}kV0*G@C|gA~NdYZSy-e&fnY^f~<(FaXHJ)ja#~^y~a;*!3``zGnQC z#e((elHQ;K=+{6@6ogn-K_a}gA|p#pptBU-n~lYJc=uY?U$x2-rq>+1UhD}7?pEPE zi$>+&@Sw%F_n6FYr6?@DW{9&pDz4%bD+bWjtE~cG1S{W zKW&$|AOv%9jb>bWiO26sHm!a8GLxLoMnmmZf$kYxLA=_kZ{{#;C@1#Gn9_iRpC|2V z*?kgh_~K$CwFUQa=d8mXL>@LE%WqOw4u6W|uR6CvZ!9c3mJ^W~|Iu>gG==yM&w|o< zL~FNBRVHE=M>*|GmMp7R3$A?6&DJwW0JRGeE6;iNje^RomYP{z6e?Y_~j9!S%5mTKi=EG2Ky2y;-nKt>tdNY~{L7E!A7uFOwg_>mmYh)35$6VauNtVGajdSb5I19UY6D#zt7w%2ApJ0R3FmYm?f!=WgUqww6xFeYpN&cBDZ4}im zs;)9V7`dnjr+i2(#4Q{TjYgYqH4GmeLBkaLk}HSxxy~o&l*^oz$_m8OQn^7M+7zXS zs!5A{qFU;atuN$C#hQaZU$KWk+67x+95O#HqG|HsByAV}v%%Awoif7w;X$%jy6$ef z3ZVRam~Xl@S(D|_5%1JyTf+|*bR7I%9ye;$JeSSVuItRz#h~y44VgT*2EnI4%tq<% zSW_4)et>@s=l&WlS1E0>*9BT0e1{58?J*S>3u?-p$rBf7n9V~`qu&VXB3Ih)*{q6l zI8~BJa!i+1nT&qDHfgEWH^UaPE%g)`t{8@Y`&N-v43*ha(QZI~XOmMnACV;Fj0jhq z6WY_JSyIbGwT9u4q?4DjwxY!PVuibgl`AbJn2l?3W@k~I`WRXJIbXlqskDRU4(JQ+ zo0yo~nM{+|J2i`bfo)U(a`qEei)ebVt}XyNYV`>)M7CyhX|3AOW9+yXMgr&usnpcr zDRzf((X*uaA)YJ^oEipM7p&^aN@TdAT!K!-+ON(&2SR75rGRLyy)eIhYN;zKXlNuE z3oR3#DALC*BfHUfX$&S)e_5EfJe zOwB?n;=!!%oR==R-mHKr@3@aREzp^S>Ae_QBtc4?jCbw5{F$NwduMCZ)aTZ&}#Q2tHb*iHx2dDCO48g9a zZjCqlB4MQ@mL2*BMnn!S?pn=2a8@LUP68O=v8xZp%w+kC{HVg9M;~D)lCBxX(@<{R z!U1k0t6d4&w)Lav-%zLmQun0?DVYwdW=I_wcT6e)O#`9mNe(cKDQ~~brZv~1 zDB{}D$)Fa+%MQi6M0)w1r7^dfS)>c%FZHH6S0H}vj7a{<%eE*(62Noe()nK%Ayo#q zoprR_CH!5PYX#||y^ea{$ujJCST;Z*7J?BKZPzS%Y!08~4qKcZFDNEl=jLQeK_U@O zrrlh-(s|FO{%s*kDi;!elw-6n@Y@N}a4H#^YHNit5gKz6YoWEYT_#F^2L|Vk8U<2c zSY3&4ZAU*J9u}-UdW6(EvBQH0w!<26Lb_;I6nG15`|U+Vmox83KPnh9{K@%P z9&?=}D5A$qYN#m2e^+N-W1J`%o`gQm<%7h|*LYj}R+IXvI=@oUBpMP78dK!n;205Nd0#OLC^!_+t$5D?1Ys}?N+EDrZ2d3uRDaI-fHK^5xG?- zrLLFeZKAm5(Se8`JYij5krjvCvJiVjvOpHZTaaE#+FWNX>8OAxwQ}r5lUhF_g z7d-%-$`M1m_*Z(}ijI|jnOV)7)~+)poPMG9&cwc zvTm)DHg2re1>#gSL8kka%(R-_t(zQrn_`qC{BE?t@fuVq@rv6EQ|Q)GEJq@JWx|!W zT)P-;?RfLtB#{9fy)5j&E{VZogn^Lbp?)`brZQwxyigf=tJ#V4+-^Vdq0rr&n#13l8 zee^q=Z#tIheNwqPpKt4mjG}Ms`rr9zDCzlE5)Gbyt5Ljw{!J);y~RGcFaKer#0J`e z*;uoiO%D7Ct<$D9BOW%b&H>d&UfBQQJ065?O)&Scv^s6m7d0QvgHZaMGWT{ySB_x* zd)zIZ2U4ioA8oBag_Hj&ZwIcUI#LN(oP<&gz47?zICU+v;`o(3>th03Q1`LOW2Pew zWsAXrsAY4Dg%~&*1#q5X`k?v$l-_VD^G@3yh!V>r^!E2`n~Te+gF3%NA4QbBnAbaK zG4a+M_=u4ZN-fQWy2BlYEPXx8DsGT2Ky@O1$KRFH8inZJzSQ&af|G+4(a>t0RI2n6 z>@hC;x+(=2>;deVPhtfApZYpbVh85VyVq#w>j(!q|G@%Ca`9S!q+wc8{~DFGTi3q& z%g0)!wZTW`NR;+Q%cj)SDVa0x1Ehpg)`Yp+54}&?)ykcS#3@DdJTF6MzGtW&y8A!B znQi>95;X9S0O;oChvRpiktb09Dee7L3m&uh-QDX9wL*!>; zfJm#QP-$UC_V%B45RCET%_+97BV>NOpab}yHjq#bP+(uKH<56lwE$2KX1}|xz(|gN zyBYtAE*%lrh($2VA4K@SRdt|O%L3G&Rm8sp{oew4qtNe;prSec|9wjZnDOs6a-bRi zPov|14%LtYj;Z4O_ouqa11zI{&w=#Hp;Y+zZa>>a)>K?V0u~ll4m7gcux8JR+FR&v zRcvSBGn3X*4S|i9x#wf`4L(zlVDaY!@8qwwzbVmwD(^(Uu_h9X(G?4DtGDHk{+HXi z>676#%E zhvhE;Utt5Af4c4|t0sY{bQ>-%g=JLXjA56n-#0?%^4YFUAOr{qW5D>jGSw(Usxv*h#Zzj3~KeePb>r;2P{5X z&5s1iF#26pR03UQ?3&J+(=cCJ=zmG^zXf6__q*+NuV1Ey4`q?V+p;5k62YPHN@fVg zv{!WV8nv0oh9qy^XIhN1rspAL&n_a{=j^b)zEKtwCAMp2O9CJgmhRZgiw4vUn=@5| zdb*w;+H1R<-6|(mPQ~WaG_+@?;QOuF6S$9KIVX7iRz&kOug5u14~j6^rYo@j8lw;h zEwc%rySQ8r;-zjCo?cFJkabnnR_@}6$3WB~`65T7L^6GutOfP9?S`ivPj-f${FcW|VD=!&18AGtk#NIT$C41Hh4VXt_+ za}x0u_RT3Rq$6KOcj}S zk7ItEYDeF>e6|Cn!aA?XP?&PCB(-rg_HAQw?$C!P8`q|Us7-P*!`Gk~FXejZCq>x* zobKPNhCp+4gtBbo@r3O?BK_~-|Mj+Q4g_9J%ertG(ccpJ^BG{5~-M}GThQ*d-5xX#G zNJpmQ!^2@!vmHr-uZYp&&Gpg&B72=pBGdpF6ki2#-pLZ`uFK_MueEtZVzzIgN!Tw8 zJ+SnUDcd^4t)3laY1?oxT3 zq032srJ>AT&14bBBa>NRby89~8F+;CK}R_q;s{5khO)wb$We!|^xw3$2I@b4Ub9N#qF zwr@Ra;k844-m_2y5x+P&qzsolT_yFU_)koHy17|@IxK|{5$@=&63GOKQqj~oBAz&e=ty@{$#h`miei{*|R1wQ-J6LuLj%kYsM}Dti3k5820~l z4uVjbAn~i48xmBq0FZ5EGik`dGiMlXN*^9MyE(O+eDKYxcwIOKbZD)5_R`V72>g)Q zPH%koG)d45DIHHUdHh$oZrE_5|1m3h$RH9@Qn@{AD=Wy`XYs_f9zT3vGyDs-Z`0F| z>y~S4Ylz^(qoar2G=r03jJh2?Jw5u0(QghCJ)+YpF}wiQW{JiHI`5UUfFz7xSdg$wEuutC>I2T`(@0`Awbfi=71&^$#ZDCLi#Gb9e& zG<1Wh(L<m*hf79te7`) zyea~agT0YewLK%=q2y?HbBnZF?YhA$nH!-zJpLVCKPz`kjeW;fm!*0hqQrGh4 zNcyM5#D3lXEGiii#Ty@0;Fkj2ZNhUjBb=um2P-g*!0U)Q*lO}4-y<-5`{ki>aXc&O zdoK(&Hg>QSJHuFp2%+~^(SA}1xYa~+nDe(!t1c8X>uo}Lm03ia1csi?4)1zNzZSUJ zNJ)WsiV`67JT~}A7u^{9tX@;Z?bJ2B?n8$maKcAE*F07ykT-fq>^AR-ZG1WKK6|!w zSxl;U#@@_`kdTm6Lixqw{Vnh1`M}AcT0NEwrR}gN3-DG)2)G)Z1|5)FYECKYn}8caMek<4w8vH{7h3&8HEyASe}}#tJ`D8xIZkV5WC^ONrz@Ja}jROd#V zz`gkF2!ECnACQCQ=+Ub3S;>yf@E8QZz^<``p9o?Fg@X#`agP7kZo@m2y4usKfpFf1 zK}fim0B!@=G79x+emH>7hrQ8DXvu@KylqXC!mJ~+L;{zkl!1aL9U|Q5f+yeh{^i-i z$OHuj@IHTsmx6H+beJ&jhYsT{Yebz0=F4c$(_uEh=wBwwJ9%xDf{pfN03jE?cdmF$^nYPLO1vQI?E*A(*hjy z+dv~>04HTM6R(NLOAbLx*z4Csvk9ZYYbT+K|Hqq(^kz5ynsWcV9t3&-qLxK|!2vqS zUTe@MLBzLK%$cq(xVF(Re^w0vUSfoNLU+e$^t8lVWVT8RpW@R~% z@|wf^vnsSHO2Y|dBQ=IPy$vr31DGKopy>rRqk%<*6AFR}6!hTqdbD%ncIXmnTNMI$ zE`J$IQZ9S)~>SIBg~=;B!e4Ops_eC{WzoMeSO|xFtFo)p+sBid6p| zp5Zpg5L(^6CH6Sth&V8x63B$7S|P1B13;!Mmy4e&j=POs$+z+tGWOE^9jNx6lcC^R z33V$gM`*@;3#EG-CiNz@S2DbsTb%%mKpgmnH{xi=g+I4Smpi1xaQa_e>qS9RwNNeG}v?Zws5u|}Nx%nax18b6*juLj(Ti0EjOIRcx=e-x${5-z<-EI6o4 zOI?%LmZkXBR6h*~qzf2OiuNL=wYMm62nailT(Uywl=1{2XyHW>FZV>w$EgLT>Gpny z#l+K4uf_g#=`bMyQZ7v$naQb%*aiggGmnf{0PX!Skn|A_mm7^f^m$tptWFxRHf>F% zr1%1sEfsm$U1|t%=poMaEl@{Ufcx%RN#BZn6!vFv%1~95TkmcO?iW9O{TvN%WMu`? z*Jso;EI;X!e7TxcKo)1#gh8u|hhV^LLBXMrfO>j6wIH5Kr?d|nch9x9f}EZHNA>~q zXj9y%;ZC2fK4b69QE#1AE8ewkd@T^$InmOYBXSdlYWXeDZ?@QT)}CmR@)XMD1=ck+ zJ47#UZ#U_=-euw0g#NRv049n`YerhxdO`tt)t~ixdSnz~bBm-@fNGAB1QeP#G8vJO zO}&TOl)77DZchL7l5=-|Z9pUaKgwhqEvTAYGh8wABR~A>bm0-}`KH?qxr=eizScNv+ZXFcKsq6~cqY@YJhIXPRao}rIT&sbcJ_kJ_9 zmX6(&>glh%H!+xrGte_=ORl><;ui*uL=Jb$OGLSl+NHBmQay!)EIcVj#YK8=nQepU zx@N0qMD8xyjUPVn=VDq9tc`JTefwlRXmVbqwX-SYhS+O;eOs!IY69%FmkZcL?9aYR z$zFJnA^jBVWc-0l@g@>I|lN^620I&pZ0N$g=!DndAnH zPvfl{1e;aDe|nZCAsq&C0MpOF&s#Wwfk5tyoln($9;)Y)oAY5}M^kMN2_a#yIu{l& z2pR2MK|Xp{soYe}3pc|{tq!#_a3y9uQdZ`5A-!5T6D4V*BH1e)F?(eKiUVFquV}@$ z=a(c!VpPpaGr#V#yP6;vTvJcxVvMqyp~YP3m}5N&_*}hQExVM2?gZky1=n43Hx%azh`= z--Vho3lI4Dq^hzoHw|tMeVNW6HK5CiNx{Tg19@C=aMInK z#-Jpi(=dSZRM4Q4?)gUWOz6S})3@w6CnsQER7hQ#D!MZ*uZl+-=9D~xsQo6spnlA? z;RLYomCGlhPHrly$fTjlbC`6As}Y}v9WXE&B0Gi2tx{sbtF2MDHKn5dK|9N3n{$N ze0lbM+{wt*m3YdP+>WK7HYliJVzxeWDBGIL$KOuTa>2{HOd47a7!J_b-G6TQ z%UjAj{+Sz%U#i`W#z;CpS=UKO+L9=4fIAlxw^6UUAZ$)wN({>`i&lgPQ9~{v>J#pO z+Dj13+gr(i33V-+v?61lt`Fq9wU)3KHIlnpG)=y&2pays7~CP60wy|E$;L^wlD~X+ zO)WYnZCv_x?mZ08M>@jiY6sR~EO?!-DRImy!IW-+>ZSqB44_2qbs;p6H#O43!(6Q z47x!cSD;lIeQs~jH~6P#e)BP$y@i(MTz2-3?Qo!MEGKiEQg@RFIqUy(Z+y&=j+WM@ z7hb4Ovey`@E`abAS{){o&+VCB6@ua5N|ZL%(e(5*K2Digv;C@NeuO`z2i-sldLa`; zFIV+a3L(+akL_(Dyxfm6u~EE}L^P`fUy| zwZSq*&K6n}WK#(8Jk|wmOWIVpQczu#YxA257F`7lf}o%pjUC}fu`rlW>%BaR&E^re zI7XD&hF%o@

4-0SsqhCAjM&5goPSX2UY8*MS3!I~rl+Z(*$FX0MnCysFSm*i2NX$$sp zm>nYRm`syWd~Xjr;qZj4q}@M$l@9+{aG>wVOkrggcv)aCpp;`c`VM#O@IEBBhzYTr zC9mQt81ey9PufB+bgMds9bu72#yg^*S5zW1Wzz=T!muH)mD zU%!}tjtYb=)d&*7tubVvsZOIil~VlhK57W#b>(-OcK`0555#L)c*4oY+CF_aQ#lqY za?{dHSUVNf7*2s_wuL@p459Pzc4k8uqzG_oVYEAWdwxw4$!7`DfjN|O<{`zbHAjK8 zB8nb6V+t0wC+4^%Frc-jo(QLLC1Z_R-$VZoQja|RPd+F#;8p;jArJT-ei#`YI{wIpniz2F*_ahQi-;C|u>BEGo=<#c= zel~JamI6ao*$phlkKSbOT3WWTR0jyy#X_7py@81{s0yk^xjW??39s98$GC&nQD;IR zgY>s$ahbD}nNA(ErY~)~eE8Kg#$x>{O<&O)f6L1W`grE@*zLph=6R7?Py>sRnwF7# z#s_q~`;6=^8tu1?Tr`^G8R)45$1?9LM~2^3xipAAPP=7X$v5yrbHz~Tbeb1EneJgh z>VfSwor7wM=S!98zG5UNV2PbZrGQyy-47Z_V;@;?!Wmb~=GdM>Q*{|XQn8*o%h5&$GU~V*T`Ee8P#0*vLqW=>Z2VaqRsuhG=k-mWPeHnT2mvGixhmM^FKn zI$?#=u267U!S_{{YV(oIGAdH5FcGg{2WU)t2kxX@k8*2*%46l`Ji3x}E zN_H``LD8Ei@$ri|GlhYg5vq(@Uznmcs830WM@3dKTQ*(eYi6uoN-7LB;s+=qcbz|2 z0PK{^D#n_Ds=k$as}F zh@?mwyz1UDXxP69{6 zyPXJ>cw#DT)KPU9Rx6UD9Nh3&YJ48ZuaL}p>=2$;V8iQXH$B0?aF)meo^R^e|hn+qNkQu}S`sZamF6=Eqt_ zyAEXaO;Gs#jG2^<>ANMk5j0QbP_a#Xg1pYebT(DBLp<=1sssH9>J+I@ZAe!|N1|Sx z5qm@}WwM-baK!vzB1;&P(ntN#R5f1t8f(YTfvp`{OWq0QlXJ%m$q>cOd9-6+Ju)KZ ztwGo6Xl#yQC-;}9tjbB?TU$CRFqul8QtVlt=D$k zG-*ff->F~kK|iwKHAPfl?3LKJ(-hFH$uE%1d}2&l-OM(Al0M}0&h_qbsN1@+EuY~h z*UInKC9|EXmfZN1wIRYb8qZFpzt`J1Sfx|>1KxOoA0@!VdDFF<-KI$#!_ICe6kAdg zr@Z_ySaK`~bdmyrRC(IGtLX5J*zuaiO`YqPffBSQrnWZxw(+-Vr#tI{Oq0Y01io_X z0YUt*_OW;kqF@Au<1Z}C1i4@5VwyndOyKkDtqg1V-L4o}E9}E4HTkQQNbF|onCEED6=*S|n_h=WoO8-yWX)$Xa!3p`y4bs7o{;rc zI-4S<%1UZlT3Sj<`qR_iB5G;WbgCjj*IVDO3S*jf%9t!27%iQ;kHbhbeG9FcOauv9gi{e*)_naL|>7Hgw$5!u;A)49TRf5A#Qda3bwT|LqG*Q;KZPzz%4R{ zIBO|zKeP?mH>`6xB1(@S#=!WNj@VA`qu?A=P1yD|fhcaO*#W%USd3(PD?^I4d=c7j06v$tL|crkXJG`3En90~KYk>FHDEA$PvcCCj1TejO~2mWKpGj&?_{Y_1~q3_0#BPd`&1!Ow$Fr znTq-|)Dr#gb_eOquXGrsEr<;r2hGl*qEW;!DA^jBD7gdh9;GYc7s*2c4|%rMeSTaU zLRD$8zdJ6{uP!Iu*vYPj^fTNX?eU(>PUwQnc#2A;fn7o5R$4`+=-(#NX`ow*&_A7j zT=#swphmv*?K56WR8gfcKEbYWuoYL*N_eiX5;r|GxKx-Nl_})I9~KqUNi2kF!doS+ zK>m=dN)ggWI3516ytcg@CpZ+R%TT)D;`oet*$elo#f2|U_9zAqri9pv<~o_tUUl!y z;<;{{73s91llkY{@_l^Pl0)VE5^wbuts_GcSn~5#m*l|AkMh~sbD&<|r%M?YHyW*u zWZj;VJkz?B55GI^5|(CF+W{zCzUeEA-p8rhT)%j^y_0(r2@!%N1e*)uM;oY8l!Gf* zK{h?|*=0?tL)%GIyu`K_8%Vj+PEP|IGesKI%DZ07@{VrN+n&@M+ei9L*E#xdV`yAQ zpNFFmt?4403O&QZe^LCbK>wONHeVA!_jPTf=yHfPT)=;Ox0EP%>pNz4_BdC|MbuiN z)e)Db33PZ+l8(23RX!8<$sy-WPSxV&_2A)X$NTaIHP4>9g%t8$7VGY1%T!ap=!OTs z{098k8)AQ7d3kM!VrLnpun!Uxu~AfMpF}n|0wCrI zAOqM7GZKK1Z;VE_0z!ndp=8mR6|>%(rDiK1+5!B)Wntr%qL{}FRVa~kVtctAN5-A8 zTF4y#oZv!g7y30^XV32 z#4ik^>2;t8t1OGR`-bb!43qra(@X@>vRp}>E@?s^sr3W;JL?h#UwMHkj9v(`)FmhV zfA*s_$yd~u>E0TKN*urlAw?4zbKOdAi(OwcxU!Uk$7%jiore%bEZ2 zdN&$ZWDmRegMzfad;{Y(wx_A|sD;~;j9yA*7Zo%J-;y2E`Mi9P&H=*06j9ZD3)65@ zz9537X#Cn&c9d1%s(hc5Dy`M0o{J{9VXfLlBh%n@YH~%I;fT7qRIpNu1cW-$0Lk~) z*kAiYwfC_D?mH}y&3gD>-l0a@DLBjlBcg)=*ZdR^uJtCRccpob+PVS)cHQ>|ce!m^ z$qB)(dsj^uwHi{v%|5I|_}32Gid2l4P;*O?8_quACkA#+1AP%KOx9kch$!QWgzvfW zW3pRE`)ru#@hGFt2r)iYTdtB?DI=_v*6y$+8GhG;9bZa@8Wh%#$k_>}si;iSVabZb z8sz<*gf(nAHy&&2XL?1{XQGsK-H#9fKU>emC3G0d4b~0y!ND1>Us9EG(`I=R0jLraP1b(<4i<_JFux9VZ8uw_8tyVWO{8 zho=@u&!t(q+bg4?Z3A7 z9MEZ~Jg}_Yd1o|ygL#pVFO;8fL2ct=C1%(k(IdU9@{8kGBUtgP+*exOdK)p{vQNkx z;T78U839fKr0BbLO5dtQ8L0k0vd#gzl4X7S;l#EkwyjArnTa_Q+qSJ8I}_WsZQHiZ ziJfoHz4zSvKj&L3E7`rWy1RDOTh-N7zxR3Qa#d}5Ss=+(spDV;vzg-#6a$Ji1f5WT z>qm-J(LxcfIHyOxP2D)YG%Xc^bG8*f=2q6w6opOaVx+5d7WQwQ4bXjj1qUbZNqvj}+!(<%zuPxv zVYNFl{Jr&Ax~Pr#NZsDDqKYoHAvM2eEWGXgo~L*2W$I?%xJFU7$f7A~Hueb0@4Di) z1eI#TMfF~TsVui&`Ei5-LlH*0&9+K~!w6de%!UoIO zba_!5_99pghZzX51aqqggnYj103{H^;-D?~Gp`H%1dJ zMMdeQJDyKy@A15n2A;_1b>ZU`8vU!k(yLAbcH9H&HpC_Owk#px3u(2x)qU?Bxb2pNe+;4ClB`x|q>4 zQG$tZ|61X4ZuVT{Up|FpRHrXTIbXFP4tRDGRtgUkLxZ%{vrFcfIYH*94uY@4A5Y56 z5mxM;6k_z1XdbjMw&p3}_Fc#*G`DFd(wc`+TnVvfSJF+T3XaV=qUAv_dGgdPBk--k zcI1_mbHHPw3TsYZdhQTOb7~~Lh*qUk$(C2;Ikl5lk>U2mxmx4#Yrl97p<9Opp>e?d z6<@zs#gB)s%>5fO+|F_JdwsRa$_;U&Yz%05vARlefREo&&7M=~1>zsnbmuDSsL$w5~ZKAA%1c{GrDj?UR7Mkrs9UnhFMUzay#Vke~ zuLQ#NRL`&X-J?!cKK%NGpjsD39d>)|hmj%^c{QE2<~N(G9Z#lO<;?Vi1j@X`{T>Y6 z_#WPvw)fqqZe3IEtYDJVGGQ{h5t#^Iiz@th)hM$)E~JQ+Z!~Utn21r9T+epcX?i8) z5k!>qG%oLh^DXK#Cjv6iGAg3!E+unq!U{@Fe6xE%56U0zNWwk~qJmCoTpBfqN2Cd!XGP4lP z|Lw-4jCYfX7&bmc>sP-}=OMwPBZ;7o-FQ%#e-JmlSX9x)){bMh8uiW%pW>HKDrjHC z)AyAiNN@cm2iHtcF&VDa{w^UXwfh@m$cc-VAjVn~ywkG?EztZA7TSxyQ>)Rwwk~ePrq1U#IQnjjrx+-ArhxTvSmS{4=H!H`$@x@r|? zQmm^02EHOaJf9epm(=FgB95FQ@t`Ecf1MLeW-+z3g3*U8ymF^s9>ml z0Y_?IAvUK4(=E2Wm>e|`C{q0 z>(kwLAo^;oG49r-O9Dj(!hoPuNl{(BC-Cueabqw@LE+b;mi^byPU?mk*Z9g;UAY1P z{l@uk{n1}Lp>Ov)ij*uC_CySi+;C138IoL&pdZPMr#7+;deY8VAARs!0( zGTAs|&sT?gkxmotW@YCDVLbSs1U`Z^+KSvUI!QrONq`H=((F()`m_+ZxMC($%MB4g zUkC|95bc}#dV>LY^`EGBI?zqM)3nWyyhB0ru=k`X*{6CAi2qs5aPO;I`Uz zm_`RbM+3Skm5tgI6X1BI?3av$xuZ`PrbYSK(Txsn*ISbX)7mrsYLI z%y%P7pn*~bv0g!CUpHSlt&}(EU~n%qIS|df#i%AEh>->W@hPSYWF4!jMq$6!gv!YP zk&GoMmLSiDg_2ra*Y%il%$93TPIqB@^zJlnaTo?|X%Q(3f0@_Y>oQS$2USnW1)JLd zLgyRIAJ7vP%y{BfCOPK?IrB@iGFc6)5Tap|@8|@E1XfqN-hl}wS2$@5&zR7$xr$s5 z2OjSjl#vP_6eZKw2S|YsW=-BnWA255q3wnASB#2Tg4^TZKw8!^K;oUSa62<|JE_PK zq8Bz}-_>Wr`@w_+xj|Z;KWr&*x6#mL$j+xxD!ytV^2Wc-X7m;4XPB)F2c zC7=b2+ylD7<+tk4*I8KtSCP?A{UDa0;io*eKi7uw#}w3BVQr{dteq1@qNViXA3L@a zi9a}y^8VO_rODYYVRd|J_O+8?6I%ErVAbv*itkQ@WJjY)&zsPH2JRQ#h=F zLc-BNJ@62fvP@Y0Utc$YxdV$GwY_OdlpuIAD{(15$)O8^4{k(M%F;s%H%)WT&5yPdQ<=A1wC#M zIw>+qU$3tn$``I|MtEO4}Q8Ppn};H(Hk~*PeK`F*qez5Q6G5EYn`Cz&~+3Qk!;(aL*aGLbBQHm=onT5g{`t)xzBaf9&bK6Tho{&2QC zN~@7g^t;lo={34n^~4)D@hyoki!kE`0sP(_9*NJ(^MAdD<8}yMK^ryZB~YQs_dx%c z{P^_?`ML{yj*Tw7TZim^)f%x2y3~XmjI^8#s-FmSl>`KjzQH0pjo#7Wh~|^5l+7pv zuO{;n7O2e8>*;Cw0v-Q#vR3MSBgw=o7Uqq5X9YLOc_kU|u)xXsnPDlV8I8O0{b~BZ zK|}4gX)C~E8m8`B->+y1ZS75mtjD#95#GSfs#O>yg;{sQ?9$aUIPdGy8 z6cGy>CIG!A_Aj45>|3<}JRumTwH)HI^ErtGlr8h#PXYApLh4aEmYzw%5SzPPYFdxyULH=3qiiyIE++OENUT2rXj*yNT{!l zKmsiZi>iN2MSP6ZBujp4dws&UN^IK~QFk3)mhN9Pi5=(t7;O?}lKeh&4Fs%ZcMV9MTX_N-jw4UBBFp2=hC5{yvCp znk8K?k=KEJ0obU9QKTJQ^hm)|EVHm}41#=OMqy}ls(jbDQvHnjPGxd3Wa4m+&kp7n?u(2@`_87431Se{ z?sdg!miz1%&yRM7Hr3kSHQvD;bU&+G?KdJ>VSaD3@~mu#gY~zD0Y2+?c*Z^SCI*B$ zRnf9yo#(WXEms^>kwDG;V`asL+(!#Uq_Cx(&uCi(_}b4&Y72ZnY`u2m8YM?Dy`8^W z3fu_1v)L9yVW^&Q7M{Ej5lWnxJWr4(R>X*-9my4lhIK|#l+QCWW-xHNlH#4^l);vg zO6%L+co?cC?Noo?Y9=c#ezl$btnZv}nk@BXmBE-drb;mK&>S^H79+oZs5ObDtsw8q zk6*{n?BkX96-;G@aYOUR@8qPG@5T)77X~?^p6^&upXXMvP@1k(bw`q(!L`4%1ch?m z-Lis0F_MCQhv+&KlYqSgfYer&B&#IF&m&dMpL%~JJ~IJE$pL^k$kC)4n;g2x)7xx> z7i?g%r6(HF4J<6g+`;>dT!NYJ2M?Fo*`&9~gCw2J8I=Ts<=dIgsqx_J{dp4B#yn}O z4a2c#wcga4G(vyVtIw;{;=TT%7JU=_!xp1>axE441W^$QXwQ(v^&hmw3NB!TO|u}m_%0g|mz^}}ml zNOLI5wFIn`?z@+E8{^N1k{elc3q((&D`hXqKW|Ti-sI$7r&|j=^ILAHW2?~y@G?rR z@T^;l2af{mc4=$f^9VqCBTnJbv@HV7vDiCD?vk4B4Ec-8y|jt)Elm^q<|i*gY|M)* zS5viBH>EvZDdYMoz9=2I)mkD(fW z7F5dEz-nVQQm*9Q%{7XYDbY+WW#vdsI&DX-%B=6prX^wX{gf{_KETGSvX*y%Xjb)T z+nJkZ*cnU3KmoDjG4IK)(p+3J#k&N=E1wKMcKx!w=ub>OOEX0S?@qeJCE)i&Z!R*~ z=|b+nDht8z4d8}dAox}Nxpc#0=1t_Tpd6#T&+)>SyEHfVfqRpAX=e(9vwttrclj?C zfPR1UG+-XjUv3%Va-WfMZ&6~qt6J%q!k6f>Lo#@F4{|yL&6@D>xu#AF%yRHI9EwFc zfe_gy42{#wT||dKaOcYs!qH zGdU;|dtK#yw?N+v0i6$C-huxkL+~Q3?U6CmPY8fG`&t-F5-TsN#{DD8BcvsBoM`kA>zO{uz41GuXnYn+p z5`r8ZtR@McS;KOj`BBvyJv_8rI~N2qF#%zrJP;X>xmEDE;l9kOcV+{8fj+e{lZGmh zRK{{_8^1r_8zfXUL%0C#O1@gD?<;}ou0i+z=_l;-u#$4r%t@oikNecUaP+$5Sq;|j zqxA__>dIpC*)XQ3&y06|8>ho{rR>V~Vz3+4CZ(0j4&}q@4v&{c$?g0n8Q>cyl7-FZ5qQWHB zwy;cR0uHsxdP7B)<1}IE)brF#PVXH;jKxkeAm$KpkeVn0e`re6Q+|TG-xCLmJ3LoBXFP&_Lb-Ze z8|bgww*Dtu-zo(b!w|Zwv?-l;Sd=gt&*I}nmlpEN;}&?v0y8`wSlTdX2%6RTL>V`w zlba@T72}F^L6(RmnOF+{ViI5JYMQ)@4T}h$$ia=4JOKb=*BQwa>>J}sML#FRc=Fpq1INl2urX%M%X)| zPA!@W%8f6|@o_1vi+L!qJ-v@a zXIMnCN=v2Tb1fZ_`;1gg!kzYpvx1m&#=J;cpsvX=K!sWAB z{oM?g>q~@o9hcn`1W9S->kr+l(|i(tw_1?bQFf{E3-iiO`86U(iz|<_FQuQ3C=vUzIIOSGf zrAQS}qnA!sDg(0f`1m<~cv*0??0TRi1LmJ{g6dy^wZuAVWxx zk-6a+-$=FKo={r^qqc0R(L`v%z@XSzkf)Hv+`UTQOhtYJ-_RuK5=@!s5J#x|K%#Qc zUT%jwws&d*f0Id5iou}y#p}y`6A#S1ie_VK)=!G?>%fi`gj#zv(k?-{xBPb8*}Ovt zfeg(v3|%{C5zlj(cxpsv^QZh2^h(wD4L}}D!lN2-k~3+o!%um`)IP8mi{b|SQ6vX6 zTII8cG+q^!mn^~Q0z;Y_r#MV)xlJ2_ok?2Wog6-6X?x4^USTiW&di~|q z0smDjPc7DB%L5yg#m^`sNFcmKebtOD(AMUso8pRuhX;krRmg#pJS2H!W}yI`O*#ts z8d`qv%;P>Q<`{Q6j=-COj)jX<0b)(+?eK*-U81fvWwQwVx%~mXI6%5;xA?6wTNJ zpF|_rgH{%ktvSi=R_ILL^{Su(Wj}d^&V7>|bp)OitdmXppZL|V?ez+oo zY*L?&6+@J?r=m@BxVv(AUY>lL@~3STXVqlpITDpQ;+3>YnXjF?!auv*OVv8O$0g5d zggheRoD%SFU-Vb1Pik3Oir=Q5$|BC8Ms{+lkg|V^NW|KcX%`Aho5pm$I|M&$NGVNG zb(2wSh_3p*@_NY;TECBawa9*2ud>t$+ZtUz+oKFJQMgehrk)>>g2hV~K@@mSBs#7@ zue@4kxBjc)T_SjpiqKfjN<`kwy!U$C7f-b^tR{FilqDcvweEawPL{+7|IJ}$d8I&J zLBN#8cpa(&(_Er}Wd9tO>rs?cqYXBSl=IzxdaCc;Yc91R?3Tqkf~YvK;4mr$-rXmE zD#(12fshadlgS=wcXPs?`YqJ*66d5?7nH4>`$@A$!}*oZk2O2(`&G6&?Kt zoS$swLOaCfw(U)jzz;lCLhrPF8W99hu!N?3M^M6SrnZjKn~Lunf`d~B$zEG6O<&v2 zzV%B&t=?8PXwN&O=6P%4>>lhOdP`JTJRS{-wEt{6TjW^rz`c_&+5uXcB)Bbk(^hm6 zTVMJTWXEo(6G9}g5^&YkPtBU?P*;#(yw1>m^BlhJq>%U1nQ{9*TFP{`K&!m$p{dC- zOphI%7}}~*Q06FcHY3w}vs!yM%!Rb{?tJkP*T^PbgoUtHwH$=z7*1t>`?=F=T#SJz zg~{szqR?*TO-jIb_`nC}%$#=SA)WzR_45S*3ZK5#ZFq!`*2nkrK+SLQ#?`M4qAUvv;5j`12;qC^6@JKlfIaTHpR-=d@PpW zt??_FWq1yesVU8L+crDcGb>Hb5671k8-P-l#Gj+H`a0XpXy}(*MQ_z9*@O5|ebp~nOQ9Jd!@P2AdwBD_izhD4r3?&`O2y!?WM2wbX0Tk< zY$#87GvfG8k(;1l?Cx9UcXF@6d7?UK>nFIbnQXUNQ}X%t(?N8oG7^EbJMBJ-E?VL< z%wz~mnQ^itJt-E(?0PSlNdeWiBbkmS*j6YtTii1fFrRQtitgqnW#Uw$p+>joeun%c z7pE&Fo$DtvVQG!&HWh50(K0fkCq8b)_svg5ptOLD3>2H9#SFeVl=(9GvMovit)6Z@ zRF@s#=EE6;$s}!`YhEM(wwITtz%L_p9foZBqvr#9;e|0qM!>F( ze`TpM)qQ(a*HN^RoW>Dl0ygkrBroGe&~L*5eAag}bW#vvwXQ z_{LJ#akqUL{gSN=VcdSd2%&aon*|)nCYvledBAP~YY=Y?OBDdS)lf8Sc38I#`gCpR zSvUz0Ffr$N(ZqDzgpA0Cc>9on$A2DwB#y@_TJ4_6qD&D?AYPw4Fze$KA!0ya=d#dE z%Uz{)mzK|MTl7iqGew+U|6od`u+YoXFZE1TuNA>a#SE|KDW*>_DyhIt z0WzKxU~4x-yXeaGG#w|{OaOh8_)Fede&ny z`UOln=Ml@Q#=e1hy$;j<{@q1<7~iIU2>1Q8bErhopda=m%M4zu3meoFudFC-e;a3r z^kn(lk~;?gidBT3lMGkXt*C#fpFYAsZmREh?a;m)!ZFjt0nOM?eM6cG$2v_5auknA zp!sre!SME359P>s^)1UNrft}PoVvBygfzy*oJM?XNG>=fSG3W*L(c5jRI#o8W!&dZ zizfhQ0edDsuQ3cAbsH?d<}){7SS#THLdD#(v7<&2DAzK(hAewON1=zuX<5NNb3S)* z>FOdlHOR;*nrt^=AeC`^Ff~Gd*`(=pFnGQqK$C z(0t83%RK`pZCq@35x2NW2G>S9a6gvv6*nqGx@?d`sak!mXB{S&A}Ro3ro=&}wjf#W zqIHqDHwlg z{Vs)#rHu5&D~Z>Fkt0nQ30^huMw+mdZZ!KW_$xq6(Y$|^)?AW27Bk=r{?XYtS5TAk z4fWDroIj>+nf2gNt7t&EUToeyPI6(H$ko*)b%dFdYIJ6MWVw6>UqR%XM2u{8UttQ$ zMpEu>^F;EtkmH_;pLhA6tN7rRfQ##;7`iz{UMN2DZISrNYI3N{8!~TEh6q(c5F9lO zYW_h#YInNDJi*G{0~)(>XRaYLTI00>x~)77yImNX72TxlEkqfA81x|g20SkdzQ<~X zm46}s@eFErzt&{BZHM$*k`|*d)hl~y?$?GT4!?Il>QQ2lEGe8lZwtgv-0K}Is^0nUhoaKBK zZmI~vT@#0w98Z*1;lb~DPTFlH!|>#s=+-UV8rm-Q+ajF{%B*!VP5H`eBaWCmG;qwN zP#lV(r^9DmPoNi{oy}>~lgQh2R(m-Xn*zwD@^f#)3!It1V=$p_3NLrf4hs2{o~s2& zQbuY|nY(2qJKQ+0WmCMaYdkTbjFd?^9kgHD{SpvrnX1kGVQD+NtnF6>9x<>;6;*8u ztIcGYP(jyl`3^cd_)*03O2NcKguJmkr=@##FH$xB_&F}R;0vXz-7VY5c5L*MO8U34 z*62I--Kd9KCs6qx60QR?NJcuzI6(Q!YV+d*Pg=G%S>~9%+YzmGoj@r=%*N(wuUEDF zC|k};VPdPi()>6rz#0wu&llz1@0b{IRBf=}{Zma;1wzE2H*2vjjc?#8{Ww5j2~9*acsjs@jNKMUJsI}I<@0%?&O*;tI;M-J zT1%WATM#IWicprvwS7IC`YYZHm;ZK-%G=d==viWB4(_IM@?}< zO55iPaP@>*ZS6O?p-s};D707!Pm5>mHbG6UJ=_&H6u(Zu=STXBGVq+q{#yJW0n|t8 z?L!*2e&0z58mC<)TT$Ow5z{EP*DwxG(4tzE#-i^eqOnS^aTrMRWVGk6Jl}qDXM{QA zJV-p34N>LM-8pG$n569o`tEKJ;Xgc7*$6f)0cDuZk6lQ5t-U~ddjP*^FzZ%{l(RO z*h?@kxiEQ!1f%xRH?+eloiNED(P&8nw~4}P{q!1f2c>;EYou(S0*(l~g3WOpYq8kG zsyrcgn9}0=9%j_VW;$*KF6lac1n_6;&E~32^q1XKVo!T$zd)tR^g?*qyqNd!U0LZKa9uzgwU^{kgQEBdb&wk@Z+0yEm>6e2y!P`DX>NF>&B`Fx) zW`_1Qy&z}RgE&R&3GR5?>tGbwPP*h{D%KRz<45P8Gg)_V;62+0b1x#OK2EviY#+B2 zim4}XoVifG0!uq^1fz$o(WT>s3a56h#g-K}TLuMi$|_EiCs+n#B5vGj}_c_SW@AV@P&8^w=>VUnYV|e6jNTcOInpw8ayyj`D;nf8#~GBXzN9h zn%b>K&H+11S}J#H8rLzRd(TJW-jit}UhRsSFS~)9!u+S1QCB(-#RF<+z)50D&UjZ% zDTUW~`W>~PE%EF^FStJlpSZN^Q_5=!>6yCD*e_N0pbhY~27CDkS&6-peR*BOdMT2p z3!keyzLVN$v(T_yD+o&|co!OSyJk4DCQZ_^rK-YbpM+gol1>$M4|bDzpd{aBi}qm* zaMsu|b=8`V;&PB|?B-VVU`Q&8XfFwR0(B9vjfPihMOyScp1Sw!cmjS3nk>NfVRYB7 zXKZPac@lZVej&BOZyunwN&)JE>|=gy-I0B!xa6Wk3P4*Z_q~vWLLPLK`)yEadDiubPRONN^G|{kG zjm8jI+aH!3`F=hDh>4<2aO$cou;DcyVJ1}Byf>;hdNIb?3&?|>+~WYp6;$&qU{xYu z;fjwQ4+H@9b+IJ=bUJmQ98NZVlkR_@nNsAfN9pz2FFuU?Se@$oRH=<$aMM`DC+5|x zbqp~yEx!i8+={dEsv(rWHGo3?gaqOtfk{3=P=Lue>VkO_bA4tL>-8K=yq-Gx8-QUK zkN1(rw^Bt&TmQ4uex+}r-G2nYn6M%8%2%E6mpQP>b#eYemOl9dAtZ2L$Dsh4-@|3G*(tbq4(NRoytQqYfpAlmSG(^)zy=vb2f0ad{A|2gmhUZuALuZW|j z=URt0(EBY+qv>l&DYbIf73Ww@3R;&4B;3<5WpU=ihPe!)l;3}(i~e&qkoeId1_C2+ z=z`ufsTq`QR ztv_&rxFH0Jv1tH7agcaj1JcrHR89*bZf-M1iA08I^Tlv7<7PHn5@4#|gSWiE_MYPZ zq}4$FeN}icKn}@Bn6v=F0=#bq_)iF6-3mJD81{r=9=N1-SQ-R8DLy6<>c}{rD2#j! z;48g?fhW7TI5-@L`>R3zEu01dqY?xp;V5P0f;6sO7Enw?<<919;=7~szVy_S&Mf@E zpqqEzAfD2Ox)4(0)J%1r^rJiBQ5R^3`2HKPFjst>5l$6=;p{E>RVU3639v5#l;K1Oq5y9W^Ll54H&;& zi~X$8?7j%9sUib0Qq7K1V-CmpuEQ}u8~eKGOCUMn328B! zd~ze*8#M!49B}l&;cF3KVPgDe)A>n&;IA=((t#y;LV|Z^_wYHXt&V*`B_b+b&MN_2 zv*z7Z&(h#1;yoLRd}7~)Z8**TPR5u1_=r8O^GK2yrc|oS;{y5jU|RYNYC{2k=u_nD zsI0f5<*3S5quk3*nhaecww*bg>$SJxUN4}bH;++IHzy@xiaSGrrk+AX3@|X3w^r@b ztMTLwiE2~;Dfml?^sh?q%788^P{+x?xj7eiEj=>Jbk~5XSKy-a>DnVU_d%%oj-BBa z>Twg%^2%;L^S^D`6$yPLRow#tgy8(n zi$1+6&TAnohLe0-HOEGW(7{z=M}u3(W{C?BODWq>U(YP@{{GPEJQ!PP+17pbgAed^YI?ex>N6JB;m(d; zP&~F?{QB0d0C4%D0=nwL1QJUcIGTyFjtG&l#-hWD>GO(L5-scIaNu50F!v{EAqvq;OxKMhJ#Z8rkvY{uVAqX>;(=X)D-*tEJzMw^*M;W<<9YXIFNG=%ATSmg0k$>^ zvABw8&TT_segTRV1`0ysMJQ03`5y(@hVF9HA!ShRHgil9R|b|NJ6a{Z#KzU1l60&5 zU$FE)PN;xMs*{was5(6{r?IXKq*W;=X>e8iiDCMiq)m60U*(U&@c)Jr-~oQTk9cQy zl^V8@Ap!BysVx^wHJKB^@{S}6o6LGS4Oww=A8kC>jCfmbL!wrvS9?XpL_x{o($dlH zuD(1C^T<+x9!qjrdwcri;VOEydZShGEx_KE3qRkQ6K`sUBLoy%VJv6E_HY8I9Z9QP zQ4~pEEj9!~K2T7HUS^4UdU2toxVO_ER#8!ruwV|9qCXyY`f^pIs}G7%TO{b_?oKZe z6#^9rk_N9gQJlbZ4C|#5U$;%V8iwk2B+(@|Dh zqako&;r1s$xc_-!FfBd(Mm0*eS`44!0t%E#)cK4loe&Y3+k3--#a&jnlC`2QY!4cy z31xzoBy^$gwkEId<7V>v59S-w%M}bYE&l0GA?+_0Xe20`92l{~8XVRQ&ArP$ zD$b3gKmp+*72*C%2w&&%%LKpJ>k!I~^xfA<=g7|lX**+#`|Fe$kCuBwW0xJQ0;5}b zdFb|bBXsot`X40vY5QSDBTyIh&rPrAA>qDUuDOohqpxBUNRyCowB0Ri*>uUlqcH8r zs<6UIjrABW5z1IM2GZ7HM3GhI@~a6-l;Dcl_+&2(y8JVWe)*K6`9OA^pC8b`P*e=B zR~?Mh%sT*kA$79of@1a8`1t4|6%vDH-=BYv*FW9vg-9p5RmWzU?ac3Oys3c=xj6it z4LN$FL?_y3D9`^I-sk_HIOqgHbj>CUzl3#HGhw$>+1g=4y|8Mqf(`L=uan5eMlGua zqKVt&GvQ%}17(~4-nkK>RoK=?=AaOifsjnwrx+y>7C9KsWFgn%lEEAD>ft_RPF!G~ z#y`K}nV(OxRWchbcxixc#=?DMZVj%@ORDX@TXc~@jG9mzi+D}t{MRg^f1Vd80E#mV z{!md&#=azS3VZ4$n^C92S*4~s!_Gdx6Mi4lt(DH|3W>(pj!*$Ykj6@082!n;JId{! zlQp24byFT)6ne;~+M$ohji;_wMa8H0#Th&BIyRpvkd=ZT1=+8rUjw>|5S-$es31I6 z(EAJg5W5eT6g-g~cp#vN?Bg!mv-uvr)_omC}CIzl;Tc@-^c;0hA@;#HMuyqYQ2>o9|Xa*gdLp3afQE>-8s zq~a#_AD!GSG;Q=Iy2;wI-5^l7#GXt4ao8}JPt582zlMq8h3HBmQ?-!$pS*-TsA}uy zTVSO94QVhb#T|;87Is#L&Gi)$miXGCt7#J~r!2aB3QyOsDCtRi?7(Ja%$+HupU ziEwPoPWnZDiI_?>?inN<=J!gSr`hX8Be8Xgdax z6crHLoy_w_WDA@n{vWrPBLXmP40SI2?=-n!>>&GA9$;%h6tnn+p{#x=e{k_QrsXTpx|IqtFR>U@V7#{Onk z{2Q2*p!l-pN@>+6qS>|ECFo&3S|i7E>qU%NmcB+Xpdm!WWYw)`Q~9SD7)8J4Elpc$t2B-*p%r zTzz*JeGOAp@Cl$5K||N=U)#$lMs}5WRISFWsym0$EE&T%-0NlupOD|=Nb2jCQS^U}lBN(K`ZTCp%w2~IqUXKI*rB3x z<6XdvUZ!k?{ZIn&*@Nv`^vjPG`{#`WrDOyS3Aui6C+vx(;L^&f7cA{)heVNS*SzTb z-pa0nr8uzdF`FoaPl%E2$YWs)h?dxPa#bZ6PSg-nj_Kw3TL2RQGDvkAiBFUPaiL+4 z`oL9Uiz(M58=)6zs-F#Q&5EIWe8V^n=`rWrR=d30T8bK){Me0M+e(bA_%eCQrN;fI zKbw8*V-kxg<{BupOSoAWWB%nN^xu#QX9fL9US^s`c~65C^MZm@^p)EPZozO1i~jrO z*_o1|7@Ww~Z@pHSif#Sy7?Ju4{v2zF23+AR<;f;og5XU*9W_1`rOFYovsDO;X}1Om zb%P4m#MKm!;)9_!v444O|F;U6HXwu!(eJv$hA7S;o%Zg6T@Oi-ebfuvF>FHh3&3d; zit6b)@FLqjjK(5LFZyTPUApS;8k>|cl!Ad&yYT!d{TyVP5zP5J*?-hCKFLK0 zy8hcF_TM8g@OPFZGQQIwOk-qlA*s z|F#Eqpk*b(h;3>2e(UQwu_)n} z9fvJNfYjw8bPn@1Den_9w*oNDTC*vA7GnT^kPX+?IUe{YcT=t^% zHj;UrFPfyX!2LpLfGSlX zNaxUSL~LW#;h*$tdK$Y7(BzuH>Apbpzkf0&(mza3E`V~`068L-~xVc74Mrn7nTsVrS`qXJ) zY=7F)i*huiWGHGFE{imQ*|2xK|9bp%RCVl5+VTyJlb_%HZ$$<^{!)BIQJ3cC!QkEK z^P{Q8BvI~rQ=C8W72)%h)@DAvw%da7l{Fp6iGeuL|5E;x4YH%Uw0^a8suL_Ku~{GI zjGa`>=_l{1z3|WL;(<=1pOvS|bHGIjjWIXMkdV)lO_g>h+rccA(SX&{aJvvRnXz+} z^(rE|?651t``5a>-Mt=CYq)Y?Zu$ph^kndz+>BY5n^60rF|E;fzWj#PY zoqruqn}AN{;>PgA-HI-xRt{r*-QN^A*GsC4OY+$SB?Xj zk`$Td$`X}udP?eN)g4kZn(%#I{7#elVqz>OG7y`0S&)aRW+mzuZ;-ov4!T+_uAGZ7 zLkcCH!DEI|np&&>9UEEDkyZ?US-M?~CgKMI4Bx(xW3u`mHA}V_oJBfmlej*lVaBEu zny?q>BJ>oqoM~CO! z3dm1?==}^?qs0pz-QKGxGNXZ9egHS3geoKHrxdr+hR$Vb z2b)Xi)$A4me(p%yRiJ1f%Ux~|6kT^`&2g~`YhbPAL?6GBJXRYzYQoiWeBzeIzij;RXpR%7YhCS?49Q9 zPD^11lUtNdF=uC>3K+mul*z_8dz4qqU5?b6tiIs=4MB_`*y=8{0-D`5Sj{1ciokUo zusdPtS4C*V&8z2qQ_*p1SpdVaj$1OTzH@7FNqd7Guwc=4mW}8Y6wROS(qO?nN;uHP z16=9LA5}Pppc?gHI*s{B0;`ebb!p2+4a{f}73|!Ki^Vb>f-RN;YA92^!zu%Jq7OWS zMvIW}mmg4g(2QbiIT4`Rk;nfOD{0AQ#~}q%V&GjM(_KlEmkTBZsK3$m#&7NS>|IGd zr$|C0qWnt3j4q-Q3elIOTH%@{)CN915CPzDMvu>J1LbGC)vE(mn5S=&T zOpNurji7=Wxp^A0O4Y1HAyfrysa#kIW+r*vPt$d^k`XoBrSGpo60?b6Ma0gG0?p&= z`S*tT$S2iz(@z}sM8_c3t39n%vKxBF4JDX3jB~Mkj|0JzmjxArC0{6RnTUrf_yksl zhPrqZutNb6QU1czFys$+u^tnN6U+@PuHwMztIR8qf73Dn%jk9134bY0dzNvQSQ z9GdN{@(-Gd`M#kLz(KQwgkoIQR4|a#obf`*at9y;0(+G2Qi^#Mh?7`JIf_6E)~$(# zW1|Up8YLwC_OOD&F2K0*>#<;eqwz`HTNYhpqDZh2w|xa_bqV__JqH*w8DPIdTL(?Y z`ct<<`n}ogKTsHYx4(SK;Fsa@!GoOYZ>Y3v@kS~LOKmEhu?={^3=S+ar$ZA{t%twH z##Ebq*SS;%KAJ+WNeLQ5lA&)Z=9hMsgCNjfde;%<%{4!`ZS` zo*|iZss4~j^uZk>z#Ab$@U{NCI9=ng*^TvpE*Dy~IOeqI0i(xRv6Ef_+}K{KSy%eA z_1J=@Bc{?o#Yq*5FhmGcydu^l_MGzBsZ~e3gU*%F>bFqThpx(x4N2t~Q44UJ204zs zOJ+flZp^e=ZR-%g?K2Y(Pp5sO-la{KeqZJqm`myKzd0MPID#XNWVp}?6^!(*pBr?s z(X7ms&m_wfnob!hMrdUB7!8n5RlrM=rNyh#9hm0Im`=H)Yma(nXIHlHf}-oVtOFOu z6qH#7XdExpSnLcX;bIj>>2okXRg;h|Bg^vqtohoY<*++4y;}F4)|DFDzxnIw3MNuw zYf7+3zxzqUZZ)RoO;yHmAs<$Qmi6=Br)3c-H9G^?Pn{Ms#r<%9%02fbb-Av_1hEfR zW_p|B!Zwa&4A_AS78h;WmiIJzt6Flf?0(6<@A>9}Bro1i=}7`3i`I%Tp*|O1ST5^U z6C4j^rZ@4lQVrO!&omeEP8*ytwEpRmMY}tXDN4>bPo$_S1-kenySf^G5x%kIkWko( z2v-8BM??)qsv$+TGX5>K>T5T|MCaY1fe=xQ2dGt(7w?G=4^+}?j5e>Bxp3Dyng zm#fG~JzG7`k9w(m;Q@wjEjVj+x&PNx>bc}pZP0*rqZ?T^wy)Sw&>@qTkU*roQ(|{G zh28VvidUT@VV)86Y2(7=n}eEZxqH+Z8+bjbKr&VXJhzjK9cn!pr=<42c3^){EV4-p zJ^Pf^fXF&R%HMgl6A>N`$4z_tYPTAgFx=a{#yDrb(kMp`=;^tAf1K<1cg7m(wy?t` zjTn`PuZ@2SztD%r#S{Ij!O;#2$VVuZ3G1s6eaR;$&Wik_*NsS(3tr2lm zGrDO5m0?;Dg(;b~V3}gnBJ?QgD5#=TuEtdznSH zaD?mz21`?l_F~g=n!*CsN>pixQC!=*r1e{vu*3%w%!IXsujv#Fa!^`jQXa4FR5iDz zN>$j!^2pG3WFL0W1WcrFeVhCg?{upRKT9(84akd*VxEnt6rNJhcKh&ag}mdIsVCbg zPgB$iGaN;m?R* z+q4YaTRjQps7#m2SRAC?6w=Q3G|ATj940QLVkp!%oeT>l4Xd_Zp!R!E7bWpDI0V2P zc3PBBACelHhn{?QYdELba0^pk4bUY$$)#R^-f8lMxz`k11QsK=pu zfXwVXd#@PywYUXbG$ITVG=U$g#5!TN_oga@0uQVudhBln;uNhy;bhQ1)tZ;q#ekWN zrZ4xebEx1W?bU)!dAfWcdyJOlb>$fD?ba^r@X47|u*#(#%8A5eiwjn!S=9PAgu?SV z4r4}(D!nZth`)@gwDQYWFlYEIMv}<^p#6wv(9jl*174}GsFpveXAw_5s$>v!tpT>m z8;a-(4WtRz!qZk<0z2TM3B`#ZPal{YsHofw4lJ*9Sv1P_#lu9&1=VDVn+G$ zkuJhD1CnfE6&K=f(AQ@T-^CgnTec8HZ(!XmHncu?zl-8Xp{^k<(+C|XYK=^H> zUlT{yNAQE?U;HUCrhz2(`E4GQqkQDykDUyuj+4?g?i0t3$cYqMw$xtf4|X>SmL5=} zryYmxXG4PZu9Rq6p#fX7yXWo4FuKiNjtB{|X(0<)zysWY_LCVZ*`I+gr)L?22Hk^tXJYg4z^#&5zK1X&#wfPU zsgNzNwn+kFdMrx`>A1=Kap^gxP}3WMIM;qfXR$^sUdBbYrvXg`T?Xs_f5ORsUqQL2`61*lVCvZvIhS_EmKMB;J0+FQ?f6vVy^bnd2~EmB=Z+aL)6>%+Q3g z`E4R}ec>t@slk$FV7&_soM+o=uVh|JDj2H0s3f75U8Fn0Z{G;KU=N3=pv;feZ|I%? z0e$iAnq8~d#!ywYD!zEke$=5A`A{_qe}UZu)9Lt<&Uue6>IM$Lw0mf*eZA>wX&&4kt1yPbRI= zdbv^N<9q=lr~&abV>}vKsy=e_bjkOOQs^DqjS+IkooG?Y) zz$uz|G1(iVDGR#(v9ko55BJ*>)lr&n6?$h*&r&}@QhT9fWFxNp*F%x49G(}2CJh+= z(U}r9ervA>p~ZSy+6O0!LFehbpap}+o(j!jF|Z~42CM!OBEWRiV+1kK^z@-tL2<0Z z6YZB8iyh)i&}x|hgT81bVZr+D2P)Hx5ZI3Hn@vWTBJT>?_ju;6N9Q#n;kK>RiyO2O zTcOO_KZ6s1{zict{e8O2#FcO7aguXEWXejj3z1Cwi1qCxs-d%C@Ub0P*KC$(9UHyy zUW7H&IDAutgQt9eP&vcxQl2<9nj9h~)j-=<41o?^OGeBiKB`@YGw_c)QLrtTrt{j- zwirXY(=1BD4Fe_}aJ2GsaynXT_vW`;Y+iEbTL{ctga$)u ze54qDm|3qTM)052T;Rgio+EBoJYytHR%Bv>|u~eBFeGpb!lEe8+Vqn*eEM`jZ zIi)+4TC`}GjbjVPGVD&~x~B^K)$k+v(GeMDpWNujQ#O9B(zKBkL6WgsLf5C2e}ps* z7DkZX9jzx5L9?!yzJNF;;$Qw02W?EST@=%7`;O)X?c;Djr*$)N0!4c|0zyrY9Y>p=~-?#-{=~+vKcko_Kiy2)8 zIdtxg)awx;212gE6O+m$2p26hp(mEiavi|zT|%-kG}~I>r2@}u|AmqD!TDahpg)_i zo5%V{@#Yno`A|9EY_&cN`Avh`ZLz574H4D6|A1D+TvVHE5jE+rAw>qwTU@l|h-)}S zTG)ntUAS>@b$#o(dGJQal8F#tyQ-B)iK5>ButuhdmQ!SzgVOq(IOIg)MuH*(C_rB2 zKMK$YGyAh}_rmB?*c{NP^Z!4$`PUCJ2wx*CxUXhvI8`4Axp`W-AQV#Mb8+n;qe^k% z*#cY;*su*mwAJ;=A?9jJv-)UC@tCt}@DYC+^Q)!z)EZ>lzEP|d5(06CkPs5m@nB^o zD0ULO@|K529>&CG+cJI$=9mKt03+%_OK%0Yq-#X z1VGm9Wc=v-5bpTVn*OMji@Z>?C){2`rrn4s&Euj!d5$=oBWwGDG7>DD@9$kFGn>@{ z3n-y)JTgPda11#-ZWrU1b#4t_2=E(r-xwLLecV%*fL^_USiC<(v#WKbrYd|u{8zW) zCYi_WbT9=JMSVwhYFyU^2>~+-ZrAUk!sqen3jFd;-7|?*n$2(Hj2tn9IGtZ33pr%Q z!lxgs3KotNQj^`(^1Fk{rTF*n-&@2!N32Em#$OhZ78;_SHW0iLi=UYy~H#d20IeP^r|vihEUUfp|Mz6k}7osEf5hr@r`om@jzqf ze=TTcbslsX`DQpvHU{JcaAez@7jR6SlQ3Z#F;jiD71Bw)(<}e^o^G3VoKur1=LU91 z`Lz@R=hsf4EeBLI^S*{~#hSiO*{AL8mVP95wts9wmc!?(Dg-Zo*qGOc(Eo^9k4O>d zlX>ML@q>B-r>&@}oLyKl0uvNHn_?zYw9FX#HG(`Y!?<>+@rcI7(?u!X8X6mxOOIzq zZ={kR^pilQYBtQ*_bZNA4m&KIQJBnfza5uypqzgPkPD5zc}wE6}C2MdR#xscaNn+XqnRlb2`*X6+!U))|e9k|=#kL-oo#kKNz zyJmk+0%!}?Wg08_#+vRqyc~DreX|wS(++n(>FD(RnexTCJ@s4PYJ>Yi>R(FvG?H56 z|4Dm^^uhbrF-cds3*v`PeNjS|vVPxk{Cx~6W0z^c>~%IIG_`N<*){gamnax(-)0e* zg0^5ObHx2v4!XGOK%rTzM192i|68$gl z6VH5=Mt}-xalbXs9tK>Zg=(`jtfbAW55OJC&PR)NvYN+1uhG}z(GM=`4lQJsl_wRK zOVfBg-TwW*^+0F~kHbNb@NWk#*h)v?$sI-&qL)K5GL36lu0(&&ayJOVdJJlFWHf*Ww^JFgjwjNEF>w^bk)-5G1T^!B-4iH{` z^W!7zN%ts^R6e|8Us6%S-TSddg_KcHy1``#0SFP~b6&mBYi;%gXysw%Qi$%Zz&wp1 z1DXvIxpKb`rP^PtONIkf|7Tr@d{>mTdKL}@i2RgtLpe}q=Y0nnXQmoqxfsOZKEFal zU|yd@GugHM!K+@9)$B{B;X;72gd*;-{ZI}j;&DmU;$IK%#IDGK*1KP0>C1|YQM~w` zf_Rp5WoYiFKRkHWH<;cnx=@rN(W!lduvt*LM;nu^XF{KO6WmAge|k|V2&6Mrcmm7P zx+6Ai^yBK}4XW`JwKvuo_%oT{1p^zf1)vko(QNeP~3{?x+XTD=7O*Dnt~=g z2d?wR$G{kix9@}|mkMOHLtGYKE3O)0N|IVgh2e9YMd3`rBiE~2l;zw;u|fnYqxR0H zZ6B7!+ShGgls!dVfAXj|$X@1?B#?|u|H*dEN6LEu#T-W45uNX^-S7O2!JLkc4lf;V z?wNEM!;w_Nj*DE<6stah*#AN1CjlQ*V2JsoKIMLhGcdy?CA*SAZ^YYE#3$H7)7pj5 zg1U>T#-HB*&Hh~>gEYd_ZCi)9pCO9=qLAKT8~R`2U_b-qvHyhTru;_&Lm!%qhPUhO zi5BoxLlXWTr_gsr&gkU&MKuS?I{%$W_s{P!ai7vouFn;(4`p8zsr?Q?&o=LTeMJ7P z3iyCI0$W^&jFR$J_ko7e@E}yBn_X$yZzcu2aSs@cGC2j?8zTjb0|OthQ#`ytvsz)$ z+g0!9nG($!$$wM&>i-5i;B@eD`P3`lNw?Cdf2qMC1+6EXL@uhm(djVV^G5x6B2HDu zb%?P~>EAdC!1~S6e74i`MM9g$Z^8f;+F)GqOd{VD>&E?(x|L{2#eA9Tdk@sb@@OzF zSzt$YlN`DW!I+YU6lFie|2-b zSToq5Rcg+yL*Q!y&GU3M+4S{v#?4|uxGH)s0*ngT<)WYb1%_vZ>`Iot)sQgYf$dQ( z85}t50M{TIRnekJ+Px7BlCe6lFLqYvC9?U59}R5fo)Z2?A;k26&LMWV++8q8DrKc| z{Zy}^3r|=saTdEB5@4^pp^--Dca=e55KZaSM)3wXKhvpm^g1 z03G2$4M-L%!NHClM1QI$Tqt~Qvz9*l1~A%VK&Oo?Ju z9R9_9b8S&pY!G^xv%w|eb0qT?kgZ5E{f(*cGT{wBM10=> zPNEDaav*rWuBbl~ay(r@fZ^E*dHQZ#Esl{qZ|fcmKF;-F#oTX%I2oNR%_p{#N*+MP z4)W?6GU2=ZY8?`UpQgT4%y`&pxdqB3fSi8sE0`(k4cxC&s6pT#up&7PMLc~0kM}*O zaXPi?Ou{|iUr|FQU6-HXL%SI^QkjFwqM2H)Hw(Q+e=icQjb&11e1^WZHMO<_xU;_e z^?5CnxtQxY>9F6?sTUJg3g~uqD0|TXiH$(<;r>ANaPYC494n^5LKQbDNy-q^7}P0* zMrhY_Jo#w@-AUNs;R?cNprlxHE9w`Uc-u3VYh(53(R@|4N!_ZBD#-{>RpHP@#TLHs zGSPGW0QbzV`vKL1VBqW|tUs2IsV3wxsgN=ELK6Sk#me>IfbZGCkZF5WW0}bNW{_O5 zq}RNJT2@%CV5Y_3$*OKT;XeJAh2XNh>)^vuV-1Kelhyqn-MZhZQNbUIRqhkofMiYg zufPz=OEir{OW87luiXTivBF?~$DWcng?Q8za3q<>J^wv*65ePz#f^5I)&6Sk-F;GT zsLkqoaa?L0WfL^rs9M~%liWyTH*X9K(PpqF#ACJe<+%hM4bAjMw`HfW{$lqzIkq%h zZizna+TsI>{4|w?`p4U*lfs3TjRoQF*tbk@!H4x_Nyu-Z&^MhRa~IrAmc}SF=Km!+ zgakKgpH!Y|Lzj&O@*U?rIMTex8B9k~@gLV9pJaRwZLobucSAUqKV66|_Gaq6NbwhP z-|3#8!i$^;#y~jZ_}+JehGUr_!!EV!#irQdczuJls*7kwbX^1aUG_f^J_hFd;^HeE z*Vz&tq(_u2G(ggiDc3{VVC74_7Ho!b=vYW(I-QDo1oYkCNqYyxN? z4Q8-dT_E4_X(_$5^SIGfNo!F!-u{)v0Me=hMV!)fQNyymW8nbI@z@IMRguVktL3~T za@;hEj0o~}e{$sPthU2yHAHhrV@hJB)ZYN*6F&h?Z#b?S`T}2{RzdxNCmEL2tZGLA zwtv^nn}0L4k#1v9_DPtfYA^#W^q^a-Zh!bxjSeYg|puCz>A_iFZH@6knFxqWj zNWD7tnOt;D+5H~zua~LlH0`xkg2*T~>jt~)*6Gd(L&yGrfjaL&uj$A6x@0GUhjkK- z&Fq_3@$b0e17LqO9{#i`@%i{+?XkHo_8ZdLO?VcXry+75uF9_J+6nb zuZ*Fw03kedH4|N~CKS*0lwz{6#l==XT1c*D-ZcB=SPXbR*bf|s+>Hfxc(Fq?s$}!e zc5%0)N?OdubvKH0X1qs^i{ryztJ0tK#2f2fFULpT+1URjr`PreDd zy${#KD7?@23Pj0Xl07xCAG~J;+xKzIrui^^uOs!bUx@$#$P-xW?Wdzvz9}slBh$3p zwnHHc@eqr{(=Qbp1MiRj$DJF41k-T^U;TUYn@hEL?>noiE)nYNfn{=wg2zyeY`6q_ zNbu7er<)4-?m(3>l@iHe*PuUoVE7>?KG@>Wrn){6s8Z;GUg)nY8x7k~S8q(rZlC@vJ!@95NRxr~e=Q91~m{W?#1_%#c-$gZFrcuAP&= zDr7C#Q$zc8_870fghYc{kfp|->9zmaE<|#cORv9J`_muD@zS(O3kjln}`Jy)SFZq+@HstZUsR`qJK4(CFu#$4k-b zAN!N%+4%Ycv0i;yB^kjM*ctE7(dgJ3A--GTgk-$}ARjQdgm!G3cHlmD+0bIWWv=GS zw3en!w9(XHix6M55atwlj|t_@5BZZ;IN|^EXp|2a!r!^Ra;L-JORA;aqzP9W3Ja{J zSm}$xDlvz*f--d{b3eV1Ro79e%k2!^MUTw(f#T_h_I96IKV%t>pZT+${a?v_t46p6 zjhM_Fs2|7D6t|1rVttma`L<&ftNTYOT$Zo)3^^+m z&rQJlov)uHfIbbk(k1H&!*}2R+a=@vV?=HILq=oaWYRViT@|4!O>-$ecD9^nVu=K^a$5sYD@nsh zFnkkLJJNy#5k9WNZ}yXml2PdU3EKfBc+LZxY>dG3{PZ%N_A5@OD?clQES`>SxEX;1 z%r``o3~0osyy_Y4CJ~dbE3S8C&z;p~%bcrr>8P}0Y4som#E!(!T9xTl6Hc}B;UtHR?xP;A^V2{*sSk236)v1`om!7Jzg^hWBv$*d3)e#gY^wslXdd*OQ zbOaD38#qad`5WQf1UoA63t&J!*Uscd(M4$Ri3Z2f0KYWt&KpmDtOTrk^CH;-AdHAF z-DjJWDqoyBP17am%d8?>(bFG(UxNGO#g527r)+`M;aeff%;vDp4PD$Ilhb3W~jCRMBc52R3+)+zPn8b0sCa`c`T^9l4#sfA(Pj`R`H+w2W%i#aCJji*s( z`g)=jP3Sv=qy`XSRinH`{mmBx)DfIJi8BNFEI{#^$#D87cNREZz-I>XV-c8M{LeV>3itG{~^~@~zfavcJGN|gZ3Mf#AK*M8LCV|C@hV+Hi*O9Bm z^N+K%IbDK9jmrQr$iD)=p4b@rWarNmBx4$&vb^_OzV9K0@+eA6;&|3`TmEucH-HzZ ztZyfXmM9ZdtvDj{;7KL3tYcQ&`2yY&lxoBt(z0^n-P1sJ=3JCo1kV2?_n?E?v<97e z!s|YBrF*wJKFWH4TFk8NWlJ93yJv&B^w!85l-STW7UkGbZT*e{>Gr=1WtNDrtMk&6 zAbG8a81Dg9_?~(l<06^y>`^fDkbb-NGoW1mea#Hx8SQ~YPYo+hA4ShTPTZ<;j<~(* zxOVn9BaF${R~Ss-JA;gn@l@sk-iIGS?}a$}iCyCFe*Wh3U){$8)$(}_KI>J9oQ(zA zI-QHA;DEDl;KY|vK%TBv)&I|5=c=;&6iV?qi>>GhHx;iCkrh%N*ONb)o0$DQ+!s$ z80gh|I0Z_NiZnD~y%b{qxbf4(7-v$xE*BC(b~zXJxJ)TmJM)DHjGNfr z+6I#`j20cw!!IUEZAV1h@7z~cz#<~-?fi&(UkcA7XYLj!*X6L|lioEpGPHEKxF z4kyiX^;>!|nJjX!z_Ms#`Y!w{McIa{Js19Fu(>fHF!m zT^S0@${T1yw#%lc+jx1K%UKO(=Fxv6+}JAEiicvFoOY$qe$mDQwN4nR?>BHDrWGVp zX?VExj_6lP?>$XQV)7Dn*y@G)U7%{0osig-)qOI@IH7t}VSlFlvC1GW?z!@feN!SX zcgU|yTXBFtX~RdNt2Y3eT6r89)*mrlAG=ND1WT*Un}I56aZ7uIZUzM@^JX;-oX@tn zi~0r0P~#Vt8Fmkz{5K9GNBCfhg2dOV!K4rU z0HV><2mVJ5617M9(NEF>Q*q#Y#(&wO%szl%Vrl)O&5_1&A?9)R z{cfhN(7Xiy0l~TnmcH5N`3`<^JXPPMd>PMw}c(UROA%=+dre11K>V~l9D3PU<#o4-AB<{< zAk;X)sOh>5>9anfGhH!Mp3k`|fc}?9?t|d6Pnw%;9zU)71k<49NhOM#`z=m_g6w`M zMYZjd<`oR^$`c3qEv;Su?5kNPP zNK=S`^D%x{P$U+Wjdy%E7`0Z|4^!c#N(ii0SS!z7G#WXv%Fw3@f2XT^Gn=QVfepGD zNW+@ssZtlpcKTS( zUeWt{{vl9_|CS6G>d)qmX!q%^Ck-6CIM;~A*O}iPZw#w+NcnqeR8O@WF5_ogk|C(Q z3Cz*&HRZ~?XLw5@iuYqqDV+!bXT>*&eXY@wUi|m+xLt2Q4t+M6PIvi z9HEIu{3PCl=>CMO3cR&?hMU#(P9J;4ZT(WB4j8-iL>rSzVXvSU82Uqn-hY~l{g;Ae z>VsA$N#MX(#c0=lOkrcC`RomuVf+O$)kwgjUX$r#0IJCQk7RS&fu?MAxVMt;%Hh!fsj|WAO#06iM{o|<5gJktOKyjQCAK3P6f0bpf zK9vq=UheYE&T6dQ%XmC)-mN}kBgOBRc`NHDwcbkucB^Klv}qGJaV!=tw1coQ(uYGS zP=^YC7=l`5600eXqVa-~?xIkQL@eq#80?d%;El!?RtcBwOw!B}8bV&}fWstCwb8-)PZbG6S0`s*{b zr(yzNog6L3PV+7ud8 zTZ342ARufn`T^oJYIGXfUd0ki&HJaSbe1S7kyYu@wrNk8dUrG9EakR%>tN)JR^t7N+Sk#jI z)eKnaH{#98?>)zIXc?ewpmwMc;rl~$m2phwJOas-azq@wR8dhAaz~+mf1X6w#kqF9 zH$GbVq?Bz|Xz+~Vj`+RsSAhW(=e1H?oyf;38WOp%ciR3%QTt79$J%AhSE!SF%%f{J zi*ef3(!e#@*UC%6-R~QqS2s0o$-pDv(eRshG)ky+2D{g5iGN#` zBF78`lCM_^gsb^o>L@p$c+jay4WO#K&8G&2-hd+M(liOzun#2Z<16Iy4(B2VLF$kG zu?`Z9>IYR-N5>pL{4J}{>dzqpPabZnGuNH1R*I!xL5zSMl>t}yzz+%%5PdqalAFKo z&`C3hV&?3(N2KK@c*p$$N7bP|I|VtSsm((1#BBCTyU~(UlqoaTvF8jVL;kbuIW+?w zQgk)fkjHL$5S&s5>0s_qL2F~-hAbWHxHTQqD>%kBtc{Vu?PN{)s0+2~O@m`-H14Iy zWb;+s$AdvGX!Br8war~sh|cg6)9mw&i$xqFAx-h5-Xp&){mZ_*N2Qz4zAT<`HvKBs zTo@n;8vjU`nTT$(PaUqOU?1qB9|ykk)xjp`NGgVRj@FV?BJXaS_uOxQ8aYJ23&P|1 z4`9V?mz@s&r1RKpc>MY5oi7v6a;VL*g&-EGQ(@J-`)Q!+Rxv~2=0}+)Y~4!m^P?=! z**gWw?V80eNvx9ok55OY73Oi#mnp0KOFmsQvg<(&A^GL{opkSW z=Qlrib*tU6)lcC0E+~>{8RZKSiu(1b3*N@PJGQHQ z>JUSilWnV=TWj`V4Vq9-fC*}rs=g%9eSnPSkInKiDpb=u>~8|x-!tKhwn|EyVMZ0B z=~2U#%XJEzP3OanL0eHJoP^x3}gYUgk(>~huffisJ57G|jnm^=QONdOR`fcNu z)c>7vqAg|Al=eg=%%Y-KvsDX=ccSm-J&>i!&Z00rZ!8gIk4z|lKkd6<@L+TLyp+2I zdqoP5KGI*9kT>U_D%&DB9sH#bK;f0o4>d9dOuHp`} zCb`O`bR1PF&oORB6}_g2L@jKp2;*^GrJKC5Y2ZI^kQW#PF#3MzuHl9)Qn#wcB&NHe z&bIJkG2WL}Lk#*zS*l^&`OYp^WfjTC66L2A#~%~F8mpDOMzv9p!hoZJTf=|&0y*pq zijNobHIh%aajFEnPr-cl3e`D4Nx@D7DvO%yqVwJQ2h*WI*_0athYba@kH2}56i*el zwTe;&Q%x1kb3_-luArAm(S+?N2W)8J@ImcpLMX0>tQ5-ZENsFnC6B!^W>$$ZkDXm_ zbGfvQ5?dJD*J_ForSUkvtF?fitWvR4utM@3eeT})?Ha7ZUPN6uwXCzzqO=uE;jlfiqbhBVA3 z6vFaL(Yo%(YdgZZYd2j70ji5?SSYb#RQ|*2)mGdsiFKcexKQIjjN@ogg3hrkBCvXm zR89>7JQ}x#Y+>Gq|Mb+h*tIoSTanie_GTRX8!vaWn5o%tDp9eeOCAvmK0sb5XT&jY z5n!1^2A0($EgYa?vZ5ri-;z8-mC%!;1S-o*HSbU^h&d(Bs84aRWNn|^dn^ZXW?>bsfj@5>(mOG2{4P8TSME5PIquJi9Abo4 z4M>sLZ-1*&hL(rL!*@#~QQM&l!yu|qr3OUy^}RGHd%u1SG8?_eg@~3UXwco|t}{(Tw7$FkNx%T=rAa45WhTYELV8KLT|god=o04zN& z1TRQ{#mG`2WlaBDoH`!c5d`ZWv6z5kn>?4XT@l9v((X{eUxv z8z0K^FQ3%QsAY>ex<*}Bz3q@h{vJlDkvbG<;hWA4T}iV2kLzU6IyImUDtb4dbGWt* z>l71OkXu@E^7xz+idszeUaXlq8G{-^>T#9c&w)K^7V13SW@HJ|&N@m{WeWdr0Z*UT z`pM7x_%>v{6`rTPBD>s|)s0qBW=ZTW0bW!SXjTO)tIS4tZ?qN$pM>0_{oC6A_+!oY z-||CPDN5&Rl_@B7vBNS0WA2)Ec_kH`JGrse^<0f&W%8{v_x_>DC68HFUdD0<)@2fR zd&_dr)MA5>I^o@yr{4l@RCC$T6E>c-|47IYm2dqKoX6=?vfKNa$c3ty_v_g_BRT_~ zSB*1VocWkoRBg~ZUP4@zI8s>j=|Gj94@fr|GCPXMZJ<#p*ymg2)6dwx&aflc^?+m%U9QOHP;vL)M(W96elYwj6iJL7jCi*n7| z<(k5Qdq0TM4`@in_+BEI(H84L3CW8l6&yiltsp9SOZAdDOPaU}iks7(VCHgz>R97G zgLzAy*m(2-r(h1;%cGS)+T>X?f^NtG%VZ;Vy1#b@mn&bX0N`p6ti*rqJvJPz}0~2_aM7xkr1dL z0>$<7$=EqF8cD69f1Y~Zi)}$-KW!TseNy2zDIPZ6z=a@zOx?IRiSI}%Ty@j|Y|)+o z=mn(QO4filvb+kbr&Fk@obV#emzzgknK{ZB70ocs!gc!b$ znW)!&-jGsAqN=B071ZHE)>|zl3lbz+!d~nJRh%kVSwVDh-AV=6i#LTWk$6sHWWj;m z>dETU{t`5EGaw|{zg>PrG?>KZGyb0yYA;;2sbBvQN3DuR$9Flzyff3vRKjI1N{^zc z#a&#CR;HLWnj%uho$$PDiH-Ux3j(YEnN4M~sp~xGbkDU>n@XR0Nk7$u}>bDKn6% zr=`M`5u3Of>Vm^{vtPb12lA5Zb3J|%_(o!6+QcQT z-sro1dUUlzG#oh&zX}(5(MS|J5KLYn0103LNQ+9O1b-Q#8#S0qRjj5ypp#@;(}0%Y zd&zG~Q9W2IDQqTMP?dgOqFHGMN!{(U3>lzaOVEkFp#@EV90#++>n17Egi4pxON4I1 zmM6#AMd@+JknY9yA1{k8mT)nuSQ=sJ+7bD!YokIVPRBA_hwf&)%sI-V=U zNivu89!_Ze80fLzvvo*nciHPLrmTOt5^w@;@U|e!c&?UMSaOABStVr1w}U9Cpr!Q^WnH=VPCZ%#EnY&Ji#`+Y2iSB1q_HoSOYP~-?0%;ea> z;@J@PkV7mi`b%`%ye@Z!lZU8F>kY=Ugq61kE0jOjd&LP$+lW(A|N4ycrRBlRNDrXp z!a|P@X^{Q@Q`%Ls^Wy%a324a1g4^S6%B(#cmaWp|Q(ck*wOvS7LAiFI?zbC11J2I6 zBE8G92wB!2sGsX)KPh_(k)wbs^KR;NXWa0}dj$uZ7@)@0{NxiJOrWP1Lj89Um<9c? z>E|&&$L1;40ulN$Ou`RL74koi0v5>iIp6oDOohnD2T8153}Py~WUVZ&=%CY}?^%iV_fZz2Ooo%x zG#OauAxXFEKc}VB`k@q4u+gr6bGT3ZClhnb_~mrKLonTC1G-H}=6AxV#}4x}%IxH2 zuKxbf7?kg^h24e5N3BrH>KU2TkPf$gIgZt;qJH)w0uf$(BAw!e5X>rlCn7RVc~S|a zfAZ$}51}6KNALD!1Fn?Q?Bcda4%}CZwDd+eMLic#cnX{(i)q9keM!kBOl2zKhxDGi zIAjg5M!VnJ!{l8YUDROKm&`>+h^-Qrftu6&NC1|Q3=8l!U{}X=0Z#<>RzCnJIVs|Q z1c~kUR?qAGd?%+E#e%GC^MS%)Cyd+N@?%^>G!io4kUl_75)5I3+OJ-x&JgEd=rsIAP)J(P$D2C1fl@i=EC6FI-Ojclz~1$jerrveTBr5>GC70Ch7wshLlK#NlJh z5wPO?^uC)Be644p(kX1xHte;;AizkCZ@1R#`5^?4H~kv@Zm9qf^=1;j5Zxj_XJv2;CY;NIQc){)V0xmDRIJ% z$n5}{Q9K>$K0g3WA!4+wV#uVCF0-4TD(5Yybv(t34b7hKHY3{ZHbO2cb@|im2PjWn zN8Jl+@CTRX68=4WKOdK8-n&c%mFL5@vp&9=&rCKek;ckOD$u3CJqrKO$zh0*)pW~> zFC`2SURT-E3T7dHYxO$5VcRsH1FerOjya!!%qE)uuf4Bu>Zu}+BX1HGYZ{K{{*}6B5W+cx^{Ae5*Dc)H z!>uc}^xw^WWB^K@y0+tp_qSaFw<0yQWD+xaURQ6;Tz5N-OHlx42oPVNg1@L=^&I{2mW?7Z0a-0(=|MA~_MWUl{6F z{NknGX=MgWU0LYg%sA)$i-Bq@KwxcgS5n}_w52Bw2US#s+n_5Nn4WX+Q-yr~Tp$?!d;AoIw9>)+8 zM2tOuwmai}V4q%;TZ6q3Y^zMVY}<$bc+*}PrUEz#s^9$u&MTt8oKOUj_lu2ZL;ClX zx72zaKR-L`v6a+}@yAGd0Php?{ja|T78we#{k~KB_vk?Hx;ASBWF_^#Mh3Ntx@{^J zj)&h1BRAb|D`-@SQ{cJnxL5V z-Q{dboshaKY zuXl^o%dkz%%*g4x4Q~>-`bm3o8vz{2wXFSJ7n|NT^CO)$x%wBq@X~R;+te>Dw-*BU z>jESEH+yU6b&F0ZlZgsGf7Y5}M7vSc>bwQ{X(){T%Z%={?Z}N6H=2Fm%FwL8@ter0w&Qv@O?IF`(tvl&$i%;wl6XSU> zW5r!Rc5yW-KqGUVF4pYP#b(gcsn~R!(N+I0Tiq8roTEJYLj4;r!Z`4`Az5m+i&XbW z7xD5c_0Q>HX*f9v&G8HyHd4RRScmsT8SkoZqv zSAUabfQQ@*<6_$9h>;Ug;vr1T_?+!n7YMV}FHem~uB= z*Q%<+K%u*-9x`2|Tym?zM z#cr=>Km=U5Fsf!j75d#@0PI}CtJbs(g>-AoRV!w!dmM+pfl#GgPl}dsJ|ky?tOGpY z*A%psGStj!v;+VzgNfU3rg2PJ`mGf|F_Q0GLrJtq>f?*nUEM?KR_c8Vyi-1)zw8Os zHhuVz%5C^tn&Zn1gdHIa9zysAcawRjQHi?GNU=UQlVB;V{7mbrYl{E^AK{m3f1ac^ zT)2sf0ax^ue(beXeQ;H2S~1I$j8&6AS^&c+NAAyc%U(^>3c$p8|M-v$S`@fSj(WHL zaCEdCh$V}W%o&rF?ZAro~^{%5xj?&{39-Z_c#cJ6iRquAwiGO&helLUTjm-pY7_ekh>N{XUBp zoap?^K(<+0uS~A9lv|%$-vV76!rlo`+)mh=ptnX zS=8LdfdBh8-Cd-xk%V7N6L5t;lWR7ZUgD+lnvE~W72kbHlm}Z}px&sD>wU}H{Mp5K zU0CYvn?O~DRaRBW0b9Mnl)}q-=`?)N!>xH#_DNpRQXFL25$!J2rLh3g*E*cucXznn z&cku`IQ=G3M5j>nSTM@0A*x$4T<;(E5$Xq1&{hPCmC+;LYFDTu&?xZiSI>`2-{$wc z(C~Iw>i-h_POoX`QqS}DxUcU0a_O*8T9Bc-Nr~NZt7jldo-CKo8LqOH0lHafzD8~E=7s;i72aXrahzams zKXhNdAq{lB;;rGi8GR^@j8UgKfdK9c*;OWo1=A}(QkK1ceDc<-x9Q)^Y@Wk+P9^q! zbD5M7w|$lj90*N)v5bicug8K4#jH#RjId!nwl3PdA6Q0Jke7)tIM>5@deer~N|ieSP3qyf`t-Y|$!AqLTdM&h@<8A+ z8|k$gP}*cWk&SabK@X$%rzQ z*?Ls>I3A@B>}RXpz4rdG)f_$D79Rez4NNCxTSr|e%PhA}ihjq_t@h~M`A+_wQ|PMY zaR)W%2KDEVGbdz{>=`H0f5NMvZ+rShYUT5XBO{$_Z0fyB`=hU?6W&#BXHZRdq{6?u zvwW?|6%AhQ&N|3ojs#AYcOml>XVgna=gr%f-gP$BzYk$3%~>d*YY0%cfWb3g6~q+# zCpQE#0I##6(fhpw_%dsxeqFza2cHG()q91sHSQlth3pIbkUekz4c!g`zQ-y$GV(mE z#_wsZZ|A|BfoHliVYy>1#>j5y`;!F8e|j&04zpP~x26@t`yQ{=oot#QB(cvz4ew9J znoPu)7Fwhs7iK~WC)o|D@e?=4HuJ6SYb)EmwfPhK1|3`f*ca0WLq&g5#2V;(?6zB| z`+cL!Z$kOF$)y$)qanXT_hWz%&#t(9#)@fOT5qeX@a8U7CcMs3)sSEC&;K8WMozx{ zL9yI=Gjus^=F*(L!qWCpGt^X{o!I=Hr(yp)aw6LQ_{!Dpium)^ z>F0V73kVlAZ7gF~pKH#wmdI^3`meBGS6kHCIh3`HCK#xcV8^?J{;P%1LOsf}VT&hb zQ|YT{ud`ZU_k5cu|5?X4VT>9>BfKDOK9lBkU}Y(2nYMR@;8m?Wt zH-OJmT9d@h>+yE6N_T^yx;{1UxALH!ANuwMbvTr>8)|-;%P8MPz5kDt_pRTYS;E&j zp$h%E2^@I+I9DJ2X8Lq@4EoSqx?!Lr@%deuM|gN%17|s zsShtJ%xE@WqUZ6syn=FW-Adg)R(Ps69yrGZDpS z-N@cqTNokaPqhAtbXe>Xhl3r7GQB!@yev(zbr^9>!C8`VB1+r5(p= z!J2>hO1~M7>-I2T~Qvl^TFP1ki@MF5if7EV;Fl+UKrii z+1K6*6IJw_yAD<+o?b;j9wNuVyEvv9M3N~XpUzupHX$~`z2aiLF$dDDnyvQNQ;84+ zRsH=(aw%Gxw3!B7E$-*1vu(@UM%d`eE|)eFaw(+_?2?lG-`4x0PMGX_DORG^o4yUF zw3Un*Ih|~1E_>Kd?M%&JV}D#QeWJ!@jwvB}i;CaUXg1;L<{%Ljn9Bd%<#k^qHZO_A z%nyAQ1pcY(_>&~hoBdq1kKV)Q2a6NgefRV756Ntt4qG;HBgPwH$Q3@->mO#Pvq&I8^EBT*H`Aa+6W?xK}AAeCUG&ZqD5i& zWNl4uzu*YsbC7%o#yeISAL^}RIqpab`-KzgXIp94t8Qzuoxq)3kSTm>iqed(S~~&t z$eu#G8RwVxvI?@YlM1J^Mp1WHKhvjQN4?~b*Vj9{zd5WsC`)@R^}KG6CBo7`^L;wU z`?P6DvrpE|$KrkWHRe;}gtuVE7T(Pf>~Cq9%ZK)|w!y7$uoZfP!Fcj zNW|-N!t9DE$L2wf;c-j2R4nA(+tY(D;HAUE<#uW+8pnOk7Qb`_TA=a&b0aD-?H&Fg zi5PEfS+kf@rB@g4VGp2sz40@B^XexVZE@(YBfEbg9rSzx=v!JHNVDGX7j#K;==016o!9d~v<# zylh6?3KsllE_?FYHtShMwH4%n4}DRD@3}2SML*nM=19dUzkWSaq8-Fnt*NO==yjR7 zt;+jHbZd?7BP-@$lEThxm5hm%(3JmH2IzMHw@u(Ki_2zb&-=&P!dH-JWt6X!T0q6% z?)KN{&FQxPaM2E0|0EL`(l{$CYjt_K;vs_SO@`Vq&tdcWyZf9-d>($#ojWivAPLP@ zlQUJU`K$TYhzQ>-iA3acYm#`z)W+w_r{{Y1!)%38JDTB)2A$9)CT(7K*B6KH-H1_s z7aBfnkllAg2hKYxRFsTPqK)lAgdZx~=b<#Ef|Xxn7+NNWoBVV-jxyGkLog@`UHw;WADhnCH~Wi&KbI=z zMdnSCMAsa*w)BhzJziZP{3&%jsBd~bPT9E%II4icc+DsHGY*Lm%nT%u_ypj(D?zMWdFKYbv~-3dHuWUQV?8KB1}h zICDm6`%qC)p}=>BxpR=pY1_#t45C}inhRLDmmO|4dsPA1j)U=3vm6#%FVN8qJ?TcZ zFgV+CG%r>)(evRUZwrKq24R~&LKnkT3x&TI!{x_oJ^Lt|OuA6_k^!`z#Z-y}Wz9z% zV=lw6_#)$TL;pHya3Vw%M1Mi}wUbIRBMaI*KdUq=(X~cEWfQE;7W<#_oMHghLLt4h z8qbj`QY(ob7+^Js&1T}_;<{cd-QL~?FM6_32+K%)r`Bo#nSfWL$;rxtWU1Cl`&3}J z7{1?S77R5=?~vqtkH#A=l$Djm7@5t)`)9M}LMtZVHK}q~v!->|(QtZQEUq=z=N_ZC z{x&1=Z6|*tv8v^`_G~ruR3@;IM>XA#3ch0LzqjkogpY!ktS;TjySkl(qe{WYJNO*PrCsS#H z2AP!~#>ULsQ0W1-LR1d7H%m-eQ)?O%Ps&GSfrKlm92?7enQ9O95<*?#H)@wa(XL~V zVA4lpB@D6ZG^0yV=y6sd<@IT@&tk&?+>t6pxk=P__c9%a7+M_-`5G_+i1#u>aO@Qv zhN{IBmp$~quoDEi;3{rA2M6Uxc8_{vV@c~$?`oP<(#cYx<5b%iQbkQD@7v5hO{-zh;$=H0@?_%36YxLJ{K=KZP%~_hlC|-J5dw^RWUR(Yqht83udeRV zzVhpl=Yd2tAKG`l4}`3z13%I|)glS#INIsz`g7$5(;24U6+&z1{riqkQ=ut!{C~B= zbQWf+tXfZL{Tzw?_+!^GAn0azZg=;K9Y479I=D9oyWbn^Rs#lWre{`wUJE7qS-<(ERA7S2 z=$DCTj0cc1>7ag;ykPsrD^@6hJ5mgC$`}4f`^~@P1@2vjVF*x-QQnb2Dxf`14hA!a zr@p-#PM#_^5K)NuYAY#7b`<4(yFOX)`m(`kf8PGM=jBVdS%1Md@H1OmTZST^c->cW zu?UDEoQpmV;gljapQ*E8MtyNQn~7LyCS75Gx;Y=nkvf^T*kpdzm84Y@>9|+#@hyvn zgoNQJ(JJ*%CJzW| zaZQCTMDumrg+`w0@PE5IZKoC#FhEn*k0L;sH~Tvo_knJf_Hox~2~G5OPx8^LpEq_c zk~v1)5B(UuVt|PU3|rb^pfgnZq7uK#uB+q=Fk7ec-j+|6Sn&2-{0&`8M1oO89Qk&# z@6pD7V$@2d+PKv3_6Rpc;_o)TbO!-&HV9(Sn4Fxtq9d}<@zhjkbvgv2Kx^N=$rSFN zgh_A(BcqS)=+L&kSn<$$|N9)aZReZaNE@*_f0zUndiupGNc*aKRP_8-=t#U6vxyB5 z2n3$bFW={qVpuOPxupI%cEQ(?^XZ&g_`ucL2WzTo!8Xj03jsdo)f*X+2NXG!NzT12*|@B`HK-xbENn^&~k$0MEDW&Ci?!VxRagcif;)I6b=0NCMQHW2zgx=qiyx?ky%#J1tp!8|}(Rn?FdmlnL#rkVB{=8CN$QxMo zFt&2+$ftN|uOM$wHCnV{V~y|Jvc-mzsa|R-J^Y}(FB|U<{PP9cXP}c(2WGmt^uNGu zi3C`Y>A0U!Yh&XJL5(al(hCW-bWZfv`cFtNj{R}A{*4y|^?go!bCtKHox3PyfU7i7%iMocHHdMVLU1 zDH?B`5A?LOw9L%Py4?*w(=0TQoZ?D#G~HwmJqdwZ5Bn?$s|ZFlR^&@+=yT(?L*%KLgMr(<<>mCfR#7Zn~lI=Y8XgJBs)J>Usa zSk1EJ?XjQCwT$pQr`Vb-m>7tos5qIwWR8k}wiL<0g;ifD&mz{N7;DPmN)bK znY0b4VPbkqF4AhgGe0Nn@;wUrSpus7ZhOIm-a-#qkPFRxbR+Pp?1oIulnEC5Ab-~eIWvho}d$y zGeLGQn}6$^T{kZ>T@4RswsEMNza-8wgA{u+L*KAVzdb3n_UfYiILh+-?9zXK31EM&K^ds=;M> zhxj2Y0Est(o#B`ym3{OCVY#$dGmL5;_nkWGu=jqGRmh*X_ zTQ9;9F5SKPn#widAc)87dSWat{!C*Vi7^~SlOMnhQ$^}LFT45sj>}oiRcMv3wJJKX zbLEm%-YKWB*D{8PM+;`4bJ}EZQWbEL=IkZP-}s~A>)w8o@NKj%{bPybmkeQ8)I1JI zIFKQ{mN5&p~#dD9~-E zAa4b_mOz?kp~2q7C5o&1zOkzM0sW0j+0 z_CGGI%|}#}pkQ6c(wMsORZi{BFBl*_FgV>DY+KyOV}ZZN+S{R93*GI#tE;Pd7Bv+W zV}uk_(Kl$bwN@$T;ZLUr>v5Y+oF`|Yy-nWLsmW!p%00cRnT@-+s6)B|iAC2L< zO&6)79E<@^F#HjXvq|7y)L}zwj@f8_nf-))%KOn#-A+ zQ=j+D_At+*Q9>V~!KRc?PCdG38*~NwRx+1%(L<-X{&?Xb{ef-P_2-^_rLN@{{0`A} z(vjNGp+5N$fD!7)x;b@2jHYlX zsFr{udkKs&sc_oY3V%ki3M|ry8?75QvF~6x&+yF8xy(SJd%Y|!ijRWNjhTD!y#a$7 zO{QIb=KJ^WW)r!(HXTprp943S`n&T5{I?DBkT;fRRR-C#yp20(-CW>EuK|iPP)~cN zuv;ODFS`Ex>^23z7>rQij5ChIGCKV_1f&7MEWOzx+CD@szyF8b$lU@%5tD zMjRcM7fRo&VDfE5c)aVc@ItI_q?hT>m>K6JYMv2zm;PyTwiIsv)rs{|D275xo1nSu z*UwQx#y5fhr>I%c%P#{E@{1jW%;xJ4B0%g?`2(sXL1j&+Yg6Fv6ay-pxR(*vBlgiq z4ghS8U_dPhuzCXhsa*Dx3=xPVA`Po{`$U%hpMhwGUA6~j1N9SUz=muvQt$Z;(o1$C zj@aoJUNg=yo#VwX&SN?q;kLsfQTQ*3PJgQFd!B)~>ZLY*sxD&Z9im`jg3HP^xejDV zOKR%vkT3OD&3kdKfQc18H}1!6)1gAuqSMvxkdP1?R`AKiB0DzCF z)(gvCH%Nm>P;8i9998FIA}B(`<|ZpdZ1@)dsik#?EP+YJXsv-t- z))4F_t-$tlx<(t-2|>S<9o8d_Qo!grMv4;JIHG1zEzd!Hm!WcMC#(jt)7Qeq=!x0t zHlcdMHq%6U-4!0T{bB-9o0=u9OBW=_(Ljq1c%^QQ?4BgfFKuY~xlax&9KE;9Nm<2h zm=Bz8jAH z#Il*FFRM0DmcF7u74|(TW0e@4NEw!&1_FX$NlH{m<)4=Z3k8*iQ|eYnK>CRXvBGiU zv$Z)PrR-hg>j?y;EQA_8{3hZ)^yGEhFcQlBY&i9PU1Rpj>CqiUdJH1(qYjvBTE$4J zV0<~@gXNopCKN6ud<8R!Qx2%sx0bGKboe=hkkU(rf-lpQdkPP0in7h|SptPR**yeJ zm1~H74tse)uhGWF!A)0B8|EDHotpY)X!o}<5d)1WkTHz`Q`hMApMfS!e3$?FrSGM= zaaQGlzTBvAQ|DnJTeDcG5}n7)%dk!gOH01#8!Z@Q+C$QZeVQM10IvWtOHg_H`1}l@VJ0>EMUsf#oE$AX%c2RH zP202+N6NKpnlH6($CF=Wi`a)`(4Nbcj+m$el0y1lli8*b_tYj$+DIf#h+WgmusS^E zlazb+0e+756;t^yy9@bCHL*TY#;r%Ndp0UP7h&J6f_FDkj+BD(m4L8i?2in>w3z); z;Z$PlnNP%Gg?rA^FT;@V#el`>KUkgbn^!BYtDz^L1G;YJdDaTNpH>RT`^Ytp{yA+$ zMMbe@Y%$H67$o?K3cR9@XIthu9l|ZeVQbh}Ni_=dkAZB?!4+bYAn)4S(x|JLfixoXC?~MqnJ{I?ISG{i7c5O5ouNrIo6G znu~DNFKJFW6@g`CFLvy9b^6GOTGx+ zWvLWr3-I@VdBrWDw%f$eMV|(s%W}>~m4*^ULN)B>q%>-ZHS_yu8GP(yK7~{LrIT3!z_KI6 zyk(Es^-NW0(Ntw+I+SI>fsFM}A}ZPL4c+`orvlFf8HYiy8Bi4Nhb=Ro-lMvUD#@M! zIi|R2h1TI@BHjvW>uTUj#ZqyBZB=#MgxFX=+!COB3p`)aX(gU-$ikj3)m^xb1;@{; z<+ABM6@%o+6?tJJonY(y&;F(xfR)%0sYf!(@}obc!=zpN5{1WQ$Zb@^3Hz|)yew6w zCvKF{q>_k#Vt*W^s4~jmYr@jnrDuWgDTE<_|i}n2O zFcb^3#icZodY<$CtAY6EwAzGqQLit2-Lb8%%)<8UeMP$c=3qj)_dQq6H;}rW)i#4T zeKY}#iD>xi#f}8&1$)|AkU}xDxL>{MbTgM2uv%RHT$NuE>Z5n5J2f(oLSM$`aalgZ zABi8m`5Y+=A1)nGX)sY7#?)S8?w%Z$vy5-rA@CfQ?{+z3mZQ3R;Pj#kqLZh|K7EfV zrgV97Nb}4XU|crh5X%@X$AMo{C^jFe6CDrdRp)xRje}D?Ypa-CQvVcoKG0v`Txe8g zJ=ldl_S+MM>H%pokQ52x_Vo7dxM$hOm$vffRw)&F;5~hJX2g5c{)AqY=9*B@35|i( zvjy*BXljp@6NAR|Iqw2cG<&RtP=M6B3Rob*5PB%4hrVz&0J=piyz$7zO!i|zVwdMv zpURnPvXFQVqJ7~Sj91#en3quUoui;b<;NjC?6A#HHg!kkOr z3r*pYjy_HNEa{0#-5M3$9XX?wG23P_~PINpJPMS>t&+dn z?@WsBtyYw)**)?`T_t(seNUdY(dfZq1$tvCt9hVynjl4X!)VEUZcW@*9EsTEc#qQ zmw`I<>1bzzaynXj;{E(P!mdD)^ttvBs1ykF;zEN!>vt+}mbf~fJz+0j*B4?&b8msv z7F5!8*XJ}I6o+@YaAJ;yM@Ke^_MLFsLM8d>r9ivxCe;j_C@+O|=btDP4Dn;__}sOK z1R{PC@NRX(nhvuuOt<1vqY$$*s)qlhn>kWHdeni7tK6TVl#SZ}z2y$8UJap}8uc-t zjc=^zu$89m!J2=aEU_QonNn|Bt>h~Y^tn>}3xvI=c8*>YKFPnl>U^I;v-=8B)(wk8 zos+DTC>qowoh^{lVMiX?ou-qtsDBcF|Ah3@x!U4ebAwDb_$fo3^8y_hOYSNCV9z}I zJ|V4wL)3>I@vdCWK2q6{(%o;_K~C^Xhr+7kT)X&`eZ-mtT2D6L+2QMvmfL2db1eg& zTS88OL#YHuhX_TW_yddOED;7dbb{T)%I6mp~FPqn(&)P@Mb!j*czM74$+fW)oZ%^8;)i+5pK!8l#EQ(fxoOe<__3&_UHZvJIQvCSkdYzjlTn$PxGaW$FaMY$LsWT{0>{F(p9F5-=$(V>|_)1 zUy+cUc{0bi_qmE7Z-B~?@F!BKl1OvKFG!u#j@}VKl$w4)k^G55< z!Gu1i-9|ry65O|9Parp#{SIXf^hGQ@`fi8^Y9&cXM_|NeM06AOovk^bm9M2d?MaH| z)-=I^uaADYiSvm`_4P;4Ej1Mni!!vP(-sCTAiSo(G8ivS(i?^?7{6T70b1b)(DN6fOo(y# z=bVm5G@Q>{tnvqlDB}}4T0f#y0PP_XQUb!mu+UJ&EjAvWmuQ`oSpNI~A(v=0@?6Bl z6J_22gtH-K|2zVHJ%lH<)qcKG_6p7-AQVxQFUGR6BI zs9&I1+t{-WB$%>l`XccGf;L)+Pc%f_jMjsXgER9*1}6utRK1LV{_5ALK!xCj>tKJs z59ze{6KEa1?ad=V75JKqLbP2|qMqO144$imJri1JYy0k2#QNU)Ti72x)+6}I16{5< zUWMf9pAnY&Lc7bA@kQ(|!1~J~d&8eI2@50}!Ss>1pYMIsXZE^v>Ej>!ay|HE1hLoF zcv_6Mg4?gkBS+EG9EOgsl6wWHJMi+@`Jif+GlZTXu$<;8I;kG!_EI^{6nPnwW-#xi zO+~%2B~ff!1w|txTPzGfbfNs)r%QUaw|hhaG1I;IohEra*{pRuW^{YPu(1gF@t*Jg zp#Hm401b)~1TTVHWBLt>%nDV{qOhZV;FwbsQzz^(vB@4iE>4WhFzf&bk?u!#v-#?6 zaMm!|tO!maZgrtr?w<(z58+^F!ax*F7w%yPg1*0X+vVJJT`|(Fnfh%DoKt3E(U( z&5}SR2+xs@PR>2S5*M~4QD^ZB%@jLUOZUBs61cZXMXtOBpmKK-Nbkf|s*GUvB^MhV zD~PYaMg7-jqWuTgr~xg7Ud{zcIa;@@R(YQlj&!j!Ho>}6V-D9^p3E*c@G>P7aFt{gI&C;r;Ri|Go9Pum+Z)jF(cl=5xp}>_6Bw(YkMA_(tLhEN&KPz3U z!pvT>t5h-}Jf`%BWrQwQT1j~iSU90%vS!7`UP}e> zIlJAdI0C+SZVne+N37ao7(_Vwzhj$;-oUwV7TL7ht#x_`uK*uPk)vAfM@T*zZjNHY zE|%z(kMM*R_i?63(2fkHIte_5!XN zNYJW^T6|Dt08XR4qg_7hEPqf}k z0Yfklr{?n2MT{#hU0)G8B>W8Z>Uz)P!&|Y1iY}ley6ZxLQZKw_P}$Y@Zb*aK?2V>> zj*A7^e#t>CJ+ij!GBZ2t|7VqC;G(VsC7avCnF1?gK?n>W?|P*#mn~fIs-TRAS@o@7 zyj7uWfO146HmSDZU-p&Ax19v2OI$AEX&x6S1Egv$ZgYV|lv?{6Xb7W=x*uvtC}r(6 zd(wX}A+S5+0n5+UV#~Tf<+}mvs0)EI zD5P%t`tBd_m~2|6@uZP4NsNcX>g;w-S=|O#2p&x8HY1Q1+e$N@TUHh+(a&A^4N1RO};Zlwk zQoo)e%+UIzG=Pgc`d0-5O$^>YQS)|3%|@p%m0x-lwNSfQYyx?~i`Wzfv7_WemTVn<%m+l@wdWoNQw2vQgVt=Q8?WLV_)Ep;9{7;v;&6tEeG9vn^IAS zm?N%OWzqq&*f<8_RrGXPv*XwVS}Fw6GR4kUq*S`GsXbUY(r%SGCQ^F7%rY+V-Q?2n zx%4kILz3?G%$rH@`w;d-X$IrsF$cb??IFU&C2>m0$853Ktcj4ZsGXq4`)eMF1xczD z=j2EgE8KQ1xzktppN?gYtLy0%EjBfM=2W`q4sqMwHkf57{5<}!B46ia3Z;bc|Q=T85bcHv7z1{P%w2fxclU1;wj$>2ifFia*jr3-WMX2j9hR;f^U zB)Zr=2dNj-Tu);^tFYUN`+cD3_rv=lq83VugabS96*@#N?kbbnp)_3QRR|h7K~}x@ z8B*_tLA$BcP;B`tu>*O9_}i!G=rjm+WTg7B>u)WaMK`6wKP_(C+Ub{u%O+IA<|_{j zN5`+mMis1+*_V*^N+RnOgo=yN45~`I{RoheZWIlYGAPZ-m6WYD{we7&uyXlRBeHu_ zMmaerKt?E%lw4BBQZwkJ_GWdlKc><)du@Z)QTl4;80PMnL_(GhFCYKzTzdjpm2bB4 z?I-#GS*sa&v?8l{6-N!d|C(0XA?wr0mUQT)Sd34gUfqls5|Ws}laRjP@VYsf%GR1{ zHp@~L7DssfVV7rA8u^1o{QY2TIszGuL~@Ue zo8fjrl151IM~|<;Z?^Me(S#EW)69n?KROaK3&^agqL^2Ub~UGva*33`R5L3#$?G6b*23i+>Xj$#ze` z{vT8sj+#qllnX|)KX2kgyH7@^54ehrp&mt!ZRQF+ahS7XA>K6p`lxy?*; z0&TrNk6WYh7W!S6ah`Beah>QvExk@%14-UUzsc#fvBT*nKlgzot=REwe( z($UMW5!1dVQ{x}DPaMNKKi8g@o;m>E$CeLx3y zw)o|7_&6;G!qOXcE1_oDdZ{_Vg%{MAT=}T#I!j687Vz1H6 zniAsK3|>?STd)3`Pk~>=!OIYTIr0yD5zna2(h-q@n?pJ{byb&0RO&;mdc(^ec|YXw znq5aNJV&gc&CUP59x%aN-BE4E`;HA=K0Fnzsu7Xk*w+2)<(^5_=5h|D7p@AY<-8F+UbT6`1s#xo%>nSZ>8Y`NVousnCSzSj2X-Wtn9z^df|( z$fN4;`LCWZO3%|uCVUApmb5dwr5kKC6rm*#c!nHhYcf&v3IAQ6;s&{Z%(y)%gJy_p zpp~m;Y^5@H6b@ONa`1-$YH2nr-`Kh5g=_?rhNN6-@Ud8)VacWaug~N&ze-I4ASi&Ac?1|h4)KWAnh9!;oz?uhU)#KPnnS#E!q+%8CGh>9fr*pBGWS<7moj>c~ zP!!~Ma=)$|Q8|#3PKYdLS5lVd?VUvpi!CvKm_3k|%kZPrjHQEHeJeiY>&z`@Hv#=A zZv8bWt9&K1j4|V1iOD}i$7PgY#M+wS!|=!_srFycu_UBOrTvygpHl-h<2uRsgyAcx z*O7;VXdEB&8dlwn@}KIPacCGBAF`V^PUyWR&k;d87aD_lfr5gR_Y$;yXkr0lD74=)&K>DaqRT zN#`Z@Mn1_dhAwKTiiRK|r_r;_nCi{iU1@q|e&R1h{`>(Eqi&R|if0 zeC~g)^jD_&zq3UCVVeSI(*LhpC4$t($grt7+t2*p(Ix-Dz?h{T&;Ik#e=hd_zr;Te i=l{Qn|Cg83r8h4s!4~nr>k$a>M@mdyv{d+w-~R!sOS1X^ diff --git a/toolbox/Megatron-DeepSpeed/docs/images/distrib_optimizer/sharding_scheme.png b/toolbox/Megatron-DeepSpeed/docs/images/distrib_optimizer/sharding_scheme.png deleted file mode 100644 index b07c25b05f9e2e7a2973caa296126c724da9f4ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 99135 zcmeEtWl)^a(k3B5kf6cc3GVI=gAeZR4DOZy!6CT2ySuwH5MXc%?vS7XLfFau?p9r? z{k6ZgYO7Aw)SUCy@$S>z&(o2r%Ce})gvc;3FsSl!Qa~6OxG5MI*bqcmXiYSPa0dGG z)=ffQ0}&B%eMjXx3=BDpyp*_xx6yfzzJvLy$HTlG*$-Qo+@{$z``7z}^Ooh_ zjgI#AAKzi$V8HxG31a-#^_#}`KKg0Y-VNy)~ zj{%^6gXSACjfaILhxw0!^1<@I#uYQ+f-^Dw8b($8U;QKY4@wdI=NjH%;G@3L zNiW(#r2nt}|1JUH|M?}*=^@zJUXdI`9iFDCNsq&6A`MhK6CmyRm96hib!bo$-YZnx zK85UfZ^chKHfc@M?#<<64=PX;PPn@H6K>*9NZcPDZE>AirI!z*;O%o>H+$;Oc_o?Z@FM zD|K`i;Y>P*v)3gy_*C8rB%G4h32Sj=%$Pp?yWvw0$cx8L@DN*{7QN{DCHaiRlnhT@dK3QaM5evh|1 z?;Z+&I@H_&RCqWQo-f+9>mWvUnPG>^tq*8%Cj!-JyVn=Bst0ip3o zA6G8qD4U7jym8{oO;4-S_B4~|N(Fy3r{8cW07eW2@CPTp#e{DEAzWbxz;=sHSkNaM zoUDI1;EeZIqDKKH^CHuZMcK*+xx^>xH1r(?CbXn+BZUplD_0M`otw&Ki^Wwe5aZkv zirL&xzm2+EW|m#kU3r!yIJ$o*H?0DZ8Ll*G@z0y-bDTmK z`oCq^G0Di*}6&<){+$&^iKM}_Vd9i0l&sa%zD?D*o` z%!JK6x%7TJvFo*D$&AR@w<_Dz97ArmA(n*On?@ba#ScQun-$g~tdeVmdv-PB4~h^s z$wd6pRN_x_Kj-!|nJYYX#89D!;AUA?0YhFx?)(-}(K*$jEm!5p{U#BGxf%uR%WrRM zgvIiSE29cR)K%s726bz0&!N^gBPbjuiu1PEDD^FFl?-AZ{;*R zIurJFI@$B7B|8ufmN!X^OMCi@#o|mZHZm+O_U5#Iuon4`Q>^s#&{Yt2BhA_m#ST~$ z%YxMCtaZwlUm=l{#`;+)c!b}ElEy+Q*Jc9&zfhsVBoCWoS5VI5!kGbkzn*SkMec}^ z&H=k6B_cXD5h+rM%}_(HAex8*{n8XRsgZ4c(On29lAK$-Ap(uu)p4;<%(!NwVvU5SAJCdvYTlKV_nwV!}?sJqu3hKq7%jPdF@UE$27HVF>Z^yVgO_&f8 zPym_AvzSdfQBw1av}ZIcXC+dF&rV&Sg2dTxKlAmlMzF1}alC@10uZrM+S@a4t|^Ks zOC9Fq0qUZe9H~xnlWK2Rtjh5xJjg3Ixcf1SlHF5b#w%2cn@aSCn(gjFT13&4__Y!u z4Jxz%$@A=0W?&|*Ge;{9_jL~jIS7JWdc)beSxUVyTXz|tW!P(P?hF&erJl@DJ@ol8!4@5O9-)}PDSrp8rZ9_Kg#r7q+;m; z+ll0;sx-AIh55kJElFfn?H#hDkUjCSdBjA;i!yy8n(XcZvW_jJ>e{3oSBzyj=~h!35Z=LIE5*!x29%eS|z%B^8!9tg&`XM zlI3J&>-M@~bg;L%_!wW3-M=w8!;Z9K1PPYXx<9AM6x}qOKST*n^`f8Nxo^DOBvCbr zU6ERuNj$_xlS_1{-L$lPLOs};|4R77-F|>$B#L&+xK}Vg>8Fg%=b@9W(fOFttktvT zKrf%Oeg@eYj~4&L#&G@U%%?&=S|lD@9uWc26<3yzx&77B@^jIpvdTXm|BMzEI1|px zXI(hzm8t;G-JJsFNE6#bmNq8u3vfz@P(8Z!wN8GUgk(f3pGs*x#J>MH18<)b!0DQM zUSm^l`MWw^vWFsHfI$oCUSKm@8ztGGomJUu*`X3gx+7qPZiswV+0-TTPwomS@mYgw z^tx8NLtz5hMU50h=r*I-*?}qKBA)goo1WfPtwrP5U)CO_y(2ad%@H}m$;u_5P4t6? z=Dqgft#i}aJ3PE2zpAjXuuD4^(pr1VNDcNhZ$su{a~U<5(#Rjv)ps~JIhU8E;!(Pf3 zmH0_LBxON^UjxD-xpPp|ma@`33O(yfJ$$T|%AX>bTP@XbGmV!&swVMTFH-#kkG~+E z85G=yylCk2kS<~UfcyKbu*mnjO+3`z6E3U)$2|Ir+cTYCKs`>3o^HxssqZ~U@=y=~dN zAY+TCIq?)~s}Bh5ZpmleO)(-_8x-0$&g10tdA>17yv#COC;C;->G_txQR_OHIO!FM zSr=_swG~~AqkHN%;gn|!o}~-T?p`(@PNNK&bzvjX{q1-HgR*wH^d)j;)msE2EXIP*EbD|I^VK0{=l0Y zYMYw!vgj+tT%2|8YU)U$zPDuD$Bf|%>(&{>h9PVj78^N|Y+V(TZ<>nfUuHndvkYlb9;T~l>52zgunukGvSQ!`kFAGh^9m~DcQMW?F=W0&sPDz(ie#$dPX!^ zrzw9zj-Zk)as@3TI<8<+^h3_~7IrT8@~HZ(tw`LvM@Kty6|? zY@MU&sV~PLHWV_11vgx<$PIPwWuvk+3s+lXhhM&K!qM0}L87ViU$ss%5Z`%qnZU-) zMFRS?6FK4*Vy0U3BN3~owdBaAsk3#zcX|kUd8C(Dj)Xzzso0}EB2AqXWr+Y}%PeoE z@@$ab`kBl&v&T7@kHoPt2gfxF@bnR0TSK;Z7$3h8#ZC-skL{Kjbn#Jg`-^R$OJwEL z?(Y0y(EU9BBC}QV7gsBLm4~mnq?e$YNoB(Ajrriy=j-=pLl(XR8@r{uKB@MlUn~0D zvwmI4UECOMmVDIeSv`o&d0XQe4R5PO>V?Q$cQzy$f{bp`)IM;3#$=GJh=cEpUIpZ~ zHX*038e4N2Ezj_w7;PQai5~x5P|S%s&)*_P#OT0Xm>LVhPqD$*r)X?!Y9ZH0Hn7oR zA~Un^EPSXT@lRh~lhCt>-)T+ipg}8RJET!E)3si|jb4!axz|tlQ!6#hg%q^~QUw8$ zjfcaNor4}0H6|m1#G5b!%CZyB5wfD$!sL;ws<5Q(#!Gl^ekAL4E|@n20ed4RAJSyx zru91Ix!e(sii)2m!+zOn_Ub^oqrlDco^q$p2uEfS*(>yvQ&~XkF9tPcn#qLz6`#ag zR4K@Tl#=?k&HFrtsBLYQw`GaUc0XT|ugAWR7&H@4x+bI{{^RyZVx~ugR?Hq^y~wtL z_f#WK(5KyZtZ?yc;no}*DFj{8vKrV(ZCl@)hdn8FeXV{mJz%Bgf1%pdZe!zD{_U?H z;~yPT`gAmP{vmLWN%556jJD4yRjB!efa!%cxwAns?_nrN_56H5+9kM}D{KC;N{BqT zXMhm>khTWzsyEkG;KJ1X~lkFxPJ<_Keigox5|q zfaJIG7KeOpH_>Lt{Tr3=Zlzt+XvyhpbU(}9Di4W=fCE#R)=^RwnZ9z&RAWcHc-A<{ z7&Z<4fe_%(@?M%ejZ7>6#VOlLQ=Oi$e*{L}_f=r3TBc9`(cD^`Y8zpj4o#&CD$!ir z=LNY(w2HJX$M@{&eEv$?=pgVZ2%Dole0uF2ZIuK{v>Cc?j+Riw;MQRG7Otym*MK0} zVXEnjlFeLqyE^W9W6EOno=Q)EFeg)`P>SWIu0T3w(UUSiwNL!q!6h5b098OfpWXPo zYO99d7xg!6(;_uVHpeXK6-P_VBuaW#0ZpY~F2Q5KjZitO84xFyil88#&k9p-Lx9-i zQ+Gwoj|fdaXVXZE-~62F09^+*a&+Fm4!6lI9IdHoBa32H?V=oT@@e9DkJht~`GDhv zE^hTm$uD7X%i)R0{L@M;oA5TTGG|w3D1CrKs@Js0cH(`Q(Y=W~FL==6nYJ{nD3t~w zIRvT8WSYMrNm88`T#Y?dIQQ^*z$W!o@mT$2L0SETlC!yXlzgkMKy$037$C-3<#amtflTCsp;s&M$e9c&mD<<5gnkEGqeHnI$&{X*c|__hhS z+PwZzi0#fzb?1#kDQ`N#7|+>LdTpMM{CsDdMG0voXBfB)0#sdO;Tfk?;_}O|!qA^R zPs=9heB{JOS(h%|H5Dewh*;4}U*3?&v{Kzx(s!FOu~>m1s~z_Cp2v$&y`V&|nwbj3 zNoh^7ef*|fCVQXLsMJVgbvbz30>kxW{)kFD^AxfBy?q+0#EXJ#?f4n+byQ4WsXgc; zcFq($TPxC**i@-o9xuT!hY3Wd#p{q$jrcGp@Tc9wq23u9B zm=%pOzWbigkv6G^ay{zW@r7X=ShGXw?8Z;DICSCwJOlM&e>rp>tmAht(wn<4inWOL z)JKi}*pl2+tqEtZ=_fp^SnFNlM%76I@>63qTypXGT>&_|?OwO#CGy|aDhz9rIFVqn zq8TSsu{o$&Cj97>Yg60Gin~z*2FC)I4u?lH#yw=)pzx2c{8SFFgX|k%kT`pyGDSvN-pahh z)Ln<^Emdm4wQNaoJSH#7juOpUaq%7|K#eKSR2-5;uSxb*b0iG8qnf^fEenfKCq}Hi;>b~3K$aUzw`#W?%ARhzY|wNr(|#>2+)jg+7= zA=q<*Zo~|!<>JRT>w#8o^ieZ$X>IGp(ZnkZWT;P=0=JbeRw+o*^H?$@=I|s`q`{J? zxbEI3Y$OmvZL^5Ob&UDG3e0_Byu}j;FMQ$KQxbMRCp*UCGGM_lBfL0WUPYH0T>>;E zV*djt{cFEsW5O2sPFcimaiSr?RX9#B*b#fWc>XZYTq(tb0cVzhUJ&b77@g@IEZ$VH zp=FaL$4emUTBPcppgf~w4D~m0gjCn&cR?1T-FOt?`PVWfax=|e9XBiM&^4X;XA_vp z8F3AIh|xbzj4aSsjctZGin=sd46;O;CEN%H783Hm{rTR7Dsgfc6dYb;I&%XA7}nM4 zc}~!(``sICXqkTENJ10#El0&#m9n`h--Mw3`CiyiZ%IY2Dc1p?A69fCkFQ{*08)hg z=?(c1yWldwI=+N#Iw|s4cGg&~Px(|MHc@N1 zVtNhRkBBP?L~()LItvj zy$gY?{6ZbEEN2WXlVrFnWOwWWWKK!Igm+J8yb&zsW>31EE8M{5ro~ zlEO2|=*wS;e?teG^uu)CMXOV92Ej^O&{9p5Yi_?zr@=N&hvVT~MS5rC}+QHa19lHCuu@j3{foTfwklhZQjhQCu%nCjBQ_ttg^?bfFL5Bek40C(F zR;W{A_I_SIe~EqmL*J=y6Y9ZkK^)vFy!pq5i|Ai~rp0+|v@tVJ+=ekp+~bs<^nG2WN-lT3A<}?bile5B<_)W4tE~Nc=isGX;c4FSaRju5bK+URrSvLomHXB za~&oDk2^jmL3&!TV#8Ws88H@n?$}I87>G%)M$4Gqt7@^FP@wJS#7_6FSia{fPrbW; zr6tucT8E#l_aqRh)nR3nHgn=hvEk1nN$T~_e}u4W4O$?KE*!=$$ex`QA<2J&o@R zItONBr9ijmOI(Ftl{6t5VejVQVv*DrOY2$D zz}HPcAWw#3Nn#`%*sA$lPzSMez^P1o!;`b?B8(hT;%cwute;6YD}i&-x?FqC;nv8f z|16JXO*bKrde4u#x^DJE@A>`O(}o$#xe4aNEoWxz)A4X)RM%Wa<{R$xkhOV;<%InL z6;*n1z0mwkz-7@lSW$%?O3p~qbl6X2usvVLvM zzinBhk?Ut|6l+sHSuw3Q%nm&o3A*)l2-Wh~dqeVDHc)kpDm!33W@MjiVmE&3Q<;Oh zP-{3uhcB6M6&rX+tw8qi2CufP;OyO8!Rp&PnW8y|46eG(neWJ9fo~?A?6|>IDYpiV zaR-#L#t2_DPGBj5>0^hm}c;n|4{j^}z6I;G5!eE;Dk9&_GzFzdOZU^ahg?9Nd6T&IzR|yy z{5oHD?NqN=$0`nfc90sv;ewxPRGvJLnVaf>^P9jdx_8{585M#YuqO;^5F0QmzL?Y< z0o;~X)#&NOxTg4i{76`N6tu@LD@iN^I)-w!b={8Lq|LwA$^;TEV3ok%!C}$Aw1$Tf zG_$cFaC~I4>=syFF&BRL>b9JjX}K?@FUJ{5d^thlx*Qq~3pgw{7KE}+IpLU!23NcxpUXwZKmdhg(3j>81u!bmdVS+IlQU#wIlCD zdCZQjT7Jl-So~@A)qZkcHOld&&*^;CAP{RowR?>BnCLvgiaSN2q?f`iRqq?gSMPIM z;?PX;?#G9hl-<72(@PH=GyBfK{mNmJ5{n&gL z^HJM{AK?_S!{T)3Mgc7CmesfKpzUdJCC0-C)rl6psbUE2HyRyXV@$r9GcPQJ)+#S)J#oeJVM-oAb?lsH+d2j$zpc}Eg8Je zd2{+y5ElGQ0vEV>-I3$8n3Diq=aw-fJ2X@#nI~H?OA3?2_J84&?_wrcXKXL$c4BKD zYGl^D+B}(sft`;jt;PdZEVH+cU74m}1s1Np?p|qYQ?iAMZ=(TzCoyl%m`#|UElLx1 zOKIN73ie+s`I?vu8q3#GC_8Z(?q*#K_0-&@jC4fQ^{L(}=ZXgY_|r8fwbfo!oU@ze zy~I?ebCWpZY^=MzwpkYJx25g8>gGvINk~G&UbwDBUz_bK`REpZxO`uF$!&AP%x%7Q zdMVZH7H7rfYuj6`F+H45sEJ(3GHB73;4y9%J?3DvYU6k_E>Z9M+=_z`zHe~du)sSD zJdfyAPrV*{?r&IGN&Xti<-X~=WnzS2g{#fm>h*yBE=yaW+^YLWEoBbLT|FsZC*ZyJ zKhTUPB*npSdPm(IKkwq|drUhC-W=E8x3`QUYukne@t+EPSWiAW^kzmHNt6Zc*!yv3 zgs`5!?esl0bDqZ-j(wfb95zQBK9-R!=MFXSexDE{QOrxMa&-J^Av;GYXY}322;&Rh zlY3>7H9ByDg*DmX!Jwy_A;(VSRq?LPC831DSU#3$QH-Lk?juia{Zy&%_yc~ZAq4Y! z)3Eck?xU4+_o{nl$@lpgwOJb<8s-tmk~T*)+lP5d#Yg0*KhtHb<0A~Yj){N>uRG~< zix|STatp&)MD94g7pVm}k-2%|GK#m`4~7 z6-hQtINF_|;*ez^5@>3WQ^2uW*!0k_0H}@|Hcy{Ag`#d>m+z z7LB;7srvH#9lhusSdxVH*AG)$d)P&~m9rSZ#+^k@zagR+8qN-2ic|CY-!M$mT553* z-hn}b>HUo6m4_XUQ4_1R>tbV$&q-m(kqnt+M}tvSSyUq4NGGOHEnC=!glq*C5ZkT= zwt$UYdeR|NjOR1b6jsSy%Dgl}adusRAK5|z&2>Sk?CuHey!53ae|dXsb;I)%OBxr` z$K-9UN7kC?uUfMcVb>X(~oZC7Tc=8TLkooLJ$LqluM^AKu0mrvUD-U&FqE;q)kdy(ykvLWU#72 zy?NUzv?>P-AeaI>dr`WMmsLNlNaO?IX0v2*eaRx+zglHF12xZREJ{WzlCqv1;B z+taV-g52bfdXC>5JJEg#oFmbkvp_!B9(SU8Jn>(APaQ8F@=TVyOWnMQ6geZ(%`In% zAch72><|@42dO`BrbU*=`>_H#B!+g@IhTk{M@vumqRT-Bv4pSl0x%@_gSWDo_mPD& z1*e}IQ~8EJjB(evV|6bz{G5Aa4rt`g`q~}lnb{PsiB_=1Cf)-{(h|>|3lB{9assS4 zccy1)ei$eu%LuMEj%uNuraVU0+yQjy*h-XedL~b?>(&VPQRiov?e)phL2QuQ2+>j+ z2T<1B!=@@n9dyn|sQ_IsAKe6!?@~aE;{4vh`FV=U^z3@7EqKHn(7mcs9%DH@(vRwT z%Wt5VrYA(CPR8u!?G!Qg0Ynl0O&yi+tGytDk20aeg1l4`zi86T==-yslMVfgbup#B z3#NGG9w4x;@*YII=`~4@W9}2Ur22BHwR@6J19GREB&HM22>Y62w(l;Y5?+U_O`P5s z%n|&2k?JW{*Ckvci$Ny^Wk4tj2O;UElR`l~14tRB$fx*84SmT&nKs=DL@MsNY#bYH z6daFskyDuCo>a_7j0i>^>7FMh{xVeSbxPie7Cb$hB=c7S+( ztow0*ID9u9_Tdffnnb6zI(mg)x^OODw?OPF02p1zu+9?M1w*%|$hWp2KVdm3R4};? z$wD)p-qfxw{MJynW>v`-P&U0q@Y<7x_N)0NNj~NCfPl3}QyKeU*w@%`k$J}=9Y`u; zOx|8Sdws1b@7OLHBiW8P)QsDO`$za=rc;Y`jmLXMXAzD90ov+Snz55fL z)IR5sk(t#*S*$ceb!iAvW0$&$5dk4Epukt5;Tn1@)~nR@q#B)j$5)lYu(Z}zWf?GM z#FX)#2t|solr^B4!0d_#5URGyU_}G$!Bets`Li+w*rGMuC8`0?`dkqi--qv2R7Y?Q-YQKIE%jN`)7@Regz@j{8PLEz)=Jv(TiQgf0@uhUMKKC8sv63&w^quH+a*2b}M(;tQ( zY2HzEKJP-NNK^ZW=azXed1zk5*40c(g$_)rfonRy26Zv(+eyWhe6MsXYzQ5da{o4U5LC;TRt@KMZ7yw5s-6n>7&M?$9r-sj1gt&GD~}cjOizt`eng(b9g^m zddf$TxwHf=-mtQgYw$|r6GPn*?@m*#|E6z8A*f37T^GlJ4+koV9|Ie+O@jk>=bePi zHijIvPsyRjt?od3Scfy8&=Vp#=3f9x5m9kwkeZ0ou(o!_uBqKuR^^foO0LLgbDSDM z6?}AMAwb=J{xOFF{xl(t%eZ)cps!4*z3iLMYHzt90_UT+FfpA})c<=jajEa}eRUc8 zt?W$hepBhom2yw|%-mrMaoMAQBF7k*+5D>Ip*o&EGwZ~H1TVL8xZRt!=CL5zCb-h= zOJJVb>sU4C8c37#Wbnp_JpZR_(SFI`7tYX_wI)HoR0V5h^cS1O#)|CbqSMv68{e@y zUBk4f<$D?@%9{I?+1!H#AHjDKmniERb3kw z={#<+GrF7$QU#pdjOiFX=ZbO5DJxEuib0D;GP2nG82!_o`=A0~uEq`gf`k3SfR{#V zK+xwU+K$+KQ3Ix0{_nHpe#@`kKiXQu5^4Li9y^7kKO-wSu$A-HDw5N^x+s;m6=O8V zo_}sr_x)Zbso&D=t2?8!NJ-yDTBZKMALTJy9uhKXusdU;KaQ5CV57Y>A@7IeVash# znU|*?Jv>)rFluKJxmHkZRpi#x(LL8+%u2dYJhv9_-nv8xiReY0c2=d%StyD5A;^vI z^h#oP>e1-4gA9hJdOTKJ`W-E|^-H3*T8Rv?q!Gaz`mS^& z#NON-~NQZX5+o*@1;vrdhczce_a!xwEuWA zaCtcCFZJXPhs%Sm(uT{`6`!TMdJ*^A**sU1ldY?}w}*}--y_i1zrL-ffrBg&?n|mx zTC)umqhD#Hb9+Zq@mw?$i#!PhOJjX!i&uHc!*6dVd(BAT?tq$s71yy8>*)78FiXPL zd`DFA!_J?X!&3>V)V}VP+5n{h1;G#oQpf3vL?|@8Hvoh8T!kr*r0n8X(YP+Z-|ZHw*9bv)CeN|||Fym*U?4zd+2OKr*78}b0@mXGna zwK{2>WYaHkRz2S<#{Bg4=Zv=3x3Uanm_BXPszjJVF2dsNTg~&3tQaO%CnBs?(4fpP zlaK`3uRo75d|r^O+;Mx;Fpia`)WeLgu|gVse=Al8pgrOEP4 zECC1^2iKCMBc)vCT-BUDUhYV^F=lo7(sc3(Z>(R|No9qH%*S}FQN!F!?sS4)$V^du zx+_|wNClS1jcNv@=)K;7>J9go_{j8TR;k~m-JI?7eq2@bt1Cr= zEXXuRb#OyJ9S?78eLC5Y-tp^`Q-NxK8Ihrd58#mcW)T95fJ?x6;_PPHZ+EGJj;FgN zY$XDq?J?u^8S;;_TDnJXOON2c9>u>+)BFLxU&qxG3fRmGOext;v>kL;Q54*vYGjjM z(vPlApsyNs5y^{u;ALyGvg1cLpXo4F7MQTgqR$V?dXuzu$i}d$n+`Y5mn+`j_NL-H zM+kf+LX0k}9X&ktp0@Ih>LIaadQQne%!!JhjZ0RG{X0L2Wh+x717e41bPYg56dQLo z3bP|g?p2M=kVt9hx)Nr`ivFC8sy~YUVQQiHH-7abnwX9A&Xh$PeDn`PGFlhmWnbIg zaVP);vb<@oh5;6pmx#=Y3@t@=6m=7W*s@((w~su?j2RR+q48G>US?&V>ZNASZUT~m z%MrPLe@NfP|J@!7wN2lkB+WN<4Zb(a+$#dKgJhg2Q8yp|5ONAoS)tQ!E;Il5_PA=M zR6^)rV^aM2wjre{&rc=u=GX}fHr*XA@lg}oVz;zuH)bOoUOhY>B>qbIj3m3`n3iBGx&a`<}2Dw17 zp~I>jMa>nCn8|x-{QRM;P}gAP><=nCsdZBz_@X@bk;k4t&YcVnm6#7R%F0|X2Ykb( zI=Qp@Id(fLsrE#)xH*-Dx4ugLhN65q*!W<$$^jwFE7hpe!LR{Qv1gn6nHk@(oM-=p z$Hg3hytuehuJp6QEgX;CWsC6I06AqXMr-(uKOk+Dp;bFDA8NiAC5}O8oRg=I!gotl zvsEF%TMGE*P3{$$5R*nq$oNldf??jKW*QBkc%5-I=ltSbOiQ&+C8cnLmHqG0;5i1| zstDjvNB;&uv%K_9-SvEgUnsULaShvP?ekPeR0Po^M+hue2?yy_-gxb(yx@3!6SWY9 zRZVV%5u|hRY*de$#!#vIH7a~@njS_X-g-yPB5loe&uA%F=Q4ka6{%PhfrD>jiTs`B0{LIbf%18Sau;i zwF0Yj`dF#B;AZr?CDYU{Jxx-q{YWsVDcUlL4)}f_$ZpYVD-{>B$$Y7<>v$k z_$_+yMB=yUYn+bJa%rXMByWx!aPa{sWmPz8%t#Bt-NGq?yE_5IIoIpPp$gva;1ukxsiE^i`2-6YUI>?k#;(xqxW;a_{0pX+Qx3UP$8Vj=m-EWm zUrS@R?JI)_V!x%g1bIWXM+K#Gcz2$9iU~FDU;ESn@dv?c8|j3f*&O6fYhu-x z6K{!W)SPTWUwAHY5;u?iW;(qus%r?d{HXAOw6}OM_#!ijLNxn(1v;}fLMHQ81vTs2 zB29qQ)s#6!Gi4u*g&Gn|$j`TX+=e8#+O1vf${NJhtfzR+qw zP^*C+7W-jGRwiHisD%Dm(#>s^czP-!A(R}7Ea5@9_?@4>kVBuZI+!5GuEl&wYC%u= z6Ubv&=WyCg%lq|E5AQ=2yvEZ}T2_WoyLAd7Qsb{;+TYLi-EdXR+OuK0EyeSxlXmtW_T!9m;yS^R(#B7Lg+wyl;3b%o86yhj(VT& zysl^(Q&K&Iz<@|?mDm#XZB{`^tw{1z)=IzN93gsaLx*4HH3Ydj7&Z$?qxqkp)B4RutpY#~c(6Ws47dpJHd>Ht-k0(56|hXu=#F579o)!?D1 zjK4=Nc0FNKwDly~#11`!T+__K<4)|GbZ&S0JEgcum(bsUaTIjm0?%AAkVE<}=wMK8 z?vE&d&={|YN`GEq5*%bmOgVQ;Wq9Ec8B|`Sn10br=+B=PLqjU0O_E+PvMV>!i&P4* zJSgAXiVjWz+1Cl@>R}!8=nqVjsw~bJf?}08JDFO@XGQTCG zMScSfmvLd_4~d=qUXzgWu*@ve6_O075|Ow`EG!5)tQz6BHD29{4nI*u_4|`PG-e2u znJUa<#13~O9#&hvD!nitAMRXlBIxCvhh79DLJ}i5=%p}(J8pl_YwGFiY3OWbZNrLfD-z!F0{Tqn?*TSX* zotv_{BUR16RYN~vQ-WFO){@3ciAQm4|{~$4);rL$y{l6CIVqj3*GF^#mp`!HfEiPEl&Xhajp{(+M zT?!2Bf)q53hwFf69?BT__m(|sXlL2(6wuT7*QLO~l}5raP1{a_igEt4WmF#88Q+Nn z&%ZAP8ol}eP0jf~>~cTR9yZ!PH~9nY54=x5@jm@I`XpjXgCYFSeiSqNFE1~PHPcgn znVKpqC{yJ?=ei*OeN6OE13BVChG#M1Kc~V4+hY{B#|P7a0Gm$ZhZ>Iw|6fyqhfYO3 z0W3%TpDhM&p)JcW1Zw|w4u3aX<8SH`&T5R*e;oxkw8iKD5A%PqY2Ar%R!e9s`)A&6 zxkO%AZyCD)4o89t=5ZjQiVh>QJ^3M=lBd)~hwl2>QT8U}=^BR%64JbA%hTVt&h~S3 zbknDQaFRf;0=!+VthTN$F75nvHfFC;7wVOajqM>V1nZw!NVl?m$G{-s$#@pDC2Q6N zIH{$3oHu3qC{ZwuF8bo18rra2-wb2yGn6D}Z3`{UyB+uwYPsF~NJtS59sPN4)-Uw_ zGWatQRJJ!&CGh`BRyrZ9fsLb2fW7tW_g}XLm-g9Eaw*yQl|v2dEIi9XO%|a$m16l; zO_?LBfG-s_V>B|ewtb1a5bt6 z%CJY~-Jxda&WkPOkH&0f?oP9)kP4v7&S?nA0vx(6H-`6Hp?^xG7zKvj#y%rKhgYTe zPv4gT8z%Wz6>E+Wjq4(70A4#dEh@Jjzt+&{9tIQ65hX~KA?e4|Wb32<*C)uPR(PQP zCYgOFtU_qFgRy6*+0I1Dg+d%1aWp(lO4c31X^G_F9pR!x1AR)ERnG;6_xGQgYyCz| zdQ7b)*>*oA$At1HZ5(fnux}Q*@{p2ZGg$T&R78B)2?ydiKz|ja{#UA=zbeHWMgAOTq_Ay}VFuOitMzkBz#6bI z=$k-V>Tnd|^60J77hf=Cgg@HRrYTG;Id z?7brs?sah2wtE|wEAspJ#L}E}-OpMj`oOmn|G~GrK{2r4*Po9F!lR|kaN9b6b%c2n zLR8RSA-}_lu;74HVbnph%n-Mqm)?p`TFyJ9ca^kK!D5LtxH@d4fVR11S%HJE~ zmqOE5oeb||z?eQ+$|nD+IdjHdii&%Pzazo*R~cfvUcSD3KJzp=iW&t0T^o)IhQnHI1yeZAD%{u8AsE*_eJ z0J7J{R9qt=y9Cv-5+sQcg=SB#dt^H<98zpd?>^1+W@ZWlVs8#9fZBX;O;oeGTd4*1x!2@PRpZB+8!L3d1ZjT z9mT;p(LI{%bP`p5czA{sL|VHMho`{HiuigYUw2@e<5nXo8_s;xQe~U0r|c^R6wiQ$a;C;j zhc$HNhy(p0HV&J;yglzwk(prplo_Wx!2;c<0l}>5@)~%zr zkHy#$Xy&6^(yFAr;#fx3qLfrwl^h(>a}iWc(D~CPV~?(-a(>+JgKc_xs^xoh2P8RW zMwWlP>OtB`{ds>*^40?3(!-&-A`EZW^cGgxI4%rBwaVYI8io_0L}!HqUn4q{AGEB~ zm9=bK&gxm=?l{}Dc{PS)uaxx9ZRHhY(VwUis1qATM=d18Reo%p7N&YH4mM-^JLVU8 zeVFx1J<@uBY*nOI4uRpphwKd#9l94e3q9PMzNqpfh9zW-p1RiBUhgcX60*r0StEN4 zyBHh0i!SvmY&`?pJ1SROpMl}J-7>+U*tzUMF1>vX6}TA7eiNjQHMLdxHK(Bm?q1Rg zxAC}#Vv*`hib>B=p#Kn>eDLruyW3=pPk{N-K3M zx3Q@fW!^?K;_CHGfa!ez@T-o;;>62rHGF1;ONY(ccfMYoJq|#_Snel|7Av)Npk@o3 zl84~d>WYX)q;a&P?RSGCg6jP3sx>r#raiqk=3P2tno(ahU z@qE%~hPk%-k__RaaKST1g=mGU{a*zXpIWZ`%1IPV>`XaCLPKg&9y3&JtO6|+*!gEG z7C_7V^LL^f>v-Ufg`toT$wsmhn$j$2#!W9DuOMezIT1ljT__YdU*E4<+1}b%As?&R z@>@m?eq&Zp0t$s$zboD%@_FUq<()XHq9!lS6?Jjx?&|I46XoUYt8Z_luVgkV6IAdN zS^w5}nAX83ykewpq^D=JUYO?btx;ig#kLm~H!hdc;{3|7nu&#pr3S=)@O`i))JJ*d zWZ{Kb_Y7n>d@|F>fwsFfxG&Lkb92MUa2GvvVfiF1`r7w<_VB@4s=Bd-HArMb*DyWu z3#FR^5ySHk>Roy6XDd_EtE=co2E30-;2ydud1@&dE-xBR z;(XomHT<{>d72oIYrXWM0fXw4y`WvgVvT;O3>hhKQK-IRF0F$5o!{>9o5x?}mlp~j zQNge&OEe6Ej6_}66C6MmzHBXtcC>wGGr?BLJ4;g+A47$#U*n^L7S;DCJ&%|-U%%O{ zmb6d4v*UtaxSabon;GnBs|iYRm#ygtJ)t^SY~A}UW`~WJR-&cWG4s$|kYY+m_Q21A zqaKhQNpOt+vA^vKfl0B2IQGR@C*52&=3#B%*Z<}MApU*EO@iUV>e+0+`-)#}un8={ zjrClBTMm3Yqwv;~%&gkj#~cjeL(2%|i1)}Br<*>Dv?T~EWG*D5PPx-4F)2#oDT$J+ z;5k?4_*yUF%=?8yFln}7_gW8ZwzhKpL3QQ1)H3XwEtALOzQ)|!tmY}l@4n(c7sXWV ztqh>po~{&fh2rwIwzcu_F4_Lc(ZxFOi)k6lbv;_JK1%$ck}1VtL^UwD&wJ*C%w*S^ z1y1nF8p!NUO`K*PnWGafdqU*5-otnUcYNtv`~Wicj14HX^yN zUHAU8>_r2Qhc06;AT|5Saq^EmC|Smh+ZH!T3wR!&qDL&FLYMN)+#m$1woMucFP}@R z<}Q32OF(9t3Gey+OV-@XdyjfHq#(PmCvGqijuz5+@Fv!#QM9ED-%SE!rtmiVg*LS| zhiGw2MI7>%>D5N3uS*TI7YTls;+6)IAHZAoeGR2;t~#u#h&^5T)jsP&Kk(FizV7$C zKGkiSte&l|h8Gs)W=)lf@*k!hG$)x!vB6yY46&``F2ZX0{0MvKGy6GDs96wp-4!iRE|-A zS#w3Gk1dk`y(ge78v{(x)MSx{KbGS)aa1M|4)g^8wC{5bHL@@YcI$qepyh+UnB^EttK>pNt6 zF&$wQB8*VKY17GX8H9;34H1BNsn&jV&(89OFA50vBjkbvx)TXiRk`QgLxZ*GiGBSw zz>VJlkB(JpPVBS#Adfs8fHO4fL?s0vPxF`)w$~8aD z@{=f8N+cUgl=txz`{<4Q@SQ_Y?@1dr zLeK9D3T3lOLCLC~`!lPW!jnnMjE8pX>+a5*!Y}yS{X@R&wK>A0eC(?(M_~s%Ws|76 z6-tbou-W6m&{f^4`!Wej9-!S$58Zd7Q3+ZFf=FXl ze5S1}C&bJdsjx{N5E_HBrW9~Qex-9Z3=Vn7=-1bC-i=PINp4~EN3)w8=S4yG?i_(F zK2Q3zs|QlJHjHFZFsVYFH}{!Y3YZsy$F79 z7`iu%@$zmbd16yjnU?bztrcTC3aZ$bL@4~>#wP_x>)E5^Ky&gNV>q9P_7u$_ohq=g{tUPj3>7PcP~24EG|5le>B+#7(YoNUD$cjza=V;%y`yo^(pp z_2AO@J{ebxqpp`*FCGdFbrFyEec%3Nr7|$0J+Em>m&WVv`t+u-=v(YIA-uiiRv4+G zJUmN&dBFr~mk+m4{?}5TFg(m5d-yLVb=uqPO#L%c^X5!$_0ZswCg4TH;TM=G2D2`- z5G^V*c;TPTW%9}5X_aH|Otf!JDnUwWqU(*<`}}fA@VjR>D09qJaaOe0F0QKXi$ych zGD*Ejzq2n2nyNeZhksPtZuH;(Tr%R&=>g^-Z9yw=puN6q4|M8EAk7DRe9HPCWB{3J zmE2#xC24Cc%q}j^Eoqw;KP`_{Nr77|}E}=I%H4j+a^% zy~-*m_O|gNPmyfE;->Gy8!>V!lf_SwrZf?%BOpJ{t~H7IHs$0%{HyM#q~_(8G>Vx! zFbAJ}nOc@QWjgS+C-8tvX)r&!WX^iil2Wl@Zu&mpm)R_`oRhLUg#-m`^;?Cj9>to1 zkZlqARB;K}_^C?$DSz<;Yikv_@nbF^)j^WxLX*craZx0?J_cnaFnSYFyK4ypNl4+& z`@&2+*MJv<@-bIE-eHb<3CRr+O z49MelMD%sdtpu4%>JtMkc`g^#N(w5i^BFwrMHc&6d!H2N8q_6PGrdB}I2+~fVm`6< zwg^nEND}#!c$BZx@jWF?nVY+B^)D^~jXIo+erOBeGl$n0~vopZLrcY?d4o#-^PzGemb6(85nb zl$3YEYSwQ<6}qRRig7O`zLfRb3tpv&GgmQYP8iDdBkpm^^T4DY>7No6rIjADQa(i; z5KKznvWGpPm!||z0Sn(YrESefn5lLv;8r1mq~l>t)p(EmvRX#%7KI9hj}oE+?s;3j zw($%sOFPgtH7?jww4cznx&7oOr&>6wK~N>Jb>BRes-abN@!^m~L^oQg%IpL^ZKuHo zO(7U7MQZj0HlQ<;J`sj0i3@iG_ULLuyk`$>FKB-;yGy^`Xk!6}3C+=@LOO`SZt8uim7c5;8VcGa9s|+0h;IB$ZUv7aWi#jgYh=bqNgKDZ zLNpuRl75sMh%oUVFb5E8p7$_<$BrEn41$w}xS77!$)t=e=iJvh#U7(P@9pZyR>MgI zXbCf$>fe>_M$jlnYgf0^8RwdO{rr3^zwkLK^Kp!AChzZmUQNHRF)gIS^FgLw27y0OGcn;*9>1$}jmW(VUa7l!wrUBBbikeS`r`CI00wWupG)IT> zXDJ`I%Y|5wao$wrr}TDwa%0?diznuxZ>zBV@#Jc1ESi#PB`<6Zm!X6Ic6bVw`egqm z9>^euipRi_*Qq2T`W+2JZ*HLk zj!lUTE;-*@v&$R?{?9Fl*~hM?PF}L_)A7C;IBNt^hH-ESdxrDWsKOekWc4-hsGUzK zeJ}aWSk?gtfuUY$HnBT8u%v9_9uxyv``s*GYo|V*x#eOmt!TdYVf!hYq}W1bb>-|s zJM~1eZGf}C9@_Jr!TZcV1mj}t*A-J|Ng6N1+;4|LolmhF=5bpzm4YP(u4LUv< z5(SZY1Nq?M1)J$2kP%^t4JEONUfcfO*2eY5nPUZ|bshmMfwjK{15S9$)9=0&y@an( zWC_BFTKFvvNl!z~pAt7K@&=6aR` zeer|vQNolqnfrkrS8{t-2`M*v$}-1_`>G_6?ocw{{5(3Ty1tU+>7&fWkQ(p%$s>=h zpz?ilVgmoJS?p6zO&Rul=IVdGT7`X+#q}zY`p;zQEJaijR=jz$l6~;QLoe^ zvNV(vBdB$?IXk<;bgvKh9@vLy$wNN#P&g@D2p!ab&9Gr!W~`Dv-2!)zEqFw)s~;j0 z*%XH5ve3Z=%6=JRuj<<`#z|4&DpsL{SOF1{1o#Q)b>5(l|1DUOO6ZZ}ZA*igEui}b zY&w!Mr;G9ey}xsT{Et2bm+6e#iUk)h7O)BP1j5z-iVZx8aFR8CEAEcjQXI!K`swN9VZDQF+6J^fiLN> zqkS$SXF3ky$(MO76G#dEZ?Hb8^?J&!wu^J!#MCxE?KQ~XjV5U#=2@HyfHlIQVmc%c zvwJ(r{nFV`&w`*0nh&D&>!bllk`%+{I=M{Pd`CU4Qv9QF0vW{O=u{nVxb+eP1{LF` zYIR$gWmDq!C2!%YO@EMyO=Q6LFB55J99D~Lt6e6UtkFs2Xw(v#2-C-m2&Zy4UrZjG z=YFpa5B+M(n-*2)k$F{jv-zPa=W{L`<>9}b=+5l6>8Sm1iVP_#h#yI_v&rK< zU!#SKI2gg7qCnvCJ2quJo9HiDmL3Z7@o?nvF#K^hJQqJhjPSpN)7kgvK~u?SXDA-p|_opK38bKgla_eaF}T`2YW{@A!w}Zu<)Co_N-wsH9exTRc|omalcz zAPLR?9!B`90k3+$*!U6*PQgBx@vLt|6?>+bUp@vSMgD(R8S}qdZqWz=a0(4Loij~~ z2yi)B_PW>|lU6}%T;haC^S<4~dtAfow`hB~T0p|%ZFUm>+QF9GPL}2wIkIC+LKN}) zy7jsRyOe&i$q5oGtmnGz$xHpnbW)J@qKJPH?Y&Zu0B%O zT^}#O7W(AtibIsOJ#wPagH1+{=W)7y%!Kk$B&(P1HH=p^WnR{m{rFd^k?k0*r%)xK z5A$UY^Hd3<#1@s1yLb696?*i$q6u6W7#KDk1PK4=%~vq9DDBkh9=xvqu*u-B#=fGK z0`Hwb{6~aqPX)j7zdwdbL6FsTXE?>I)CTR<^-E*WD@X)LT2uY$aqr63`;TDf=Zmlk z{*#};jsqU$=4HSIzmlE?w&-{hJ{OVvn=ktJZ#xd~=%ed>XZx&q|M|E;secP!wSeCL zTub)Cqs689(T+M|SQR_PTe^{^HD0`{OAeJhwO<)qTeb@Gdbt5*7%pHGwK`h5TO`P( ztG&V~SIyFSj(*4>R3*6Q{GDu_`cU5b8h%iWU#!2g?xOuU3cMlrd55AX|Ta&Ts5AlgOYG$r=WN%8HreUWiqXhQ?-_W^PPqA}1*q8u^zr+b}_v}48D*=Cz$pJTLG6fL8pBqryixY4|x`a%@c zf5)IGVAAY6`HWD=duY^1F>3R;89U&>zVXo%UIqWGulu5UAJxocj_uNne=1ytgLpbc zdcr>dZ7112Zi62sqaP)f3BjJIlh1s&5yhuFd`}=ba>tP9z`{D;_B?Q0>?2G2U3lx+ zs{1gd+k$2x)Tb$&KCyQXtRwMndjYskDlBw+Htfp%WR`f`?;rB& z!2_IBVpaBWQiz!S^un9N9UP!7=0R*rn~RFb8=rf!U|+uBq+r7m_!= z<=K%f;Ernr^dn1jLo0CZ{SD`w*_d$U$`P_uWX8@Oi<_IcQPna&*Eoi3&qrI5k3Y_& zOnQF$#fkp2f99ZGXfpqqlkkE!c%$N zF9ww|kCUVEt{1`{=nCYkG+_($yF3CY8XxF3z+0vN3@2Y-N$Mq|lg42yDaba)2pKo5`?3n=>JOPJd7v-9kf8XyjB9!l zu%Qk(ChS<~WQnzj)*kgb068rcaR=i$mgse^l7O>l$w>0~;&G+f2rY}rl4<_QbJUfH zE(n5`%}kF5*<-HnZhnBv2%~#GCCWEo6IBK)?KVtx1cjQX?apoLlwpoAldqH%5c5-D zp85L~(0tOO#5)M5QQh|4k#lHwSH8!)U*$r`A1gddd_>|OexCgU_a4H8PIO#ADmF0V z++Soy5A`MW^e2w`#Lp@1x7hYDvCLsmq!0+^n&rruoA=lHYt@cl1@lTi1e@54r{z8G zHk|%wS-C)Ms&LZcLKm@d=M~``lxO;AlnCkmqYxBHTfRVU9O4wtU;!=gTyo>z0sgNi zin6Pe2lOKrb7qrZ_d8o`nKG09m^YQON>Tyz2%a}n{|L$YT6*4s1x!as6fTB(Vf4iM zp;}`{f*;zQbcGp7!uMM)N7|s#pg6dK^}z5S@mg>u{ZsB#muFSs$Np3fee6S> zXe86(nYJc53KEHXhH(SGq$=dzAM<%U$`fjY^=1jkHp!RO`OX;xv^|{-5WG`qia$ko zxk~DMfdCg@<1zhAC(Wk0b)N_rOCyI1Gs;mbgmG%UE#6s!_+@555jV7>5bCrahT-pC z{gZBL|FSp69siT;wxz#TWenAYDL=h>DO+TYK?CKaWq9f(yDt7GY1*ACkd=bgHOB?o z(AlM4hFl?R*lkoU8){gdM2x|-ZqSZ+Z1c#`sNfmI%dsnk=QE%lUtqD5xsM*pmi#*w zMLlozQ7yO2R3bocDWF~{3V4-bu@{zm9usZ$q^`53)sn>Ikyz|VOX-EwJmy%DNPD$Z z6RAgvcKvb`zk=iz(u@ZH*#HyUzPA1U@&X)$xg#Z~JsgB1B6>TngTlH1pfI)sEsh($ zM_K#G)~%Dr-Ll91UX+275x^s`?O~yfr(*X#!^;a_`~U$yd)w1p+f!89y^&VI-O3Q; zQ*YJN5x~pjYBlj#gfPOS$~I}iG8k*@<%DggA*8SN-vfd~PdYDLX&-@nybs||_fQcY zN0)3mfTQ7VI^H{w>G(*Kt=C#flM89kg{1E7IC0kJ#TVlb-%b(+!CD_2mmfFcBd5)k zg1g~#ZJEP|&7M+6pHM@{sz9Ec+jxrKkio^P{~Wsb^b*fsAuDu|Jf!xPLWSw!?kJRw zR}5@+u73x)>~YefdoP$gdfJ1{mfzC&ZZkcL$%k#l6X@c-yWR4SbBz&*BZgI^I9rI>c&pi_1ou$8?NCHwd=#0CJp zrM!B27?58LNz1a%x5cX10D$6jAJ;K5;_o~`0PwlWZamq3Vp5>rpU2s*KY~8RwE$MO zQfI-wSd4Cpcc4zZ$DfTis^a&3eSLHf<8(_c9@qAQ-d9y=DQRgF0Vi=D+TI5)fNK}P zL7n#j<%iSbY?%AgK)~GwfY%*3^W>lar`EmjP}gL8131Ggd+MTW&py6>JQ|vLNb|aN z_jrE!4hR5j0AL>5R&IKNl)s9Dx8-H(g8KK~DIM35Am9e<7D8dJ+nlzveOPx`xdBNl*1EV}hv_~Zg1e_``knIO zgzR!U?$yb|^2$ne@qJNS@S;@r;{l*0Blw~2$yo^P?x3v|{CrEd)q!bXR$3)=Z~z<- zR+`L*y17p7w-(pdY#%mz!e|su@PgkdYw%lRI8Raz<#q7c-P@nJ~7imHXGl$95~gi=wTk7H9|>E=pz+vK1S)?|;8>0V(W> zjG**K5A5YPhOS@_DM>{Ct}^Xb@Y<9Y#Q!t2wxgoN|MZ_bs^&teZKY;hN|PcplT6U5 zTGm6$&IxUBBt*Y@&X{xn^jNZ6lumwWRhrUnCyQsfmQQ>hCwSO>PD>yiiP$F5$|as! zl3ElYYEa)m?DX3K6&*E==5INsMtP7=zs@;dS$-l`tO6Km@cUA1rR~8QWxvLbPsN|z z`$bF7(hi|$_$C2&l|!-WN$MSfaj-Lbp_`g!cQbcIP?wzIm+ahPQ?wF1MQh1@m%>}N zxgk91$~kFo$#1_~olY8dypIzWoc5PJY*x8AIAS(?K2**Pu6dY{qW+ZnmO%J=uUT<_ zv_l-|D9hMie7LE^pghnYVln)KF*c={G)D}=yd7SmhthhG-VTnHCOk!nxqwMROK{rg zJS*-9LTeD$`RnEy!RfqZnb%Fx-7+Yx!bKyladS@+8wrUUh(7bVT)@PkPD%X&)@AOjp4$Zp@BXM6 z;X{9??z-I&GQ-11LAejB{2tWak!ewrJm@ZE-DYNCF>Oe(7cWc`hDxVdh@Z+0 zSfU|=ROk637@<(?3Oomne6N(ZVktqV;U-6C7O!HbNEt}uq9J=|ni-*#=&2o#w7B(+ z9olkaL8D&eetO5TVYO)(+l*|Z_48ZJ4ZB2Epi}?o8gbcBzz9A)BqX>EbS~Kp@tU2U zpl#o|He>nr2>J7no~{L6)6nt?J^>C&0v|zXd{w1H1)Z@R#W_rrwn0>$DvSHF{48jSGUQ9z&I{jkS;NQxn<)TZ zsC2JEghS>`h8mgb?Ue_$(t-glu?f)w&TliNbB5%;M%XmQMTe`7uSrayjht~>=XMm$ z)fPr(ev)BRVOJlSTpY;vmPek19!5Z%*e2+niXf1JeK#z-Vl538oRrEU8EJE zwhndvwq}zQi)7EXVQ~>!3g!pp$kACJ4V3cbKFy}Wnr-+V1%uw#vDNW$s9FU6?zMHL zYF?MeiB^}B4?jdE72OtJ@Q9cK5Ps+M1F3Z;rDZrO-A>K?J02xhi}0mIs%bb)^U9|; z7v>ZFFh|$q!?LWVYQy0A>Vbl!pq}%2R#?;>vLuq%E^!`ib0ve9Vd+EPIAapC=9D*Z zgD!oSh~}piEj9T(VE7sTjDfFUtU}t;xut@Wt1@&1Fg2Wi;*w&gC2udpg^Rv7fD#{5 z;+U!^o~&aLZ)j8sk{(QE&k{Z}%F3;GyW}2nDQ&60JyLOBH6h-+7bIcj<#7>>@u$bM za;-DV9a+v+EAwy~JD;^nWZLB%QrE=8-fD+>i4B{C^yF|&2t{lbJY`)dKQ80$qQwDA zDQq(&Mt-BlQd*ryA7SAvP&Gifq?TOM&je8?CG5)eH~4DG&1db`|7fKJmXZ=f@j5%@ zC}j#sPg5@%Z_!6|gDdV%VG+DYOjhMkFqN|LWF{y3bg~T<&7}{W9MUGdMkvO61}UUZ zLYF&(yQO8aim7l%;q=ah$WU84TNa5WMoOzWNnkd|RJdofsY*+gXgcSUZvK&;J-o+^ z4ix_-Th)_LU;~}on^zUETya#TD+c9l|;}gYpJWWc^rt^p$3*%56Qn&F@ z6tkMOu|YL1zaIs4X`0cxEi2gmSGl>+Zgeps&2@_)M8k_E&_hg550^A0oM7wI&iJBI za|y>;0ZQDbrX$oJJXRKfO!EgDW+bz@oZMF{OiDx2!=w5{)r*eS^m8nib`%66;v~IS ziNi*6)cJ*dL2Jb}{J)Iv6s>^@$=L@?47HvtB?&|GUTk?LTc5tfoX7G;Su z-M-sKb+M35>N1UwBqi7~wsZ_RdsQ37_SeP4l!>Kz6w^p_*^c z!Ap{eJ}km`Hvn`Sa!g5%j%MsB*!!a5EZ$Dal<`m(Fhf1kVDIkC`5yIT*n^~JWH6;+ ztswy6sS;2~4*Tce9t~?xH|TRN@<>oyC>=ue z+J>aDwFEg`yLQ{f#A@;C7MG4hj~=F%#0DVuk*m%WRF?*j6A)Oq6-iz$IS~D0hvqGD zIg&%XmqoC_i+%$C*(v=BciZk7sT#bM{EuMcGe%$squTdFW(OfG7q6AgxvZW@5p-#J z4LDUvD`Vo-dIx_j?6cCWih~s^XS4vRP|yeP20lG(KFM?8m@Rb6zk+1NKMj(-zqNh3 znlDgXx%cB!yY1w=&wFRTjt+D2d3EfwS7n8**TD_oR`v$8+q5S{Rvc+HYRIkNY{M286|5Y((z+4phW2_cb1Hvfq}u7FAQq8 zt}E|8HrXD|Jl*oIbQewSdtFzgkn=0^-Jf=PQGNA&@%1%{Um-b!q$Aqa@&%Q4E_hdi zOV|A;;jT9Td8<`lZw&ek^Jl+-lUF-6nD$O?YMwCXmq}u~4$9ZShM8{!cz>Sl6(l13 zYuEU|oV+oaKb!hB@~kDqDgAXK{pLS51nU!gRjULab7!*y7OQv`YVT-q(eA%jJgyjz znv)nB8gj|&?R@Cz5&XzCrXOg0(sYs)&byyRF!i|Vx7gsY^TlESkqe^xKY4dnQaczP zDM@M1joW=r>WUlJNZFvu+=Ik;Ef(>2fPexbb7eH=L9xtts^HPJcJ>_AL9 zZ8`DU@Od9c%i8U;t#1Z*4oy&&N&xPepAv-E^z^S4&Xj|JlUlXb^#;oyCSh zvQlS3til2f)z$QE&GdjJ`e8L^8H=_CwCK*Hyu;0LjjL_@99=X#_gywwUXV#!PN)5* z&J~Tm_aj5(sLK~hkEe&jwZN5ypl>*=YrmiYz@(MXBVnJrPF>P2~9=j=1MF%PakM@*noy1+Y4~O`ynC-0Xa$~r6fB@dET{}$kc|Q@JH~UqHF82l9(|N+1X=Y|qOc*6fFt#I- z{@r87(MO*TiGzsgitbh( z(iWTOy5H(@S#ZxnE_+>XRKICV3vWBseL4WJ&xVN{T=}>6_mlnZ$+xIvn7H3> z-Tf+o#Oeb89h$b?)r|JEK24Q9F2%JvmScxZAHn;iJx<$BSs&i5Jgp%C>d|nn`Z^Kr zE8CuyRoVDq)nnE}m4i!XO7-R#9F;%ZK=y`x(IQOvfVSkwwasDfLd3?}Q=sx6>4?A` zK2ye5Bf>#cSh1ZHg+Mr|8kd6Vi=Wi-W_;^zGALvq8pK1+pQ;-7Du(~XM9w-KW3 zhp)ua{OYQzHBSiM{W;V0bYT%@J1?{qRlgCtZ0_#-z+H>XOr&sqY=ux$<%kpers#D9 z3>OhXPjx$9cN&om4t5t*7Un&*(cuPF=WEcAMcuc`#p^P&upCrGJ4$87ik=U|Qhp~J z(RCSs6;P2)mdi$^T5i6-XUeof)x?^FgzWYuHAg4QaR52wZDj+lZ`DQmW>qv4q;*Ft z8RLYW2BHZ3UXI4vdW{^6DWE)j#gd}EZ(Nv*?rSD)`bIVUv zW`Vk;WK3h}JY(9+!L1zMy-nf<`9@mEwilPluLpoZutBUc%AK%2Hm+*;PmmtmfE8zy zQA$X*>#xaHst#%#!60#5Qr?%YursLNruien+2s z?zWETWk{u1fp^7hqVN%_pX`g!jZbd|@g}MBBJ}>~GEi&jnR-<;8&IbU{zw^+B@IW~AVRE(vTHPCm=)!Rr~~#^oLBjds)Fi?0!a9^R1|{)S#WH@VtiE}Pj1TdA1l;0{M>LscW|p^u4_tkQk?d=B z_NPpHOP#lPdA;VV8F6HAyFq>12m|VBbN6C>#4Bn#3#2LQ_eNF3tH<_qbWJf0t?#Q7 z-Hr46a zmnj(deyYmPABRF=zp(AmsMvRU#a~vR>>}+3uk0?GTdt%Zm1i}xJFC-&Ru8ml32cxt znj752ptLe5T}6++DlaX~DjNue{Kt3DhVTo%=nP8aoP$@iqAZ!(sJQB{wB2jnK{Yqr zj?Q%vJ<;!8N0-SWu6d*JTFQI(Tc%5W-1qS+TkYWB|6IDjqwy;EIU9lVPGlLDlE0a7 zQuXAt+{IbIoiU#4{dU_Bf9YQ$EdoJiyI3NvXVZMoM*V&}KP`67BFGd|e{_DA^zZGo z{{eQ`VIbdb>nlhpQS7kCZRf6AFRm++(Q-I1P<#^Vjh;Rk@1G7f1;b^VR@3~%q~wfF zQMHC@*7(Cj(*F%y4lJFLgZVQey@I^Or_c9_XPNq7(70;u7LFJ}4XqW4xQg5esdJR| z(L+sm{1RMA(q+d<-Vv4IeRN1E&j~!np&(;YVq1fSaddT7{#^B>P*8O42sqbn_E7nt zEwaw>VU?Xkr;fS-pS!Ah-3VNK~}<0jC**uTd(0@ZM@ zMPZ1BGa%DHcX1ne8J(?Cu1+ch0S3bq0VVhrCIm&f>^<0d&DDDiu_-^Vi1%e(fA2u} zf3#>*iNipTwS=wcB1I|ZS@?$ij|)g`=XH%gh{3c9RyZy4w^kNcEDK0sosL~oz#95p zt6)4)mT1W&lExu0u%O|+#G}KYl|vFYf>Ct9M8eEvR+K{^%-BEWL|IZokkj-lwZ7Dn zs>2j{)&pXSvZ?N(eR-y83ab#W0_m~s?t~`Hx~4t-T$%5aN;j`TNJ#koDzq*ZYS7R@ zfs%%%2v}_J$rul-ep>ab&?izcy!{0&G!r9KG`nwcKwMk5>~1$}6km})rg zlEJV^t-i71pbx+>`PY>4s`{K(fUgE}g zB?fKWw*z3Z`XWX(;QK5XlwBl@G>g1n%BKqd#BttF)mhWJ4c33QJWd{L%bL$ef5xjQ zzWd<;Y`Mn86YMsqVDr;Ek5Sc-S!&I?TlGWY1t#dWK0V%5oZgnG>VQ_RwR4=Ie&EtT z|1uO3DbB^ExVqVBb6fRM4S>U-P^884df}!5UTh`=T?@MSuT6I3KT)wz?wqr~hu6TVhluULX=#LkD`76QyAy2Qp|kdk^&aSf2V zpkwfHF;v{H`nAmw>xM6YZ9uRZ#ur^EwwLv7Zs#N3&KHVlaJb$#%iHQ6o3r{T5;?NU zMTX%#wev#v+kH6->luNRUTkY%D5R$IqwTZ<9*iJ2w_^yP|A_JzlCK=%A0oM!w^~6% zwyo#=C|u??HMejhKqTH9J%Avt*F_84vg^iKFWyNRdrWYzZeGGeAf9)Tclne6v`X6tl z{{V@Y5olfzWUHmCJ;U{9{_zdPH`01D)OQ12(!2dbA8HL_N?*R9fn=CFcq)JezaR|V z`>R-cmjTE7o4Q64q8G?-HT^yNH(Zkz_ze_A&!!NxXh}(U-~c>u=G{i9@lJLLNVn4l z*_exlDh`7(RNfB?d{bEwi}j9X<#MUK^z+VAzKb`1+d_7;!oPR+@?J5Bfb&>L=ZJQ& zs`YHH$t67n_Z{HM6i^$|K>!E&y?uXw9|>^Ni4_d)bXOCTw`%-)F0j!9TY#HDKz^U+ z#n|U(?aRewO9%_5`F6i(=rS}aD(cy+#%649#xq!N;F`PG*Gs3-;COp~zXU8Kb&ch` zWlAt+Dv9*?n8ay}er^#AzknSLXRhAV=*Uw-H7;PbnJfAOAxnC-Ckz|U`-Y%q%?BFy zYaqwG7l-G3lsOG8?es-v-qV^QGFBM(>vQ@&7WI;k7e?UOw&~$`gT;?)GSQRjeQWL- z-#H^fRQBF2UK;xjtc$VhuttH^Pq-G%^5*E29o#J7sF4V zJwA$vur1kp!29}kgoS-6M7xoZks)cQ_-4iGaFdz<3q_2gpDV`Cc@oWu%x<1~EH0W5 zlafrCYoYEYKQt%bQyDdrjVg}+&Q+jKEk_!Y^qb=`IOkPY-W49o=~4b z8-;ywZQo!JTw{t)66ds9_2`O6d)jAjsx$XAkvNzzi?s*^Lc~=UcJM+1sCisG{GAzP zLV}|=eeK7bzzKreM5S##R8DJrEI3CH0_zB%;yC0S!%qzf5#z>^Xj2CRF|#+(#DNg^ zkvogc_uR1co3ndLEygUb1uD|TL7Dlv4jOXR(!lM@Dv7bWg;3;wc>(GwTT`7C3uBcV zA~osDmbst84D5ObBx^Vu!M89@q;+aqi|bz-9yGGQz<_YE#2tSD%&!95@p%Rn1Y(%a ztFUNF{#X9+TWDkdCMQxBi!{>dFF^xesfaoA)oz{ECGjgKTQ%%#)&=F>&gNJ zNNg%e4gtEw#xFztGhVG?(zH1_BpPNK&PxV)%L7vV#$0d4a>q@`+r`+HBq=-ol}_Y+ zg)1{zD3l1U^jj0h5ielAKY(nZ_#AbZ?{-H z2sSqFoMb5N#WVKGBER(@PA0zEzO!sTsKeXJbyL@+`Fh~u3`8&MOUz1e!0tI9u7 zrDGJ2_6A=H0Eo`OeYeCDEsrmk3ndrrLt^7@oH6#E?#H2&@cQ(#r#53!X)O(vwfD=O zh~$qyBU@X$NPlRT!!?eMb+i1genfimRo8){MRnzupY6VL(e15}J9 zVJYz~o9uRUclh54Uv!@)2|YPdly90z7?UFEqvH}<+Ax-TZ*8xW`V~WlH?4`6zNKPB zdmpxoB}%*{;e+EC%b+|TO_kFOQz*R|%|f#)Mv$h=Iga+wVA&76CL;_^!8sds2*#Jw zsa*vYVX!kBg)}F5Ln|>UxNZ+Ujq^)i#e&S*Y~k{ofzJmnQELP-;M*BZ?|B3iHuQbu zwE7k!Y3bKjDjy~ZOCdqYQ9I*E>SrLJEG>gz>huPVwn?=lBZ_?@E{+n#;Vsk;oL$

Nn`wi85R!re zY0Zw;nyg|)_pf+{9^9DFUe&@BC|-0`!mF+ysm>-5{z%Au`0>M znQoPdeOAV*X5pNTWOt5mgCCa8mx_svnl{(ElC6KgS$@4Bf<*j6y8DFiS6*YXcJ{M> zv@}?IrG#Rswn2*3wqe>h9D4_f!htGpfE#1;+#x4a*^!!y!gS(5a#}G`;}xW+n$tqioJQga;*z^xWoz^0Ur_0ef~RF>PjEcK{4g$Uz>6gAyZw~VMky2 zv=Dub)Ai^@j|}3cM)wQ*_2DYIos9Pkc6K)PL%^x9qjgB5e{{YSz^XEnMb?u-XAE>zz? zf#E9%AsH}+{ON)``42H7g5axHJkf-*9O|Dd`76kPh(XkN@=5NCR&9<;<4~M3vCMLJEKYahV7%=8t)&C*Pc9a*% zck>HL;^`**p6leIq-#Bo?;-Vo_hK#ZLHwo;8t)#h_o3&@EtO1;H+ndrP{EBI<~Fvx zlk|T5cSPXRQk!4k92)=?Tp#t{S`8)|7}=zZD*Jd}~G zcvkVj#^vS#?)~XJ@2-v>gB72pnC+9NLsP(w?9*i3Gc*2lJ06GbLJz%AHxHe)3TF55 z0GKVQ=|Q_is%5yLq8VsHg3qJu;n_-^$XGqNw%#2gl!-aXP7kT$aa1Eks%Sa9xw-jl zC&!7;%^(mGYzH>CJsz}`QUEU|WS>B3PoSUVZ&?r#eExDNo;*FY%_IUZ%K#70`^#9Z z=Lr^{GlB|Qj5x7#-)s}9x@L-+v3@z{p-@U#Pv0ymtDItExKuuku&=d$H0&z~BL3CW zVf`Mm+#6v{B&e7T`BA1aOjWzs1qqj1m zMAwY03@%$+_6@t7KMrF0%XGLHJ{D20OipYIm>J=zguh$^JcL$9-Q$<+NEIZ88Mk{r zYE}UK73!!LB;cKqHB2?@>-lA5JVK+4x#2%OxGkp15DoojqhcaL_=R35QszfGdnITk z%`2-ncs$&H8L+4B{TbEy%a6NDfdZi_QEa+JXj^k`@CXf^&$+K^!c%)FJwC2`XQ%9d zL%T1ert5dN77Z3C+XQ2Q5DH30CC{Wcccei^)Xdf6zqiqpK(C4a4_v%k$(sfOi8eFi ziQ6IN`Q(HN|94gEFF@K~Mvc{NdPx(`rEMK>ZANPCX-7-5;bHQGTp?aNtw`);jDJz{ zSM5bw5|WbDqt%Cv)=@w8v;I5#jD7vaghTT%ZqcblZAoD^IlPj*f7GSaPzyqpLngIC zp)Pf7-Q;y^{B-$GNr0yPNzO+`2NxQOHGS6j4y&K0+)}^kiNU<=o3>33|K!w^vhrC+(ky1djsHWsz8b?m%moJ{`$s`HxZV$O?1%0EQE+M0 zzuEedwTx*R>dD;jjcNpT3s}io$;HG^n-D$%D6Z>4WW4vMLWP&$oEg(`DgPuLr5;R! z^|f1+aJygZ3fR)l*C9cgxhYQcfXSUPuhCb^xQ-BMH(g|jtXCkg5j&j?<7fdI;^JYp zy28j_5=H6XxI9Jn*^;AMr)@{~zEsD{`Osnpt|w16NhPXTE?J)go@$7sRXZdg zNIa2D%O&bRbO7b-zjS~&6$j=S@nr)+%&%fOOF7;RyK^S24rhmFf&?0=2xJZ+FcH>m zF!x)!k6Yf#VE;I`57Flzo_7l?yr)CLZ>}TSiNd)duopf*24?K{;sHLfI%9IW<1O68 z?9RvjOG$8t(pQ?eOq~lbc6N5~v`;*VM@4wk3>TSMU|-mIaQ>faLS*~rh_9FqFR?tx zU&89Y!SbozxvX%Ju`a?oA5zoWicM<_3^Ddnr{ULL1nodyFo{1X9K2WyNif*2ox1-e zIF!K_S8b8=c%hskNJo&6oRq}LacSgq*x_%xvx@z7qvih~pPN9a4!YsQ9n8tc=|R#Mx{hO-Pj~2c2xQJaCq%;i@u2-p&zSz3}7*Mz*J4=})hcb~*l3!XenZbzB2A0UOCVgX3ZPMP^d3{o$Pn5~PP#v}zb7=H(}?H1;!G zBqmIOWVXvZR6(VLXP;p0l0TGb#%dv_s%w0y4BqauAgzkg&fwYKjYw8{H$hPlhf$h# z9~_f_`dj^KAZJ|x^8*{N)BGS?xX~WLtG*W_2*3Ot#GB+!D>F1)q=NiH34Qgz07CRj#?a z(59rO5X?+F?6eppY?zwJ>NajPxFwjFegVg}L(by}6?m|i) z<;q#LLoh}Pm!A)xZ)Y&eGD(8*jww=y#(?i5p+C`CARY3g(|PZunHCt&+(C`V#vgAy&V7)^DK* zj^LX9134FUiV0g>M{RFz{0_sn&wQcI<)5t)<6}$%x_5=IW4?Bytg+`>P?zX^S|C~v zlr0{Cd&>krNag>Ft$|n)R!X|6n){s98HQ)D)3lY#=IjoMyZq)i5!d^wJmO3nJA)V85w_`&JVk0B1s zcbo0zIOV>>7X!sR9A?m^-Dacwu+a&m6-8qMA@enj9mmTEfKZe{){lI0)^c;NiIH8w z_u3hnW+N%0u4=jvVrA_;>@Zuvj`!zTA7xz2~gG_gZVO7e_?E`8eJ1 z7`$r#F$T*)UlsTkzuuJpl*cGnuM&6}z&zz2tDVEkL5@U`=5O$qFZ5G}}*tzPM8 zHC&cXTc1CA-nIMu{SGHJMN=No03)<%M6$!Pt2%elO3`FAkNOhL4IyIFk(R_FE7_O` zIC&VIk{#*PKGB2j6|=y4aMqN@g9GY#aW!$8yZyZLQG^XTnU`@WIGhZ?vH960o zU~a*#Gb@ce_r50{Y)G$b_+i;x8kPErdQujJCHapwNC_m0=Hi*N8I8@cJl|QkCcQ&{ zTZgg6TV6F!%jQJ1C`*xBfmT_G6(git+IjF;q^oUNxQ5!9$g}87W5-mzKV~A%cR-<& zrZQDjwpOV~mWyZIN{3-)jZi&Y!ZIv>%T6#AZ?NF6`iv*JaC%8*RuV(j@-acfw)OJc zR&F8kGt#z6Cz>10NFtSZ*K^mWP#gD4z=Y>+G*b_5*~)Hxa%6!frnP zps5AdaHRPYTZ{2K`LPA{?`74#cE|J@Jxtul*D(K4c>CF5?IOiQF4xgK{95H&B+l;9 zd*LGvZH}-m=Gc~y;(_msc=A@LvBP}&hA){6OZL`^YwEVqFfLxPld~v2ad}8yk$s2>`{6SG@S|dWabCqn zrXk|AMT4XI_r+<0rYf~MpGTE9t$v#0xwPu3{=)`{J0rmu-DNR#rxX?ZSy>eB{z#`q zZkO8f#iGq(gtpKAA`8QVdV5Xb{KYC(l}gKA!)fTy){s+~(9Q+VVuf~JTL#XD?@UJYb3W>Z+2(hQ72YwG*(uQWK^&QmKJ^=zONihLz!A+vr`zCGN1w zXwNF~>!cOJWnu`$XxMILCB}PshGR37CPt~;?mmluo_)<#tb=A>bKaJ`vQ58Iz78l~ z26j$u_{8phpy?+F6ll=$=@JHHin6n23V!}OiU1tQ$sH zax`8aaB*npG)HD}EDYC$1NU|pVDrt3ok)^C^exL5@UUV>n)b5m@u4;)F)MjAq z$SpSd0ErVxssbrtp~#^o$h09rM?eUbAwIHh-+gf z&E&{^;HsvD%#M-$LKAR<=bF9s^5o9}LX46}xA(y!(u`}LSe901J*4a3!KQ+=o0|K1vZOQYBgYY~AKx}<#7;PK z^1a9JvF@OBj^+_*tF(=;Jt>3@nSk$%kO%7>q-V+0z2Hc9OlJ9Jjg96oNxbFLaglN2 zng%Ke@6hC1aul_wvt3g0&^CxqTLZe)ad&N9ZOX0`^!#zZGvnoYwxfUB3mvxcSHCzl zV)nk;@_p&h;M+{Uj+S%a>|1`cl|ZX)p7!k3_Uf7pKqJ(BZqsk@btDJ4Ow_Wh(E}I*XjCMYt%@d=A zPE>&9PiwBnNrsAAUJo-o8+75bM15=8InM{>vl7QMyL0;0l^hj(goNP}x?LGP9Zu

{Vn9mvw>hK8O-FYJIIs9TV zn`76y{4L>6S_J5imE|;~!Z{j_>ld}pMl#6TIEz`ymmixR4OuFfxn|o(m}-pZFyp-Z zNY&5n z;w18o!~n%093)elJM2lNRC-NG~=1wge~G@nn8QgarAgj2wppFnSgN*l=4_uyd7 z1=A#`y#!D%w~L=^nst2~t)mP4Oqp`(VZtlZvR}xuo%OaT_u)b5RV>_t17F!Yxc(W_ zq_LSEJBKIFq!A##S&jpE`?D8D5qjPdQ={@3Zk^omn{D+#BXyUKk42K<9DClT3|3O{ zQUzKWJSe_R?$r_Ncz)Pe_V{L~bMb==cFbZEKT zhROX{8Heoph37*&bh~A#gCtLbbMYSh2;S3!DNvgyp~<*&N<>{=%CodF4qEp2w2^hv z9IEZnmL}R*OZAcBGK*>0r^u;-D50zA8V+7CG_k2{7LOB4QkvJu!w06V_0MwbpPVCR zKGk{=`gcv>HV^F8IEZ;Dg>$x;q1cHl?>7wpToTkCOnu2bCt{6*KRb@yvdv0LdZ6W> zkUaNw75=Dq6G7QYVsb#NV+HzIhuzx7-rCDwY@63l)Y5>NT2MY}o~GfZ_ZbU`TtI_x z#u%Me0~Uq$dy5!km$}L0)oT8_aF+er`8a2*JLiSE5=+g4UWXiz1TJVvlkjI#_Q2HK zm_E(9S0)=>qdSGi;7j-=`5|L?txP^D_~D30xBTj@f43lCrOeU3)S2LaHsM?~@1288 z=aRAI$Y4Rw*awPi1|-TmulH!2|7^+a7%(q7=uROkJ>sx5aE-DUTdURzb(76Z(dxMT zql+vLb|x2f zPEt!Sxz-A9F#NF(7A7_i3iYzB4SuCR&iB?{N$zn74q@G}RKZzgFgLGRweA!!~-701|)SBI#P@y^>7_0pAb@6Z}*=Fry4LnV&0`$C02=~Y(?OKKk@-0jjddygF zeV05q8y8!ggq!k`Q-{)d*0{J}x4xndO|mTxg`5yGG|=-yJM-Y^RPb1FPfp?B!LIa} zH3>r>-38Q;k-YJZ;am*%Q^6#`&>BtJ?cd|(AP@%uqNM=! zwuVi35h^ce_xO_~1h|kP43$W`=~$S`z@lZ5I(ZCKQr4=#Q7_SPIW|`};^xa)r&aL5 z1>Sg$o>&k4IajaOz|FJ%MU>*0nLw_(37LeczIQfZOD(4EsAa{m6@1FYj&suBon;2! z!wR2+a6tMmF2Fo=&jUar*9Cy3U}$kAF1ANo zNcZmihkZ%>l6=0KuL1g(D*4>X+}zqH&oj`|<2$!Xj(xW3YBMNeHI>ahy!&$FTD^7X zRF)$!YBE@XMw018z>MwhJ<$X?*BB|PxMG;LZ=mPyk7F7W0?irt5M zqg=#7bJ2pSi%tRQ*itA>k9(-JT$L7a!Yf+3L-z8Oj|@Akx*Q>T&vyxmnktoL1teOI zNCy8#_g!x}Zt_6@LtAP3afKT#MVcDqwwmz`MGs@#%sW-jNlQqN0t1c@HU!bm-8d{} zzOxsQm2O-{&!p{1OL_|3$)J#ueo;T)WJ^pfCQ+5s-mhCV4F{?(UC<6JBM!VD-sBx5 z2e!catvg!Wwb`5S)f0IEp^%GWoM^ME7;ZFs##}*e4ukqHV{3UaC3B#j?}w#Yyu-#$ z>|BFWn})8g>dpDRbvE()E2(zTQ!B+O64>%h85Q*S4nNf`aZH;kRn{H{K;zhZ6+g~PLTZAyd|7JvdfW=b>*WFNTri(WJ z`M>_}-}-fMvG0;yY4pFUn1j;bilotnSMz@v$GB&GAbvale|*O!J-p>;*8czIAHm@K zZ%AneJ_C;|%Afy{XbK}R6{>uui>#y9)9zB%zKM%%P4L)gtrCf=w{~%rn z8n(fK*MGah`2Q-fhNpl_uY+rMmj7Sw;y?U=&e_3Ar!F~++y7uG3GURR1%!KP`Vd_F zM>XpMcyho++5cZgnisT54*&Nl`d=?cP4IQ{R`bQV|Cb2M{;#5IU6km*c$LUBe-&N- zm>kY{DB2X@S$DfAGg$>!6#&sHc*0#C^BHQy3GQ!2%}eqjM4KpZmW6Sfjg=}T`dsF3 zVNLT7QGfp_l%WY@XbzwSRH!ZE`b}f}X@QtbWv_WwTQ1FdkYhbV=sb$I$0@b&W57Jfx}x}?+3$}+Ugu&w2^)Gv2<1z+yEMHre61*89-_`b~T zVp?rZZY+zs)+ow$8VXE9&C^NzK}a~4DL$kd_y%$h{7?fcOI{e2Itpgn$o^X0NBM%a=w)5(BMs!;@M;F zxQ(zRW5m#8;oI}ha?Q)7ps0v0Zn%0 zPMBu1XQyM3xceAg%`D$WJ;@*iSSf!zLp_nJGX0x#4(Nv}NL$Cf ze{5N8U1)R2DIBrAJtMa;x3C}|D>1R%`%%+Y1X-t_j@%?qkd;%o&7|G85%>ou!(mm-e0?$6JE{93p-{Y587SSFuSFr75tvT@%bQV?d(ke7kz zqL5=_+2u)Johs)(=_lA}*__3s4D)zek};D#tgu-j@^yc_fpYgp&Fm$(W!2juIH^#= zGAsBwE8pp8MU^!yW?J8iWKWs!ayV`>tcNg4tmkdGW!2I$a(;%Lhwn3HF8AGBPr0Za zUZmXQlzi(jEx53L_`}rK*qeG>F8F5!Ps=&}y^#rtMYM;!gh4=!5s~fp9vuqej};{Y z{o=J6;YvmzBXJT6u(l&GSh~MV0^}ORfaQ2uHuy&#TX(&2Ge~@Uy2?XtJ}Tbnp0MYo=gi3~ZtKrl zLY$^nyDgMf+p)Jjf3ksA{e|`;GRU8+npJ!r@xe6H73mLkT7zus#|6f(0OOB$G6yr) zXX==5m>F#f%ZwzU_BOE9ElkVKkp9ljWRrx z@x`P%nx!dqRw-rE-J}Q&&s{0b36Zrukm@?T6N@&)yR!yDGj2*3YkR-C9iYSq`%ldm znrmOqz!nObeDkETp z@nK=~G>8k7aKTN93}#HSK{j`~J%W%%=6+FrRirJps+&zHk`RzsHW{gF`?OXq^HaG( z`|#Lhbu5&JI!+Xt|LK0U>nwP2IMKJ~gPk+{1Zh&ik+g>2T6xukBv0W0tsS!2g;z~3 zLCOf*Jyh6IKknlq=}J8-U}Xllhzq^EqB%A-S^xsE@AHjgwrC8mWh1X);cZYU_CX z3+)i^?DYIEPvL=iTsezWL0X0ui&19=$t~nvZO8=dt#0arDM`j-Sa)bAqG5B-WMhVg zgNtfKl}NlGr3OCf>TG6E=3vwVH^LGSRDp6#kb#&P()7waM59bWGWS;Sq6_$ z&7xUY4lLr2Vh)U!?li2Xx`T50WicuE69`>O;{co#eM~5KqkX1muT*^d#wroaNmsA3!_()~Wbk{-t`QaTqSCfuh3?Xj$aC|o3C z9`g^tb*2F$^ba3_tpqqqeqcVC<%p0S_^f(ii%qEz7xg#{Zvu%)Les0VUIeyb)X?CFlTX{dpP~QS|;ET;_f+BH^)W! zV2H4e76KX+HY`VsPGVj)*~ehJ-($(+@xG^--qbZ}7;|NxL_wfCp8W)&kD;;)>&ihh zA5Z$tJ3kwiM6(XVRjY0&NslCDFD{fKvl@d{sz6Dh0h$g`K*z}?(z@NRd`2CLy_LmZ zL=zW=JtT)YlKT4~-jIIq6!;E*L)$6V-5uDt_XII`i;*AX1SNpCgW(gGgr!RNj$h4M zoPN<$le9j$(`Tu6KO3#k6sa(y^mym!(b%QYn8Ef9KR`b{qmgT~vbmhb>|u0P$^n;1 zG=W4cjQUOIG-U%Sx+YBA>yDS%2R}>vAJ>s+t_I7~;^_bs=Hz=q5n-f6Hae`N6)HEH ziqxrHQ_|!M-EQ{o*4fXwI(w?~?6<4u4H1CQJi1hOSyc8`lMoT|4>0Ve_{@c;(X&K3 zB$eC?|A}eA86s-PZ}%*!n-FGTAoD_@NQ?FL_dem1fq%NR|NC9u*T+?ha4C}G0smXD za_hcKfDiD6oPy%(pQo>X44V(WwCsT`TxaK$n`eFQ>+1X;suaN5`s22J!Iqt+8Xd5& z>okgZlK<6{e_YT3esn9|s5%(J#&1T72lgA0G71Rf$(NysPyE?Ic?ZM(dbbiYt?jo< zk8ruF_xo8^Bm(oV$B)emDKr|g^BeQjFyy~OvGu_+62>*1>f?b>1cBeXC~fIpqfIzr z5MTp|SJ*1k@D7iBAQ*vbRvV{vo|HET&#ohnmYn7FamqRFH>HehLOXad@mg=*0 zLSI40rka<8@Lr1ZL>XM-Y%;rOsijDJ9NAC7NuPjWx_Qt2-Mt)auBDBYn4BhaU6|P* zZFHDO#AZqziX?rd6!it`U+h&whrdCUPH#MTpJHT$%r2L_iL~ju(3e?t>xS2H<5r2P z)iLfSy$MzVOaAAq(|kRk#n>8kvu>|AZB5@$ak6>*#j7c~dX7;Gz9HvJ=nxFg1wxKUr!zNH`@dlVuSL z<%D)^XF+5N4!)dQyslBwg50+AHGMPTBw*eXE-^*v-JZTiqs6U>ZekrF&*4GSa5ajb<)tn_@M&FRyH?j=2k#~>ADf<;Yb#g_+LB8L>r7zHxGNF7~+1yX?F zzveek2(2IQ@QYQ^U)aV+7N2ZN)QZNP6ho6ei?Pt0PTL`0kpSq-N#X1$Y6(fqQ5jNY z)*a7~mC~NFqHRL5jG&}LcB9iP?mE+Og=vOWl@7HLObSYDC?d<(t9w*yoYmG|rR@kK z53`Z$FqNKKsY&VS&C)0W?sm>~H<^3c+QeADcB*9RG%x(rRBp+1{!J-FJ_Ly4WE71j^q%72U<+0l$f{gt&x&m+Mf@* z;8fR_;ndQa%NboKeu^{cO&P;1$<~?~nv&w!UP?(a*BZsLEsIZ?SZ!9SuAaB_gB+_) zMx@)9@+#B3wd9`Rt1CU@p|irQyzR!9z_%8)BdKrIPT zt=2u?V<#VNC_~D!14Zgrpvjc}dXS8V{8#Mk7yxUSat?7S%L;D3AsAm0aaVfrt5e)x zam+0Wj){U)x%msb%zQ?rECGt$fNpFA4wY+iqX^G;8 zeQFu8cXcL>yhxfB*w$-ad<-y=^jJ7xsdt!R{im!o^U=I189HmGkL)+& zafTIwqLhXMM@&0vpxAQ(p1v7#jp)0st5ZV2%%w)rtU?0eKOLuO01*gDZPCxw%cqQOjNzS?)% zv-Sho$c^6OR0={Bg);~Z%1ExdtgliCIwX9rT?5|e2~iu}?WoGARNHU2je8nX44+3L zOth-vwOBrH^nk8Q6T5V4IoNeob7V!|U1|g<1Akx0^A4_~S+ZAyBlf{HmQNye43|ICvTz(y}?Y zqTdTW)y^Ft%U$!=YCMVU`k&XKP0q7o+wGgZj=}YoM?Iu@E7qKtboZ9LoDG7{P* z?5<^nxi>rA_SeYtXpc;Vm?{v&pKcMT|7gCwWGB1fxZE!s;lfdE;%t@*-4PLe00I+Z zIm@l9fsGs03xGj_eYMEP7z8Ny&<}@<9?&EqU{)a_LLg;6?L`GG4BkcePqTzLRuY9I z7OPf;BYY1xU3+=-ur0kSIL^-1$sXHQx530NVN*Gz8x!pF>J_34;rC)(IH56`y1DQ~ zja#i(|FKpyjFYmiHCE_0BNLJEOtna|TMHd}uF0K*IC&0lJyr|q@E58|YO8@J@6vot zk}hR-B8wUw@1Ba;#yvrX4P~I6g8QQ^1F7Vvz%B5!^sMeo(W9a+A!%9dr*8a`myA9y z(C!L=A$Cs#?{k|6a8}@3^;Iqag)h^#{o_1cLYJ1ohb&kRy15`_hJzO)z-lIg*nF&8 zUW7xrTvC$%v5cDz&iYcbqnN2*Wm7_vYH>bq7REY2p7{R8a!i>3Ok5Ea zxOZiwJ2rYd27ofX%b6}=ajx?;qeif{1`^}sXtk;UyYn%V`>8RlVqV*)?vV@r%mg!v z--bUFN(}ye&_*wNHPK{^OnjlJ~XxP$JX zD^-fd?2N`jO8q!+__!IV?FJ@o;6!uETGyH&XSGRKsJMw^U{Dd4i+7^nB8;cA9Rp6! z$qL=_4ZcOo%2!eeqbSlwyTlydRX4I0_yFh#l$o{lDD--Kp)qQfSmGgxa}rXsqoOF#CIa>&(uiPuX#HL8>PMt5g5%xN=^ zw*|Vn=lBc$zab6=EI4(J;SI+PF74Qc3jzY!^tEbcyp?-vbOB`SvKQYzJNX+q#vXsNVX7B}U_k8$k1UTgEyq(-}zvW=`24PTI< z(;tQ9Qe!e((Wy|D+yh%z-4D(CsVBK)^l92TrldfVXD9n7(I;fd2(2crNT4n-Ci!=w z-Y^|5TCHl)XbGkBpPT(TA%x7qg7T6J$(7f9mF=i?e3@Zr)}d)hcltP~TpslbFN^cP zImyw;5Fn#ic>^Jpzu*U84cWV&{0l>1fm@jC-)M-B94vA2K?Lmmg=vP(aH>Jh6`H2H z%wh>19GyYT6o<^Wr7ns5Vw*_PCF+RzF(G^Zu1WNN`wT`>-`oHTf2B@=5a^oizs}JW zU@NuOiuLJBj>iuXOl6?uz|?mO26|QYgz)u$?_WidA6^+H8kX}}{u5jyl7P<{*`!X8 z;$NruFR27T?62hL|M4BLrhJ9wKLytxpu`iv%aPJK+cDS|aV9vxVSEEn*TrXlCB&R_ z#F=`kT~KsqA~ktgCNCp*kGdAue3428MB~8H%eH+{I{B;fR!!V*hhnJYD%;R8r=bzc zOKJAyt4oq!GL0=^*1wt?^rtw#oQjfBWLeBs!&_Tv@3kqfkh}g;!(J=DG?U-GVQyV1H)Oal z(9wIB3V%w5(icz7Ys*Mxl4U6)-KeHLlh+6z_i&A9gejEQLejghIWnUfr!mhN8cJiB z95#8~Y}TvoYwT=f{rYy!AO3mW9pCFJBA8U%2i;WoTzYo_;d ztJ_mKnc05X$?q$ZA?mupyWcW0N|S@@KaJ6VpLbGZ`OED6y+_2EAFl}H1m{vnKQ-Bc z=@0MCX28zU%9Cz8GZitC^^5C7XCfO~CG`=IqB@<)+AwXKDXu#zL0N`0Tvm2@$JfbC z^U0*LtL__j*PaH?qwxqS=PZrCS^ZIhXUECZ3oEuhPTVWV7))T zB7iYW=l?fTZ-D@;R#?UU0{HO={c5iYkK)2XDGURS48NgX^Rdt+<)PC(VI=>_>!}O3 z!?U_NdjA~IE$y8EwU zg9X)q?2Prz^tZkoLO_7u)KrT>rt6R%62n%@nN+U*VG;3D#5#z3t0(g+U1(qlDH0wm+9dt)g$MW_Ut+KNGFHsU-wov zieQdwUmERhzAdq9h|TiI zLk+-;vt(y4xNwK6*D(7b51SQ*_p!JiYg4eE=z-ie%IG6kQwD97Kjd)jiC;*~2XGTr zDN!L+;zD|w@^3snGs@5)oX>yg5zU-T1R{C$LP{Xfu;2r$xKg3mqfG@HBm~6(7{gex zaNJY@w1vT=>c5y(rf47LYb_{=!(o#$I$v*4kpVehKBoT|(JP}fioB7gI98(iO`s^> z08=!@8Wv(QpH3?b@I00T8Va-|uEm{0f8$`XChWrJfZw`-F(w=_ea9!RKpdw)s8=~&9goYwvT^(EC-^@qIlI}`M}Z#v6^apQ^#C$J|(-> z{0XXK4N-iM%rCVHHl0ar=oGa|hdk3zlImTpo<;47g;O#Lp2f1H*H5L^kx|tBW zW9mpzN|FmWUXu5GM2>tTjh{bS)VxhCzWX^f_pIq|PBjaKOb}V3-q&XXZPKr$3^oZD{TkKEPKJ>y?WJp3W-uxWbS1@S zC_kOTNXJ$ZVuqGMYHssw8`K00D03oobMn9(v{PnB&jQ?@i^IIyk-7JNnEo6LFG5m` zk4l~w!ofbWuqdODc$LJjW%U>yU?>Rl@^9>mZ5BH=sFunboTd&@Z*>jif?;`uFpzJ% zQAn?=yUax-Kc`riOD@QLz`rzr6(M_M%z74r>E3ce?EV3bxFim9pwd#qBqYJd^%4wv zFJsDBPq#w4V;(`~IQzyFQ#~zI@*@g$)$w-MxPD7WEpvS&c=;7EwfW1!QsZaX8eAbj z?@~q>ml&1i(uy#i7>Nx3GvW^Hi1`N=huL}D!{?LlgS1)G^iQ_Jyxug!QSl~^hi-Q;g!rkK)Wly|NU8%&1 zDaBdsCX*2Lo=8y6ce&?A)7dDP0qU_j?i0D zZ$!R-yqwq5ZQPEDU<9I#bu_E zk@eEZymxxe!|sWvf2so%KN-cXE)ITmaFfe$Ra|EVy8Oj~^w25v9IlTPN+tbK<sq)svS6ig{Z7?G|Dg+-?u>qPbeTFQLt$m6|}b{J0(+apiSX#r`4ZzLlPgf zjvdZndWlSn;E7iDo=)D{0?v)Lz7nFs!9qd>@(6!<^_16uxk+-Vi6Ct#l2#D5uQAa% zUP4fW#Zzl7m3dxCr#Z6I{JA#VTO&{1=ZHO0&L)L@Ww86lnWx#y%doAe9XFkKO4uyE z57dwkMdIh4jB1w?t(G0+p60cJUrMQERaH13u!N`5hv# zZjyeJW^Kb027iW=Q1X61k(`(@k<4iaU7~4apIILCHyq|-8uJK!R$jT)9^imJeaj&O?=AWIbG$1~5>OZYko2?a;l%OmS zNi2;oH51cGWt$+YmhA4waSTTd@(xdF3L2gDk50A;Rp}I|kB*#GF6;#As+xu#kDs5S*u) zC*ip$l^b7v^e&fpDnV_ACyCaA{hlt5Gin^O_xhX4{qkwjd_zf$0;|8zs$b{=_nO#QaW&uC?5k^o$YwW+fsJT zuJ6c!=7X;gt_=&So~78_rO%Oio6FG2sxyEUilcVGA^VdE~Z}{i5*EN$~NvrQZVOjzX?<-TU$DJ zU~}diIr&ZCXI{M6J@RO5oCR)+i?c_EJr&W9CDaatvP!|?-^ggMFVroXIH0U&AMTi0 z9h_vfz2!kbx8Z%#>+R{h{e7^$99x>q6U$A;PBMTyFx~gKtr@v+FwZEP@(_0Sdzl}v zW!}4WQtqJLOVoBkxeVdNNQB_P`;0RClPbh=czuoEowMY)mv8{OSnm6S+n*FF8HikDiPql&B^!%Um{i!yL~K zHR&X!h3lbD%Du3YEYt6{L`3<)&*e@lZ1?NKDoUTcDRm8gK$z4`CN^W35l$r%O#P_B zGe2jhc~f!qDNJ&K2d&;kiDP_y&yjmQ7Fp({d!bvifTwC`+C!|(H$8Ghx+c3+5f5*Y ztJb{%y%>B4hDBM4*Ddo(hGjd|t@gt9P6w)+o%C*hMdnBqqRa3RP-&(~U?ZkiMVwkIVSZou1H3KBov@72AX5IJM3&nGH2B(-s~4wx#OIEe!pbg~vQ51J{hF#0mE;5@`@|isDBN z1kdgCf8zT8fiS2Ljo}?3IntS;|M>PLGQ>5@IQ;PDqK5+;J|STdFfNt&_dB?k;u+Z6 zqr`GtnQd?T&)fDNu>ZFSS0*X!0JtSr6P}$`ruy%RzTXpI2kNhM5-#tjb?=qhHS`iX zsg0PTgSzAPWHH2G!~|ZI!UW-`6gqcGk_fk*RCNvK_V1CiHG0Sey_^`tJjUs5e;s{& zKlkGw*)1hL09O}u{`{mWQu;@bG85|+}H4)o?G6Fzmv_c12v=pY|NQ75 z09_jDTUbZ(rZleD|IFb3I>;piM4{2v$Kzt@|2{G}JWGZ}MRK9)N@0!sa;^CJn2Y(2 zk`NGfAU5RK6*_f!g+ktb7G{L;WKye+p!9RF0i{;aP$tz@FaI zQEi*p89OZlJ8cE6v3GarLGes(}=@sJeG;C&xiaJ6boXC zgC8#4D`0R-A%Y;|c?Tg94Sj~ff(8!?y|lt-E;uj*$|6>@vcJ4EGg`iE7Nhok(M+zn zl)Kv&>ug+EVhy#pEUcY2kg(g|`AO=$j)$??f_V~=1>1SRxzkc?#!_6!` zeuh8NAeRA`>(9`lq`h^mARqg*bolD5BCcAjEO0;Ay_6Utv{1J7DXC$TF`7y~5mnFw zb-cUK%C5lolbT;3wYvIKtb)9+$Y@%Qc&d4y5H{{KY(KfwoS!uN<_@JJT__x))?9G% zo+<@57uQPpEKB2*6YK6u+DSIkh<70QnD}Tb9RsZIa<<_gJ=#3lKT%n#gz(Ee$OVcg1SFG^oa=oF4+ z+rpj?aR5q9!y$cMy&aNLDzp@)wzw1XEf4a<>^4S>GQ*<{1}e9?>q<;nqDsY`=O)W= zbc*MJa)=xZS1;9A7{lnXm;<*F2_~^6y%)VyRi9!L{18MVuEc)F#R74Jzn`4~BU7yP zDLeK&-iuK+ko;(;oA{8wt-gwWmvRs4-y;@$WW;W}`u6j0f2U9J7TkN|dHp>kU zwWbu)XlLw4ptb-3R!FqQgdIaOW-sVkJYDnmU?pV#U&g&p-BCQ=O;cz&af364#~9E7jLj6Nd&Qp?OJoU%66?yjr8=H>>i z1z&hN;$Eh@S8-0F{vf{iQ3@hs~wne!`M~y+G8Ims^WD-5|m>41x=Ye9OA+fzLLDr3Gic3ve^AmCO_8vGt-Qm zsOy9+7f%Kq1+UEaJ;2S#yqG7q-E2nCa|4tp)K=g_gTVR=r^EMe7y4!t$QSJ9=#K2u z12*HJF?g)$PM{fvBbbjO*1ZZz>;c4aSHPC%QG9yC8zBX9eW#2vANp0zBU9`b%keJz89gCt-EE`a%4WV zT=cvJ@86*iPCaU4R;n6ouL|(;Zft8+rIbtCT+vqT7}f~aEZ|PYyzS4=Ck0Q6!zHq3 zJ#W*Tg=0fs=h`k>cD8)N^bL>(vg(Cua}=aE&F{atk?t^VkFh6f|4nZZ;Nu6iKWJlG zYw`4l=3cVX{Mv=!I3~+;c3F*2wHCn{PA(MM7?_;54)3S$)Ms zUE>}rocgdbF&4{SscPlcSH74=EkSj1NDl)sFiZZe-QSaH*7uEf3o7Akz3gKJYFZ^w z0^HJRuAXCeY?s^7dZzX+xCQfgbMeA&7AqlC`xdh*9tQ zuGtTVC{!-UF@+K!fmPwFvBv;tbUi3LYGQP;#n#3r2Vyqrhuhc4Z#ve3yT>ghI!)=W zjd~0z)@QFH7TARJQw2HKO}%csKk@xjbbL7ScYT=4MUXT^fp>{@{5RZ!0Gj%8_P8|Wd0lX^v6vC9#4YnIq9O@p(3R6?^ zC~Zd3Ax7ZZ>+XchIqX*AfFiHAtn>LEET~K-W->U*WoL5rh9J7Nq~M&T!m%BB|NO^_ z>%~mC{*TX;EC}Q%u#hrQA6lB}NRUXtNQ2-D*ZY2x2v^}I~vyX0bo4rqO!_b7w7u-C%sPe*@)QL7V^;Bq)U?ERJgQb z1{>eFHZdt%8NP>|f>=Qaq_Sq5X`%=FR@3FfO?op-1=&$3k|Y2sgfuTFcf@fQB~E%p zMgdvN^n4=-+$8P~xogOjBi%tJID#O`ZwxV0DX-6e zoiZnMA|}T0;P-KmyrG(emP=9z#fn!{hL=Fg)QzO#mUrv+?!!)NxP~%^LGCor$VbR z))ITL)34r^#=|%$gC-!`|E-h9={;;;D-*IvIT9BTrXDhT)5WYquTrvc#r*q{tfSDP zP#U3e6ge+d%bijgP0Z0Onz@&`l$MJ4jksT`%l|qRiw7+U`9f>La9AosC>_9?4AZ9N zsw=d7pX}6SOfkT5C;Le~UR+H%p zsP-A`QXT`;5$YLR0;yz_ogm8bq(YdoD&`J?Ym<@~oJ0&5TJquQ>B)l0LMc!s7c9CI z1=p^Qf%!F)0gLwNF~H>2X(l?A0}x%D&W`3PFFB_5a6>b+z$@wSIFA@G(eG~NkoxlC z;=q0e{=^uaJ9cOfGd;|C1T-`t@8h<|9oXM4q&cn#wu1Rc7BvI`N=ge;dN81n%l?rz1XBiug zKTiZs{SL2IZDkEjUv)hE96`hsNngk)qtQm9(PTi+LQJ!)gCsBvK|ROii;Q#e^E3)B zVn9HZIKDm4w1HmQb!0H$DJD4GV$JyC&lqsM8^KNBIItp&cehQDM!&u{^M72H6HttQ z5A&042^o`ulmp}Uesr+9lfVcs=#s0l$SRI4T{Bx_uGV!HC_9eGsw4Qm+7}BI;;yu# z*>9_I-C`%p>+tcciW`7TT(}~M(hDV~l*0y>+>3Mo1M;&*-tGXG`(_1X!z;iy`uPiK zx=ZwfmiC|2*=QbHroe#^oV+pUKb_VMiEA66PmIF4z&b7j=*Mp^UpId|_|wTuQ>D)} zPwR~$g15CM_fW01CxW;x?y^~%4_ zTgrTo9&?HKpnnK{;V!!;-CyEr3c8dPwBAu(O-p=<-eo>uF0WWvf0VV$Gg#Qy7r_>X zPLmx_SkK=Mp6mm{_^}^>TL@g)Z2N_i9*l-Jhu)8#J6vYBU^;Zr;d=rUTdNUic3zpH z=Cd5TmT{~ji05iCB8~?YXuC)TtqC25QJ9_Ho%)fUbSZI6Rjeo+4VX(PM%K|3jM`yF z;z`=`sxgz^dOP&8lx!%NOOI_oiqedbpcOD%?}JQ0u>2xhglrAJ{p6*@^3r*h0ZzZ{ zZ%a|gFAQ@h=n*>~?aMF)~aTH556jeX{Vso()yScNWhljl-vum1ViZ8ZG{1K`kJ zQ8-qyGCaZciQC9|gu@Kf&>hF`Yicn*Pxi#f=d=Ap2Lm?pBy5GmlS`IWB`ydA8~%nV zQu=T>sfLD54I{$PB$^=_@B>ZABT@zfa_WL9iO<0|F1@ugNL^l!O#*}*Rv9aby#ZQi zc~Ee>zt0RAql#9YoH(1YUaII%qO+ZE>uq6Ol_dY;6 zCeCS!Iz-hTMtDih{Q>-qk~Nc=^ZfT0$`UnPCR)n9_&h!Qeg!**$H3TUYmNm7H;wNn zA<-ByWkCGEZPp>YxUs?H=g^}?i>lm7acym}4t`8YWwb7DeG$x_(o~8>~Djws|JCe0szJNQ;_ zBAR*?lm+TRRFloQjj>7KcIL4o@gNEM)m1l6?1#$_HW{SeL|FS^CK+V6-kt<2ldLm* zEe-OD@$2`hGtoUo1GQx?VIS@sU1#Fn21jsAl}D$}ksJ6FJZv4JbObA>q(iQfpDO3u zlmw4EOY&{CPOfNhGWre|dm_0DF>ko}wK163X|8xM5!Y{rxc!hGTOrmAMVQaCzIUs? zz1nuk9sIwY^cU%{X$hpQ+~xTf7VWMx8(jGs*1)RCFkZdvwexhSUiGiTLQfU?m*`?Y zKV{CAOP5@+~KjTAXsUW&_AMnuj|jT17*{Ndq{%ypdcW>)b||PFEDy z18JDH^N5FC@ye{lwt!CiZ|%tw>__Svj16h`%1SHI2rkP5e#XD-8m-FLo&1>( z7RcB7V%pZ+){o&&a#pBP0(ZNp| z{%<)G9LkkCf5-$4n|-Y}iYsrl K3;}A?#Dw<`TkFq39gboj+rj_nh3Q2P}DlKWH zF73N5&f!C8uEeHZCjRxq%K+&h3W+UubYT`XguA1#kPa};Cc=gBBaRv=NemRocsLMe zNNWYJ^sm3A&`Sb8j(epnEqT{Jc(rIKJ+ZARnhl2S(td9NW?&hIQ7zn@f3Eci$>6<8 zpnf`}AT}OJwgXO9z%C+AJOf#czCl0OgU7pUDf@^bcA@3-F7_*;my>EeH zIHc;LY_tzpARV9N_G9&J%VXupETI)mX&KrbD)1)tuu;BaRhvN_# zzoS?u4_8_Y9pf_uX6b35K*}wsXNL^Pq5jdj{x79juj@0#Ix6Q6W%)~7`5!{`=U-zV z?mpthX8e0e|KoEL7=UVOM)e2b+^16iA8-8i)NfD}5Xwiaxro>Uu2v5Q*w*{Wl!?JO zDS07!IddzWEIxK-@MZQx{QV>FO*5CbgZbJw28%47Hf5seq9EnobHCUpTSs8nH#_dT2Pb2J7u{IP~v+xf=1zW{O&-O zEz~XuC3ZN9xX(Ksz{|W` zZFT;BIFR3psACQJrHSp)M)|f=thh#;%66kw4QI5#qP&c3%S#u5-8zn7^R_JLL_u?Y<}qdk+>r+Y?V z!lys(P+>tchsuuZ2uoRHuYKkrc^8gewxUoltg5Wa)nu4%&sj$xT?T7`-?0Na__(Bc zSl>$U75zn`>*e<>wE9fTYGBe(`@6<$ZP_@m3e#1 zNQQm>D7;9%ZE)~#5AU4s+qzW3#7{kSr3kzb zL}0z*q08KgiG4Ff{ceOFT&W^#**XR{f!f<6rKa}?qpHMU=7%D#>S<<@tf~4=+Mr^o zJ_u8WSE&58K^Zvpy_Wbx8%;4hu?e9qc3x+}%0Ew)cfVxJi!{}KEh0e9WL*sc<$JGd1o;j3SdYLrb9t=(s0&Di14Ep} zm89_)SF4V6Om3v!#`lxXY?cf35QE(#s<^-D!@*d55=tvS|CLpTjg{F+iSr(a`F;`{ zli86#rKNL*Ej&}paeSR!gnUZ~z`XBy_=S!kx%m%_BD+FRHOT$%m@6W}06 z?Xqd#1uH0FQm7J#hAw+&XJ>n-xXxh{0E({0Tz%&v4AKv;A6MIYHXfA_RBBJtGxhbI zs*gs0uCHx@I$wqUtLY@B7 z9U30o>M9Ww^XaaZeN8tqxx4G1X^=#raDSam+joQvmHJuP+pBC~&q#ewm$=@lL=AOe z{Au}lR|eK!61^@|UF0E@_Tn$d$G%OQY+$4^aVQ!f)*~`|($_ZG!qp>d^$65sCde-a z$M@_yqrlf=Cbvd(nm@)+mdi$TnuFOK+)r zo;H!`;m*jVa`6{-1&&cB5BDKaP>xy}+7g=FS?Uzw7( zwlOs8#NB*=Y)3d3ZWjQDJWgL8>H02QXq}U<3#V#XD&_F%dhMQlQI!YR*l6h~;46$t zT?=3#9~1_p%bwHWENoi$MK>5ixi>TiPj3deZAIO0kNJ?FQ5RH5|BeX?g|`2u3@sH3 zq1l}DLWTjDVGl{osWb|MCv`p2s(d_{%%m8(i=!;FcyM{grHfMr=np-Pq@vA7!}}+W z3f~(9SblIxgVJT2(zptJZW_5b@Rwz^aZ{s28@}H-NTF}ncy!d9wXn@iePrYtcz@F^k4{W!%A38tUZzZlsmyQhNdxF{eiqlhvNQlYR< zs3oDrdLtY03-f#G#u~>;g$xB}klJ|-BhEL0R6|s+1=Bb>cBZfJ4mQqA^2`Iv^bgae zdwUwgm0`l2kstLKfnhQi5g7&i!DRM9$05)2WUC|tTztf%sa2qwpI9*^@COTX3J zR2)7voI`FocQ_-UCG8!1vx~j8$hOEwdl3)%N(SM!*ZmHw9#5OLeqOp=HSG5I2oayA z(9|u%A@}Yrlv}yTVMp@0Q^b_rPLSjeLk-7iYp4M0vf5^Ax+-MnH!Y(a$fk(n#;s0wy4MyBJabb82S)(N6Vf-@ zw4MBK^m~8wt7Q7(NB~5pO0be>s|g7F_5hMjWN19!V1DX%LEnI%MP{!G?2=5p)mz2b z;3rkleG52{CIVb3hYCH@>&zVLtO=mgz+0Wqm*p7o#`CmkIJ?R@hG|vL^A1IgM|&y> z)3`sKITdXnjSbQ0%QKM@#fq$pt)_7ae!Wf1O}N-G3|}erfkmiu$jcd9=qll5TEJz* zG0Tp_`Wf)@U;+>RJlUn|H?W6eKDqS=*XeF)sEZW`%|upai81SM6qp-Gcaom?Ld5Aw zN~Us^(yy6mur=ez?Rw|Eo6c^xn;0w|GbDQ^+EnGVt`?u1C-418?}H|YI5~t`l(FM6 z81}t|>wBz^h}wrTwq{oGO@!~MaB{3^3SuU3!8@3ru?SGbhmg_;IZ}LlC<{x%@buPp zyi736{N;$E&x^6t>V&bBUY4I?k0dK=>pSFU5y$9+*4u_KfEOkw{ZKj7OC&28^XITos{wFhYMU!(18qgA<^aWRrIBoB z_ymirgT4Hf7o*d(9`re$@*$rxPuVV|1jk8fg&2#NHBt`btNWkF#l5Z049bfen=XHn zlCUj$&%GaN9lr9uQpxII86XF^3W$H^V}6p6uv=o!t@|0sY`5&62QOfUXmJQN{oEli1b}>KZkiwm3u3K)o6dniGCD9{hkdWNvGA$!&T?&vP*&!! z^_PAI0f{5VKm+E0)Nng=^PW?MyxUlLYSS~O&dZJ)d7!h6(YsFFMeCRSBlhbg7zQZ} z;)<(Z^~n=fp~7>r;Bttt_dYHEoG`T1PepULeaEzjenjzvlai3v*tL%dp6(mf4k1&T zSO2-33)LdCO6#%%INn*x{JgzfqgU=Vyg@OlZd^vJa_jw^6Npm6^=kq|G@v{9zBJ#a z!T=Jrk+iDlm+}0S1a0@*DVadP^ax0VfuZS8_Othbu7WWkI*F7mDJmB`9xcZGI@_O? zHNMj{)B8Sghx(tvO`vGo))TpBD4W2;WA?-REa6~OcLB?l$NFC=C~wD@r*BZ)uFakC zC}X~IR7ly6B?muOftRpnd~R%>-R715 zEbcsMFK4`2O)ab2^Yzzkg8i^Uv50{PR*}{Jggm=!M!ruswU**Ghx)=3EsE(*un4kA zyqA#Jv3as;Ou|O(L@&{ekw{*F0m#@3$sZSRUjwq)R~c|N5o@-y(qN|Vg*dc6$=GT7 zLS)DK10FUsR#xS#(>B$k<+{3Jdo-GG%dIAOaUki%zqQxIEbuOjkus>X{$hw0`F!g? zMrGKBJsSHfXDiU%LQeGtC%1)!Lq&YlyVm@~~w0gO;Mod)%CoSsE|+L*)#g-{YwjIUGc zzVFJ%mtX(DMH|yHC9;9i02xadVe>0Z;?>|lD6}@sHlb^(U@D=et~7J;D@EaH2lj8r z3H;elOEQu$&Ph2ah^xE(6_6Mdn#eaS-~fsH8kp|AO^?-Y=_b#T8txP%E8J`-zP*f& zQz$UvKx;QSMI)yERlExI;DulvdtACBH}wY>g+JtsuI`g=Y8{$wCj%{w1?$n^nM)Nt zbJ2XNH_f(x`~QoT-u+e4o$Y9{E&Cdxftn+iKbGLjownc#1(NEGjEO2Wn*X)S;)u zvo_W>M+?rt_hXIDZxEC)7lDK-FYpP7`QYy*ZOB?G?(#DmAtXX|#~UvI1qM`G1jc#xUK9_SRBY5C1R429HqFMwNQ1_u zA)UiY#~HF>NX_zxbD~HaZebe~48SFc_LrzwSp(60jc&#XHE@M%Z{?qWE{=;BtnFQ6 zcV(WnV1sds&h3Wnd%pNCWJ9J@&73C700dUp_FfqwG_YO`^j@y++`|{97eBA5i zy_mtl47OttzZb#yldm1_ptLp(WXi99Bt6fG-w?lA$9U}@f{o$I$v;VK{+Ua7BQ>B$ zas^YAdBP1N`ipJ-D=PuWg5XFJmSOW)uSFCT`O&6zT`u{CNw%0u(5$wN*PLvJT>|rx zu`Yo<$$RRi*Ks7jzxCp=1}O_8smp+cCY~ES;>4x_mOn*koBv~vKRPLY@E}_pZzP)36uKVXf7eLPbc|9DNa2T`cet!gj zyYD>G&ppN@*oUALKxTY`dTvP1%|5T}vhzv#ygskZKGG}`2gg!#;RaxH1#1^cXM}j%^RBo5v4BI+eO#lfJ1VU4QE66VE;GnRpUF5NFhIQS(5l>uW}uJ2 z8{@q04kurqls>XC>gRo)+PB_6hYcExeGnFqV;=?_o6Jf4H&oUZ66&p+yCgta@V{ZU zP#}n(pee6J0}ryjtgsR!;5uOR_$~oW&?1wk*0&qXf!je;s;H-8RbVKGIpt>6BL7F z2L`dvk>`(9vd=*%uGDu>A^PKSJ?7Fp;{tkarB>kaw$n(UPaq_#ly;WV{Y>aCb($vo zS^1_5qgaFiKFDx+1}z($7SQWOV+>=%NIxzqAvdy}wPh(uAO_Wb65-XdE{k^HE26D~ zaD^H)Zik~B?#|}fTH1v)3hY5R_M8jAr@EFp>}`;e3>hvPx#yzN-PZsL0~A8EIG+q^ z-2KC+3-aTcCxVaTo^mfV{ zkO7sn^vovT)dc_X&K{>MzAwT#X;aft_r|rAX2#wkzgv;d#rz|wlFgFK4#^KJ*5V%Z z$cmbp^2b{n;1DzWFh3;QF`N>HnUg}SNf))rzXL_nJ=QeUwH>7jwNCg2LwDq!;WW`7 zQmTGgIr%Ul>tCI}y)ipa;|Iqbw!h6ow3weHV+8Y)7X--Q+5Y&WpEaxrR1jl@FDJ>U zV@B97L`v0n3fnGJl9&?dfq-zsY`lL|V*96aZlg)#F11h#N|;=w>kwh+m{T3b`?a*@Px-cT z@{Da6JQ4Qu-(>Glg0SYM72Q5}Zv1JUos(oIOQETp>e}go0m7kHj4?zxu39oPqCAGL zAuMmjEZR7um<1E%)bk zba72w`pGJAiuooQ`FV#VjOqh{GovfZ8tpNJhQvx-oF@z5O%fY)=ZCLc)<>T6ag(&m z8#zJ_<>nSgTs!{CV~)a=-5np%QT_(UFZV+*o}Hq!l^*R8EsPXu z9THG}=|EhN%=`=klLdX_d#+g##Kw&FL!c>qA1ZnR1vxUB1N?tyeKEFNUAuQ#<9j41`9XDy5On5Mj{fx0U5&WNlyT7Oc;i;5#43 zj5Pw^;JE!|`HU6`#Jq#jT={lftT+toIgA7%>=0*{TbnNtvyo-gCb-=mjI_2TZB?OC z7jE20`R$Nt^kyQ&+?lqcfe+a;>Qz?%CvC;07P@+Eyy3;G-(vXPAPbm`;niC*?snFw{ODJs^hatqN9$I6I3aenng-WmdlF3(#Q6{!`4vY zhxzZdmCjrFM=oZ^fco!IhZ)+-@I+#Z?PeWT%oMn9w%KjPaqfSk3(zwh`2NcCRC>MD zVOm1d<91jTiZ-mR9N~XXShw?IiJlsMG<0JyS5QZVoKxprw6~0cGs&l=9Deeu zCQ<5*3FqRO4aZCc3q}sj%vpn%;q%Td4KE8Z^%UxEpJsi3x$B6SsAUP_l?kK|T)QTR zNRJj_iz=UnqQ!xi^ne0k7yUQG?1sdlrJH+vc*$u!dm{iJ0N@U z4o@MnX-+oYz_}yEBB47U`QjD zEGYm>2kh+`{T-#$a&yRl3TT8c&`MWQ*4~=8Q4wT%B7pQ{4lq{O_oyOJ(8Ee6r1`!% z(EKJ4UZT60aD#vo01M?GYA)@{4rJy0WmVp(|CpCfV(-90C(iZsmsJm(@Rve7u$oC@ zDh4f44|)a62b!^GPs+BAlG}+IRmNRr=q2?-X*VuQ@g;aRw`gQqY-7!AEV0%j zgohV&y2`80V8~&L)wAL(P{w;>=jq!5O)zh@0Zi@L5%9Kal9*>s3GfAV-!YI~D!68s z;-w3gcfOL*atL_w`U5;z^n>2lQ*AVbw38Ivjt{xwyLj3se)`psT$#JRVaPqECEQF= zg_~!w;y)#qWV;I3kJ7pgLpA70wExIkAh!NO+_;@UV>w?aA=b3V4BOqqAyGF__-5F? zhqZaX!Ou>0gFmm;UrX8`I}h#4l4=c51(`>ldquM-*o7v z5whOb_@14s*TkO-U4Wn%-a(QpjwJw)od9O0V~q8sj#-cOj7rOW;l%ygQ!5SW{OU*# zwWYQ{%SM!ztIO%rW)I_IJT*N>l59@N+U%v2L5W5Gf+35If?AgsZ z*dRwBRaHe^j;zs?%53vd4k>06aY37D=HzQ@X6eU2j9)O5IiSO7k5@m?AX3xDXp1SjO;$=4={ehf34TiS_8DgbB zJ1p`S{+VFoe*By7MEFz(?z7Vm^IfFiGsCbB2MTE*V1{cyV1RIPV5p~!&@+dEC`2ZB zoMEk7eS@vM4`{`dX=>5Gso{iP=8Jq3G+7Lvi1?}f-NtA_VoS zrmBpPV(l$>cmJ|z3Ze=^`vwA%fOhEI?Y7PBjn@xEU7rWxt~i43kFgRPmz|d}9Ac*I zQA`Ax;@O1?%D}4EVDpRC%pEy_uWbhKpfb1qPbwRB4Mg`;iJSXhyiq+JoaTb+U9b?I7}T+Dyb$z2RV+L3{nf-XY530K*|fq zz8H?6dZ^f=?Y+{&d-T_i8Bnw&N1t$~CEF5m;N?|`;S)XvpZb(>NoD>9C@Ho~npBY7 zUO4T8^AHZLVljLR6}(J>ad`AxDesM;LIfbg8RHKg8_AT+0RXgG=dO!ugYLyFsd(ZZ5o^Ia&uPgghh`@z)n$$Im`lJr~<|rcH(4GvPTuL z9$nxt)GGMaRGueA)7%Hv^WF@9UIQkrTmoi6#Fm|e8Nvuw7}tv`TIFD05G%r>=q^yF zZ*Lg<7EY)}YiEU~a`@q`D9&S+7mAoXFGD$xFgXehx}OR=>EE(hdEvDkRS7bbCw+;d z{7a_$zOn6bhLHpAPs(u9ltA#$ShSAF7eSlPVdoS!2y#9C<7<}CFc;um^K?4SIq45J1f&6DXkh_!5Hr@8QZMll@T!N~3`IsNqXu>d=2%Bm)A8O^WQ4dw}fIQb4#&YyZnoWfuzL+5R1XH+z7f@cJ*RN zbgk@^0#^YjPsXv{MCp!w32y^wL@2qr1v9Xo3Ejs;prM?TAK;8e>NG$>BFJD>!rOne zlYMz8{?VX%WRZ@+1t*Z#YSRTo=REPjpbq#DqYDyNp^%~{hk|*CpeDosQENsNyxZU1 z?oc4g{Ia{&$p+i&vx{mJMbIS9Vi>AZVpWjMmqm7P%W*#ikSK+=>y_v)2DdX<#MlGl z1dsqu<;p1H+AXmWdoXzTF@nVX*hFxG+xbWUG@Ae%XY_A-0DL%*>TIv4hh_iMf#rsj z9dzy1O#wFNwZ3ItpQm%y)C*nE<9@HVR4Q>N?pwfXch7@sA+S_tzez!Cboo9G&#DjP z9Pt>8)Y}n=020HuPyp_!0R;<4e*Y_Rs7V57?6rZ%8vRFu`34J;`1F3p1o)h64xzqAvM3T43qHT+=)TXo$t|ENnD)$TnIY ztwUC1UG4WJ?G&I&CT@}!t{XT4_Ad6(=3_2+Ulu5mpErfs*Nde3YS(f^=Qh6p-|MI0 z_ltpf`m61>WFT-gWTwp5@DI}2+CHL`$VU|f@Bm}H5gkn;z+PJ zMa#(%ivvTFeWvY|^ySW$!))km0BCT`iG92%ws6&EXn8Q)#Ma)vW1f080(wNI^!0xO zj*FS8$6ff_RUZF>2mNxkB-7Z7QAb~e%r4K!aMNGuzncOhck8^rorh6k?X1%!^hy~7Q}uY7#k!#RKbf1)lZvtPjMRsVzxfee&FED z$0^+fvfS5P-!-kyl8#OqpSsmA6JL~9?Od8Plv6em6Zl$)8X zH9C-Zhqqngi6*A4iT}VjDt9Nm4?-0&U zhlQ;jE({vlab2>XRqi`2>kis0GMJUMoHakPj=v$2^#FD@B!9ad@Id@S$ zu8CE-St%)BL++VKLgv<=iRFZl@S|(<(-S@=f@!GK3$;#Ll}S?Jhqgl3q^|c5Z<&Xs zQvp~}-CUi_NK~ zO0FTqZO5?pjFuWU#LNpQRnD3DFojFU@K^9Rxw^b;lY@*;Ra={bG54xgW7+pY(2~H) zhWA7Khb9u-_H33^qQMa#gy&kdPAHKwG<@1VK0em_ftO?@J-IPYd$YP}9z?yk?DB|! z@Yu9${?p#+4;(M|nuW3aBO)T=lgoAf62ABSPX7nBtO_g*&$@apwFM-{;*Cin{o-lV z(o*lu$?D1MrFmkWMf&jb4ggX+f2d?>enb$g*oYGAgz%8<7y638ZqA~uz40rgIK)G9 z5AVPd>vcZ-}(Q04GMSs|GWlmT|d1BQ~X11En(u^A_NG?p%NeZGmJ1GEm8l99lNER*#n&4-C2-u=y^SP`ZR*TNbk^I@1A2O?B&@i=7>+tFnpbwo|20}bXm5@A*(`UDKb0FAgHLE zX`};N4wzw>mVaDkHq=WQ>qmS0n-wgConTB^MSQkzi}LC7^m2MKV$kPbwa$ZksJOEI zBLtw~LRKdbnYJII4Y$eY^=jCdD6_|qvM;0UCQk6P(iH^S=ga3}e>_F@Gz7b)8iu4h z4WvMLkMqGLwpnugwq;bm%AuaT*y}y>f{^}>WNxtv1qQ1W3N4IBGS5&<2|kXk8Jcfu zG)mD8g5Ot5)03Jk{-2J64iXIaDnclE#)*rS0ka|QI^W3bWk{|mTKXmaKGCFO1olI& zOch2otnK3_=1tIs5+uO&Wnj}->cE3=l^rtTjMH>t)s&tFXJ@el8 zpAXAul+iJj6mZt6$BUN8|Pd-rm}9+M>} zrEBJ|!r(joHxwu>hjKi?1RML#B6;hXYmv5k2&p}I=B+l&X$uhy>fU)Ve0*i64veby zU)=|_%zmd0S!-H%d%`M>1RGmWl0Z8||I3UJG;kC%K1oAw=#`I6ivV8lZ))zF{wFne zf%mr;;d2tjfw9zVUNGkvVR+};f&K0HHFC7Jm{FI{zd$@u+C_NOjSI0!vR;Tfkm8F~EK8DFzlt8hr9Gpv_F{S@7ce9kITikB9EVWJk%Kcsg;m zmy3h@hi~2kGBCiF&|d!m#4M__08%B{LFH(_r?1oP& zOgt{yc;9J4b$h(szOx{}P5%hH$tOL|K~2=%fdkZki$UCHY1v1BijTkH=U4|b{zQ4F zKZe(0b4Sw%e|{#@I@dI_{GBy~NbvIy2E_kxDjenFk_W-QT*PLt{y%*R(@7_drTp;I z;H4(?wz28#W&6cA*rG;>2MJ-{qc(cj11*(Px&0L?4k?a~eMhmkJRWYvMH}}-L7oN2 zq#hXvjGE)cP{HuYLFyOqL%R+9@&C6}(+L#RqhNe?N8~!#0e>&9EN^AFrRso9DblNr zo4-U_lLM>p+^wM*VOIUOXW@Y&ESB7aR&|dGY{he>*$VZ#F-vTTg@v$(k8$)kQgXhZq6lTk+`viaZJ+2Tn+3WB=l=#62o7>jWkTIYL=}`nc z1(EC`Yz9Jrkd-<@)5RRg+EC!Z7`4Ny2EcU$2@#RB!^B|b6OW}>6ho^-y46CfWIzRF zAQ6$ZNsH1_s$^6SE8n>LUv-~OvmdT>P9{Js>^OX;JY6q;{y3a~JExSMu6oWbepoo% z??~C@g2t^hOf4^s*<=~rwbVq_Ab*O2H1c+5xGz=5N*xRMkv)R#xE2UsR5r4Nj~HX< zc_xqMbk6W9*%I7o#H@ztMfOR3{MiKe@94NoJ$tp8c}e-`rzaB$yQGp_#N3`b-@|IUDn;9@TBIzUv1gL` z?u=qxTM)okPKx0c47+JzT@wM3(Y@TGk%(kYx*dej?K{~QyA=HM?$+16J3seJ?>~6w zILaa~t1|gS8pBi|!l6u z`qE&Eg3SHhR#SB0Nt%xey!6ra1e3+KCGbePnV;$A>kc%q-%83TIdld~SR%J)u&>jd zv;yu)+)Q*cR!R&phHs^$x#|#cD8OkzWSjCj5<1>ya0;5O_oYhpjf*LKzZ{-u=u46M zf{~{I(K=;#;|43_x*USuQBF_-%h~%nkD_cj_EXjeCIuz+%l%P3FbacVwq=3Ka{i?w zW(~v!=FumL?Cg%EIxR#B!tPX z3wSAP~n$2Y<4xONmB|*G`&I<+K&K_FSLnP~s!=Ub(*W)f?Geq%$Yb zed)LQUN$K=y>hKF5BQO~`=&h|aFTx?B?NTig?&|AOgZ|MHj|&+^t=vC7NoTmzoK{x z)qR1VN{UF~*vdm#*!Qy)H)Bye?ytQ!*lMiCwi>gsZQG5L9ouGO+vvC3bKZN;x!=z`JA3WLx#k*k%rVin zDXvJ|@2j5&)%Q}F5`xq;4&KZ!1t5z1mb=*KiL8+p++W_dRdUCQ`U@RMPGz3wu%N;V zEEDmh0NnBffjq93s%Q!JmFTkUSxG`(&A&fZI&-o&PMNLgH8EE-5CB|gJz2hk&&Rpe zKs(?K=dVyhD2NSToD&xSP8J0$xzWrnuI|i<7InT{<8~C(Vf;^{fdl zj@rY9*P^bh!IhjpFLTWSkI<~Pj!#f7MJrvY9uF$phH2vSHqV%3{4WGSP2C_THn?Tg zR8GLKlQkIZw*xI#k{1`=o~?ZK2-oc@`f+yJdW0tEu~{Kwb}rGH1*;JDamnK&8K6%##c5Dxbf1NV3JjG?sYorq!LOT8>4{vv4bCOtrdf>J( zJ`<>l{r3@f>7ZHSPe9WU+v9>t&6mcBb2(2(Wh{ENFs>m_!bV@Dga#I^jjRyoz8?R! z1XNHY0xB9hB%G}6Jx6vwMLs`2+l@v3WQ^&_;I#YrqmFnp+C3Wo`=#dhiye>>R$VVq zc`62KPkl2KL!==483pF!!S441)9)K5Dv|j$8gkE(&B?>)Dhbd(_8j>Wmw6&u6&$T~ zX5=%l;@sn(;&8k4JIAJTE(8qYbGvQziP?wGU4Etux|ZVI@8@PdN@<*lDm{GL<%UP}DZHLsz-fBz9r%$SMATZFA06`HV>DEN4r`u%3czDL>g z>w0(S9#M&t~r9lgow z&MaA`6+6RlkH6gk{^Pm;=r_a7F7s%HzoY-xD*Wl{pQ%9nv@Y}D*lJV;{NukqA@*OB z1YsCodmOJD`2Rl}BLN!weft6BxPLeFzu#?%0ntuvxlA*TjO#(5RD6MDTL5Y?qLDAa z)+<59>61Y(1GO0Ew{%gFjhEEJrg3e2wt;fzdJk&MoBqQ@EI0EHVDU~hM|u;M73 zR8;blTr*;|MPBn9B)oMI0G{}^vF`tH0pfa^&`L?OGK|o>v17@d$ju@a`g)1fjbK$a zh<1)CVq#7OMSf(Vj+=;CKsZ-zDdHuH@$B1;af?LRg+(BAZM5GtS&K2vGq#24>SA(* zIGe{G^asWmm42|^v-_t7Oui|$XbeX8v8lZCYhR3HO*&BniHa#`!fo0T&h`GYBMn(b zCU&)DKjE}FwQ-^os(hPhK2U0!3oxQOG=sWQ`W#%~H`~o10wG)OD}Fb5HwNs-6-a}2 zFybVmVE|;Jg%&JLk|08c<4Y2dP-xI(Bo@qD$dwy`^9XZr9xO|boe=OqX&RqyNC zx@EB3)C>i!hqH$AdT+`JM@w9iGe8mIAbsFz@Z?b4?crfo=#a$kpzNkI7g=uUac4e$ zyps%XN9l5$kJVJ^{#NuJR=u(1(bdZI0B^t7T58)2*??TlmOX*`m{TKyTs~u7m=8OH zLR$X1Ue-5A16fm;176ppCbwLs)X2=^^n6p#B)ss9%#*~i;tPYg2ewi@#9=%SZN_qh z(V!$k8p^41d}SfedJL9ta?!TTLS@G{rU5iL*L;C zm^lB*8hE9QZv5R$%^Yo=$+NJG8ovttn^y#34|Xqb)~@-+BQ-eaQDBJuPbqq!%d4ez zfe9B_6xVsXVvxX6p*7z&R=HQ4_gUih$Bm}phu{K)Zq(w8yN!}tjP)tDveS#N>J3)2 z%bEoaAV(Q}MB#Ag3DF;okzV_dCOOC zyo!o>QHd^Y7fa9+6Yvk%1F!F&?@&N?ICKh1xeWF`Y+% zNhK4&_cnip3!F+kNg+j!xR1m^t#(t0*0OnDMU{S(FHQL!b0)ub04IjO%C-k)T~{o) z{cq?CAyaf#u4d~K_+~Zje}6&69tn9>#FpLKSH)>Rw-T<)HzJ?(Wm-R?RUVrwETgqlaM6@ z52D0I&ZzLK15Im+m!s!vDcNdvRw34RwNIUT-}8IejlkI=2-3{4f1WRGQaKZ*e>q2m z)y?o&HwplDSG_wWz|DIt3%zZit3%DF028v~pa7tE%fQr|j8|ToEU|GPt z2X_5CiBUiJloEluhhwO?($Jj zk>%Ti!fv_A`q1x?8^)BGBa+6cvMV0PU4=ngzUO0?=|`Ci!Cc1?2P#&ef)JqL*nJUZcDtx?y=Z_EiqdfrB%GY-<}75YdLonDkoi%}H&_8@5Yb z+rQpUcG{!{qc3=YijjVPQnB1DVmaf`L~>hwr*`qGoIZ~gR)uBAR5u>AYS04(U% z_$&a?h=aj?2DjX|RWS-U@>HSor}QCRW7Z!%AjrL5>L#eE1>X`rfhAyrm<+J!bunM%6~!72L%Z&&ojURbx{!+uRn zGCYkZM>h#3N;0JXrKy$l63T4A$`_dCgb_6&P&M|;J;(NOiBq)Kz~qxNjgwHt$zkkk zaRO|J`QsoO&(z^8I=rd`C*)WE9R%&!t5N(S+{{A!{Up62X-Lb@2Z2pT6uc*OM)<(H zpEE)TW-f(!lWB3oJzJxrqx@Cfs#O7VEXWTTvW#remKJ_NmX_jvZHq%qnfX`!xx0jq zM)C^#2b5r#%#SDRxVizKFR&Y)b+C3g^0!FJ-VRvRts6S@G+V<`^{ zBFwrYs7cv68FSAUl~5w}Y;pq-E>gPhqLm19X}Aveq2j5pjV2?pxzIfg33G0uCHvdd zBi?cEdpy3?>3mWCN)rk+yC&HJcTSZ?Q+`AAHW2FP%rIGSMbM9%2t|0Fgh0zAEg-8F zFUUPNyN*RT42f23x(D&f_7ZJ}%AKcxN-OO>cMPYz6!M9M=~`!2DjxiOg;jAlbrxG|ZGc)tI!QppcuB z1R0Q8KY|SDK^DKCoLDMTsrf}#;}n)PYb4{Ibc>BvbJ3XH40bT~IzlRh1)f6{bu5CL zEWl!X$K2Bb*0r^qwLUXQiO(k}nMj`3Ubn{_exT0OfR?%K?E{U2zvL+T5Fb~Br#Wh@ zI~?f7u?^w^eZ^KIP_-2&UYgtfmLG>XW|tWZY*C?kzD)|m(T6y+biKqU;KCPLK|soe zQuk9w&kTBNV!-=CL^fu-Oo3*!X_>?p2iy`xU6VXK^VfB}7!&SlGUd49+N{vkwaEfP ze_zZch`tbnt27v*WVNB3BnT|tCwD2>NbJzei+~>%xMl<3X?ukQXEM|wXx$!Wb=9fk zy$?{VhO$|G09WEUlds4VmhHddvOu^oNk{7!)Wbj^u&+B=apXj4hC+nA%DyTe!>aU| zP_TetGy9O|!S~>UOf9d1AY0EesP<>o<(2#BkLJTtmP6fUQR=$^%dtei-HgvQk`pMj zQw3&&ByY?xMXLA{6SnD^9zMy}44X=O3Pn1Otx`}dY*?()vc(vlhzAXrZxBZMd*I?8 z=&?n}64XbWTIQRjMOBLi(0uql_k7NLWDgb~i(rBA!kZ(JTvzoBk@* z7Aorv>&*e65_E;Q$v33&w9#eA8#Jh4@+^|6RsU*i3V5=AIsGXCck9_|T=mDHC-i?W z5&V7C*!^_;{HoKn_I}{+BGio6Z?X9qGhNQIjfs2W=Dpyl7Jja|lX@Z|-&9)YCb(%lz9*npI~rylL+Bqh7P_hna@Hxvd4P zpdjxzn0d_=f1leLIj+j5Vq^mowc;Fryg(x9BBU9=hBS)Kv|Jd&V4RJB`F1X7STG@| z8QXBGDw7bf;&*tVf2e3|#X);;7|0C!)7-E=cChqD6 zj%+7;#w(KF+r^H-Vj4azMOB{Wg1_3K8&P+vj@O2c$>;q1v9NM1X?$F~3MR>xdsea5 zcviz??-vKDC~1z!_;?L+!jpXi=mJXw!=C1PCB65{u!BV5NW0f{$byytWj518bjwX0mEMRK`%pptmmj&;i#W*N%0o3`_lxA@agS_V}PIb z9%G;-YD(So$H0%_>Td8hfA*hFO8XYV@VR^wGO!OU_&5gofghw^5G+?X+^mvR^&jr! zU8d+;avFQ$32_pBbA&p}Pg^D!yz4A$j;ukBrjWeaDTCWi*uNiggLb8JHx)4#Y0NOd z*y3!!zNeTQX{c-;R?WQtlnz{4~A%lCz2E5*Tw?k6x)Jlhi92 zgwh~^-A3!){@JZ;mbMdn?}PPKkc!Of^z-`8_Tnh>wkZ8@yjA%B=7kem#z zcD1Fq2J7?hxHhl#AHy6!m1-XYSgAYRxr`j~LgQl){1=tE>p|I$aX_}I;1Hj;!E#`w zS6q}Sx>i-=ky6D?Vnb2eyAUNgDjt3iW2(A&LszyJpysePf$td&eW&k?99z?&WDG1eA#3R#+9Wv#Zw&gHW`OLCM*92Fk6 z{4w;{a!9+~sJ|Ql7*9y?BUh4Oe*b|?qreTy@D(Uf+z%6;izkl|;O(JJQC|!&d%Vpm zG|CSP5aV+@@7d=?f!VY7D?Vf7F{PPU1ILu~yJyk*`-Z96Zf~-Q;BC@~n#Zc?AuP^m zRUpQGB5ZJdjZ{j6sUdURb1-Kz_1!U6<`$dcno@H=g9!w!%n+3Jhg zH=%q}Ofg<-u~7S6w?F`Cl&m4vDql=ljx;C&0!`Zy?vW*Mdk2ruJ9M7Z&*YSA13JJx z!0yJJ8T>nlZk1v#fi7H#j<+rnH9udBhqM?bM-rU~)7gjX4uIIE{ufHaM}fA-vjbPC z$8%I2BoI#<3^e~`GW-y^86zJu!U^YZx+mLW;9ej|K3u|0QG!DAQ6S{Mn4>)GXg6RWZ-8 z@zoLFxg?>vK^_@mXk}r?Zz);nbatl_S&P2OhEB_=R{aHZNXCQzk^z=t zdpj>yh4x_P+pr?{kg5k|jPBZ)xr;(z&ib`~oQuuFit*EWwQMlh7C#Wc9%Nx87{Xd= z8!Hd7+<<|Xv(HtaHJbX7#@&{n!hVKv8C_MDhydjs&$!Sm`o)fW;vn0Csv`~CWKhu+ zpP|h-EuF7X@%$@Z9sU|zUAYaD6cWeJ`t7JK)`zT;=Qq#kG^roX7l%Tt0R_R*NIK-g zi728xWaj9H22twB6VMT=y%JS>klYP0pAD~1oxW-+sSVWCsg58^>_ZBh1C|%T%0rpY zh^-y%`OFT}Hn1V+M3#3#XAIR~_^4)_5%eL6h)mr>?`2e&Y3t8Zl4?nN9Fl!Q?Q(44 zsWsF3z9ZEUkF|cEAo?|3^Jw`UOrht=oP10P&~E?K2+{3%V+gV~%4D57a)W7nOJgEc07L2Jed_9KgB?O&&%KOk^3 zMFk&j{Cqp>_&5>DNP1^qCWmTr+?Ec2#SX`B4u)#l>E^(i&Nr!++Bep#PL*cQs-|Y{ zW*5!lom*!KsImCER`GX+`qUtr68pTB}<2HmBn@eoHl7`y4~-I5?o)8 zh&;+PJVq28a%#79ix{;nopixoDUjQrLVIuO+4j_!xK>?Kz1^Rt$e$UM+&H)yn|^wM zhAu=~UOxYT+=|yXXlWrvchfMZUQR5BK=*BbZ13DTc_3T|>SN~+pShu#3&+IR=?*HDl${MiLFj*fDONy+IiVnAa1_zX>>zxvHA!m z*14z2`^S={e^&xtgnw~a3FPnBUJNaB;PUHya#92%2l+n_KyRNFR;6UvdVKMOUjwJz4x6cDW?lzx!%D)yYD{x2Z~Nf6;XT4aeMfj&J-or*4QR3S+TRMlY`oq%B7p`W!Byv zVY%2sfE63xv5LdpQXX?PL;RT#dkpYR{*Pyk7(*rX*KMs(#0H-jN)7D+y zS93MPHwQOg3Fi;hm;>yi{>+P0fD5j;`ypma#pqTSWpry$SFt>*{f^jI`*Pb?wH|6;20xVcHh3K$k?%x( zbp$pQDhJwG_2AuUix)Lz6pi|fnJD^UiAyR{g>@>GpHDpW3KRoT2?$wfMjaAeMLy}B9yDrm>a0FD*>2MZi)LlyO(F2a3y3FZh z;k$A8!Rgd$^E*3Rbm-UsYKG%2D5=&E89E!E-eH{yadSLqXK-hneTd{6N7i}kg%i!V z)=42=BR>7zWX90eOml^jIl3F~Pd^cf9$h&#%ua^~Zyr*vEsqY*_aWn~%{-hgZ&Rkc zGRE#(WQ;9NZ{1-cW}TsB69as_>xI~Jvh_Er@2#v%laH`EulB$MX09o&Je=Z+&D8bv z(}LP7*!FURBdad&P0>9)QNl0gA~VjT8$T$<*SF5QKmvra3A$x zloXw{&vjW3_p9{$K7L}8JN1L?W>DJ8nqAw6K3{d1M0+0=4|hK4Z)Kovg5SNp=(Vrp zlFxx@-})bIt-q`p(zgip^86U&mc8lvbJ_Gvkb)4F;N#aI9qqGiETZ#aqwH*@^6FqM8 z+THBB)uGIzj)gqYDEjF8-$2zcQ!O{<=6&x&QtqY_W4o$c(Aoq5GVPD$^o^@4QbfgB zqXgYsGMMu5CYt=?#wBX-a0FR+k0*?zk{Hq@qc`;zkF>T;g;Pwmdd<^X@UdR7ycF2{ zEkD*nq4i=yO8m+lolE8OCh~U#uIYdB;*5po3nhsQe1#jRm<%zeu|9-hj;EiVU~T8{ zx#6I}c0h!RIpDj#iw31rpD&QtE4@cZPJu+P@l|Y^&RU@4A2KV4=CF;J>$BxHCCH1u z+2slipfG&1(B|;e28f!=m>RP8Ihnt7pZbJTn#ufp|Ihz#NS>Bn3TewK8H9JA? zw=36IfxEa3kKVR~Og+sF@H*F)Ma;v)T&r{0C|{3ufvsF~%%$exjq@M-TC+{f=IDma zCBn}2+4?~+*%qF~a|DlcqH?CH-)WvF?~L?cdb!6*s>-S2<+&B!JYAAcue$gosk>JI7rN4yByN7^n)4gx#x9JzuBP?oXbHrP~0jVrWz(ZU+Z|?Jz zeM*atrLf~H0snh_J1pCDWCl?Pd;Wa`Z)X6)xZ&ces}xfBYUZTf$smm;6eEW$j`3fc zpYX+C8RV%$!w*I{Rd2;VCUwf58_uxoKAvQf-oECR)N0M0VQ8UQW?^%XbcA{9h4iI< zja8BN^Dd+NQ0%mCUT3gE{Y%@7XJpe@iNwnoyy8dwBlu=Ka&bztgg1-a3EEFXM}@+G zrcP7CJ!3Op4*j^~)-m2-oj zQ7a2uzo!$Tzmi{FGhh!D^sOnu`cwu#@89FjZ%JAvig-W2p`0BN7$Ji7FX=+9W!M<% z0TVt+Fe$l`UGWvVeMvSuZ2b0K2Kd(SO5R2G&NNf+d4F!z;jniUyaccDPOuQw$s6d1LiDYk6qPRaEm6S;a zBBbj%8_YpQ7^yO;?O?k1qi+{|slylhT#*mrS5|F|&TsyG*K)1s3vH^ua;U_Ng0x!FKKf5sbwKKys@mo zFZ9ZRdaXT=bkpFzypSJtt^^F)+2q^|xUgGteWT@K-BQ1zl%L4Iq<7H3Wo2ZdBuSi# zOus8$(xvQrw59!UMCEUDY!!`+m{%i64(wv13oo#UE9P!Z8TF=eV_0lxb%AjtkfnM)`d>4 z0nowG9AgGLBmQ{eARRg>i_N>W?|S|acp>8rTRy+vk&t~mUf&E_o1%!XI+saZha*NW zPv8aLUQ#$G^6=Tc>zYUn)1aRN$@KNfYpvgzJY)zlMwgZMpqL(ZjLnXv_m+}rLPkiK zA7CMqC9a=$Lzgn>M4k_xMb&O^M)D%#A5e&?Y#aXJ0#M)NOTQ#{Q6nY`DBmw*Jrg;L zkxj5^mSAevK4sQuKe?8ktnz(>Xw_2A$$o#+)qlF* zeN?s5tdR%%Ts$!llXDPpx=7}df+3|(mlnPaOl`{afiVtjYXPd@-uEZMN}{L=+= zE4rh{ZyQTIYNWTt+h@fJuybPTAFzXeuoD`+1hn+B0-vsizgS~mk`wrCJiCb|X`XD9 zT%IP$CPV5QmsPyqI1Ue=jj2AheNpbAC0&c^ znT}diw+pRarWudgo}AnpK{Dj7_V(RJNg1EkTnKfUiI z<3oY+^^qt%K?vSL*gwE#GURrEZyc0nDC}Ki$Nnw)!Fezl;cg2ZcGT�VDt^;&NUJ z#W*YOh2h|$>Lmua1-}_03OII%{4oH?-|Y~g3$){dS=|_GnxUm}g$c-6 z`5MriGg8JdstV1n1;Dwk&^6lSs3(WUbp#oSA5&}0u{Ms!>TWnX&M{jvI7pDw`y{nL zI`;R|Rl1BB7g$vgE--STsLY5ALKp0YUf(RoIO;8UG5)NC&KMB@TML>sn7=U#Wl>|E zW`b<~fLOQ)D%W$H^*j-~HjGF}@g!UiRDc;wLg?1sTsN}WU~WQhrX%!?I#gm_PEWOh zlvc;th=a4oXZj|L>szu#3L(n0V#oo&CjSm@%j&C=;!$j}rOm;H^3di!=6%Cob z#VV9>etk=N2WphxS3X{#p_z%PkAi&wr|Gq#*~tHcY>ZE5C#>#e4m0w&I;m%w8XKy5 z^A&0$aGB-3kvfvTJ~Iq{uK`ScdkGefP7uxC^W6wn!h=%-Mcl$kf^^QNl3eD_7o9wd zrGa%in8<{eB62c3!O+dN_6wgNj$vh%bn-|4Rq%w02fo`rtx%LLL_R4eV$bloOP$&; z+~h3*Kv|S5ni;09PGDfs-a5ybQYdd-1lo_)Lvv1^ANn>pxIC5Dck;|kd2;qgqhMgZ z%miV~uxTJh>2K+_zSt@d#^BPqf)oy`9-wOQLxIkW;Nsa9l)lRg0`7IM#Vg!{C10!o zbU+xAV@R!{r}1TS`Ivv!i6*s__8 zUOr|p$!ftY^G$;LcpMzxzPokh)koJL)ESXOxX!vScwd$~QmRTTmpx*=mDHri6=4@0 zUW)>GiAwQl5b-bD`+@rJSGg%jlThDP+U-Sd=gEhtcerR$zwyxdKUn{MZ}1Z2Z(s7C z*Q{+*(J}@{6vrV}Eb>2e5iA{HkWh5iVn3(&U0rn&RTW83USEYZ|0`ipI;O++I2+b0 zgaxRG>aoYeR0^Rf1s&OI8uvj_tGg1vLVB&+9lwy~M&c|)L#5x}pJ{S*yUEmS#+(e> zYKyw9WLJ1O4~f1tLz;*PFELV^_*|hs%=Do zd5Vu$r&wmpzjc@#MSJruF#cVY>{hRgLW=RwjHNYNCZuU^>s4HdRVBfr_zT&Tm)_)N zi^QcMdzq!qejaBle#VZe9Lk_-FZe?2xmR_ug9%(0spCdsMp48ftARxG*}zqHk!6xi zPYo#hmaDOp)~Uuxx-H|+JVc~)Wdcde5k8NT2SiC5LBK5#tuUnNFsD|vE&rMX9-Y#c zSKYLP7Nk>Z;L;w-c{cFp9K`k%qGAik8kJJy=-bgxDa^6+^TScuo%=kuwftmX%RL@E zd(WCWpZmgd*xJPEx2&gIsS?{j=ql*@-q8*V(es>Y68OGoK>@Uuq3T9l)j`fx_rmGq zjS3Oi^mQ($M`)YL&23)0E%Cp84+Zsz(P&orwc|Zv9h%|h*5qE_%2?^--e3==^`N|q zJlcwew%xY{3E z`w?tGoqGXLH*b^FKn%d)L2}w27jz_*er1OlR(DG%)DQA?G`&5y;)(^220KfHDya(~ zlt+URIvH4)NH*DQLTuPuEjw)axe!sHwj3MTb3_5JzA(96f; z)2c{N@KXOMXQU_!B$Ua)GNd*Eb51Vc&sO1!fjjEC zzRKkNRxQ;jHpto#=;IIk@a)We;RYC$^y5{4o8DXJP=wjqE zH3QDZP*j;;1O_)@LAOl`_g!IE?aL1~zx*+iD#wtNz)Qy94d!hJ$&VS{_-&u>M^N|k zgq`p2*R32%isGNFpi?=5zc(&@-TsnB+p6tsZS^cMG`Uby(Lp^>)dF~pUQ?tkKt{PY z!EH+d@`AZHwZmsgfEb4*sEmw^-@S&P%gLT`*467}H<8t|ZIS)&KSWSqVec5ug5yAQ z+RdoXhIFw##GjV!*~hkX33raz$WGvaSXs1(`_3WqMEUq_eo&_ulVd_*o;I_QGIY02s z~|A$T5iVdW$>cLCW|HbY8;@d)?hZO%Nl`dB)D_0R$ zOzPYPA212YOE})zQL{UdhbRTy;MMDiB`IfWQW-M@^^ zOcb|)5&oa=^Vl3*0*~VCJA9^MxWMXQD)3{G5j#{%_k7Xc#(}QKGmKAxB_YY!N`5E? z{LMd|kJV*kEH%N8-c(DOP3E+D@R1*iP=0oYQl>Wy6}a~p1t}8$bNPh6qp0d8v{C;L zS5kH=3W`L6bv-+1mt?N2D3>N0D>%6+yY1oD(t!=pC3Au9iC05v*!j>hUMM0S8Wk~GM~w&_H`u*VDqFEuR7!fY@0GMF3N>>L z9oFE_QpoS}QB6AezYdAjDsbf$D@%1?)F`VL85IInk$xIKc&kdkr(+y2aL$p=x6;VN zweKgo1mWJnUPwXe9~6jI6viQZlPp$BoU5;1JfMK7ET{YNPK|#&F?=el{5p(?moucs z`tZo5mULv6-gpm}y{t)t?fww3X7y!Wbz3M>>(gER;b*-C&M;r6d`oeg_hXSMGRXw( zXLluzz-BaO3_MW4sNSZuCIw}z6Ta%tMMT%}Cp;PS45=vpbAXzgQe( z+ZK`QWcN86+(@ylKKH<>`CzNnqYcK{h1Z!rL2y@gVL0k1T|yRFLpk6>B&T}0nRGVO zaxb`~lukG5$W<7v!Ip^+ErbXxkOltsHHqDbb2TE`qE)CbChh?@;bab=o)%9=z^g~6 z7>3+<9`LV1Z{9KG zf2aLR!vndG*=ha?>1NtAIWinw(hzRb97@l+bmutpAmN^$Bifrws1z5f+MMi7{hb8v zSOFkNq@s;1kTVFsO}{pCgG@L}^{bawW|_R`ZQIv*>$vA$b3qo_3Pb)%h^Zti_a zf0$$aB5~pIQZhCnd#g0EjIQW#@sa1T`wTfw`&q!6etQ0Fb?u^asX~#eQ?q4%?BX5A z($gd3Z3C7lhkm*<9~gv^n{%?V>-OHz$tl5$>{NGS>mkZ{bd?NUB5IqsC<>)LX52<9 zYzw9$f{IuRkepsh|L{8RcDqz?TU~N{1Pt=YG1>embrM5a!Vo8vGxjjAeR<(_ZZvvB zw@xYDKeJQ_*RKg^Wth!rNyg1B;N^!1EfI22y**#WSIXGDt@4xJ82asD4w1O?F?m=+ z^cf59?#)v}4(bUA5P0s-ccV`48x!(9bI;z6Jz|;R^~V;L940R|zdE~pfT#@>O6h1mP*dQ!mWFsp8Nfo7JB1BHS{67+~xFe(}3m=xiL$-Axz;%(M~4rp<8FR zF|X?KM(UQ&7y)91ONGG~9HP}D7i1;Sne>1t3%b~glsz)RAMDZ@bLFJ2m(t~Y@5^?& zW=b6jGSTCzSu}X+A%(7)=*b{xf|(aQ{@?zgsI_c;DjCt+C_!!#ls-@4dTKX?9x8vQ zncK=y*G;0ji75%-ATKJJ&%JS$VfjffjU*K86=`N=`fW=0kcI0b zqHWyuJ~9qH4$=6EGet;OYrus=iA_!-%&1p>ygD6`Z}^saS&Kd3%_!YM8@t#X#ySnM z`u!;S>sevfEa!-CGHMMAgLdi?G*JaGwPV`|CeeRz%WK6o#$Wht93#Z2FtQ3&rdVT7K-YH`vfm+}l0R5^{gzx0 zdia-3p{$40iCvv<-#0G1c~$rD>uOF_J1)4pp0*vDYU6O<@=Ol9D|JzkxXBk6p`8*! zBN6r1-*?0XI|k2g5{6G1B?u%T$j0a;`7R?$k>S7HZGhdEtP$>uXLJ4so$IdRW2Clx zxAfP>5Yxe;UdBdTG75⪼X&8kqvgX#c6gWH7LagK37*&>YEz0eVj{futyww9X;@J zu1*4}Sl)?V!SvtRxo5$5ssc6IPXLN}XVJ1P&+z>CW&dPUpL2;VE!f$(<0j~4_#5vN zqSaYM3Qzf)3}ZDn%RKCEhLT#)}p$-czc~ZtYGSLHES4unU$S$)qf6z6I7%5jo2t1`#2PKo59DgMg z);z?ymZ5!|IWJ%))^VV4-9H`}(1QR5L$MQ?^Jbi+8ALc=#88Fru0p@@KhGYDX;Nh8 zu-Dj&1fh`o`)2pkEa*jO4H$kMlL^QjW{Uw89KVl-qR-?rd`>dMLtTbe1H}tWjyOEQ z1%%|8X?y+}={#O{rHRM5ILRV?{avN26^#^-57T@O@pCbm7^&mPp!qk~>@HlZgBz7> zuh?`=iFBf>wii4F!tNtFhrS>db{R^#D{`^ATP_KbER}|}3`sHha*90dYFXJ$J|S%} z&Tua@-Yp-jE9CnyzS3JTt3H*yx$jTt?WNV&J!47fNkxDfB`~Z_YhkSWAD2HD)TiZ8 z^B0FB?N3mYN|^1o<5qBYyX^32(gf+SCOfBz|M#IUYg4?Q*C1lp zMa4FmRvT^~`!5+Nxm>N8r2a1#Nc4*lokp2e+t0ne9V&Z>(|wZ}Ad+=nEqSiUwQ#syP3km{wV9yNzjOpP_<3FtfehAjvGA2WV_#0$T-aSSDy4$4m8zym}+bX%W*p z58eBhGFwZ-}fxnGU9-@^F0o`Nm}!}l`?$9gO%TwwGC ztZQ5#f^JD3JP&`w-jcMLSM~g4)}1T5^zw79@@XCy!>44qX^a<+H7+sOYmE07b+Vk@ z5H%C%_MKI7;Mz1k1mmMr%)D6PCURX&IoyE)`*dgJ;Zx$`D(xd6{Tqc^e`JMgejGMa z*Rh(p)L`!^%W${WDIGSmNyIG%*RsM$QuE1!smiujntuZbo-W3@!hT@eat;2k>;7kJ z`e#TGe|D!(-d(p*P|2R|>QiYP{V{>|ZetiRmjtAjj49%|Y6#?EcKA3!XUg@KD<&AR z@h5$j2NKNvI1j^(Wd7qk{{d-&QK)yh4k14JMgak_LW2IZ;mRMwK4Fw;7}bExy^_#+ zAPM4spJAEdLJ73n_OQzb$%n5<%J@nWYRvI-8=-Du0j{brc&X|hof2qZs9c=^7)jr9 z%(C|PXt1?YpLqYv8EoYTf5_Q-Kjwyz;b}nYK*JH{Px7O9qn( ztZP`*5P1VEu7&puY^55&_YnR|?mznS&%AJP04X3Ima3`9|MANC=M+a%0AI>=?pKum zxgG!grp!zSW>=O-1jGE-GyY%mv#kkyEfYpU*#Vgo%v#{v*!5+-J++y{R`RJojtTw5 z&ra%l>c-IlH~E$AN-SE#O4`7lF~;}wyBL}N!iyMgfS)_Dsj54-VY*>4)B^SfUk1eO zg&PxTQw)fG)I;|Stz;dV2zKX9Yc5O4yO=#FPRc}4lj*AFoFl>VIr&lDqd%Zl4CDj5 zhOGmdun?{mPBRj-h45fFC!=Fg>R=p+4FODD-JXTn#y=<`8-xXRyVVi2p`x8gaA{|E z^WtEXUQP+B#+nzu!DD>9Aq1|LT~47dZhP*S7hE3X)k z#$`NxUA+FXD0Ju=K(1Bxf}gp1#kBr3ZTSa0;Bcr!0`9;uDxr?1ZDt$jZPJSNW>88> zmsm=ziW5G^##J_q+3S4Ehs-usNpMabE<@gMf?#v(URkMR)_b|8eaOLiia`R6#q#~x zK})4L_jED70WPK6*$7%R4K(i3fP|A%6#uw!IcR^o5xy|mlQuE%qH;@<8x#rybl?JV zCiGlqX`*;-r>)*pABqiJ%}ti7?$DW(f!Ga}mB7*iPLwvvO!A7p@EynHE+w#2jq{lo8 zS3!MBP$>qO{5^2eR6X+u)MTO`&&L7_sk(~fw2}Iq?%q@kZpt}a#eGXyL$MyM@%$T>R zUCooZ5>ERGnKCj$$lqWR1fK5wI6!rDkBLzxTfjNxr_{hIj}HTD zeMl{$BFCC+T>7qXssEObSCr|=GS)Ev@zIWj#33`z_j4iqAsaj@=UDl$o_4M$@_{A- z6CJ`+5*|1Q>w2I-lB`z_e=|PJIxHv!{01+I76m)Q=+%tMpkd$m-o2LGZz zp1JyIWd(v)wvRcYvk1~b_8t(>W3l!(quU)sjW6Y~}nr^O1t@5jVPlx4h5eyTYT0rLPBnRLIbr?QS68 zGnv1REXX0V-8EpSn9LtfW_zO`rY23X{n(gVdjA8FJHH4jY2C+ z->N6C05yIdY#mw*6W8bPwZq|iJ6uLTMGRJ6wv4eT!d;awr5o$|FS?gw_ricVPHou~ba!wJIr>8tx4t4)&)dxt^>+qSv*NFnR75KCq zQnxAn02frF78g>BAU?Joh#?CM^VU3+{Y;|L;YD7=>IGO+H)>GEJ@qXu=x2Ao_A^x5 z*ouZsylxx1%04BpNHWWeWQ!G>qvWhA7=pB08~ouyV4`ugp#d{3eeQ022AFE>;uw!d z96Ntb=1nh!CdHNUxj&fY=xX|iuv*tlF6l%NOgeD=l3gl9+NIcNP&-mWiqCCHm`;o6 z?f-L(;QxpRL>-1z0b9<^0jXP-`gRVO^P_?@X5HLUv|dh^R6@sI5_I)M`Mdya3@j}5 zM{P!Jn51^doa;Sdm@nA++t~5U4r)vWJ(tgODk{(xhf~;_P^tGD!m7mnzv`~~FUsa? z1A>C2gwi6QG)p&vAYFni-LQ0bE+8e{NSA=Hgp_nicSyG&A&o2{{oeS9Pkg@rz{_vL zx$k?=%$&K-HF3^VTDm3N)sFF0dVE(CK`g_kEo`mk?nc|03HMOfJ!c$TJ-E8c5A4X` zrNF6ms?gf#mK;1`bcZh)6upsvkxiqHO^|wCMR^yFIc%|lYdCMu0%-f_Jy}}<<;WK> zZGD_x@M=bF%ygD2>~%=-6&Q-oz)H?Qn!R6<&G@C|0GARIbbpZzZzh`XS`28sHQeuE z67i6ElT}~M*~qbALENHNYPY^$X_COI1OcOnt4R!TuUunu-i+R9H8z<92w2f6x$s@hwtV<#z#7i%{cNUg?&=iczDJmjmL#jAmnOICv2 zFoVlu;P$ecbS3P@tt&Uf1?R@=h>}xt0jul2H@j3G*G&rnCI@X9R~s3F17zOxy(`?f znm1}w_%ilAhhDg{=kE(#e~T1a2&>X#06>C=0XPCYB#klpFBS_x;LS^>NHwEFrp$zZ zqQ4p8UxJ&0$xYv>2kX1vJoz6fp&Ve9BU&sE{|Db3lm=uKxNV0w)Zw4Z87Q#a@XhzN zdW^vLZ^b1s!UP+0`0syq2^w+`RG={M$y3exWbBvw= zr-3t>IR_~Y5jfvSvJLz)%mbPd2!7}GAg3>r6_nGmH-FaN$MG(W(M%GM&3sOhU#x+tyvJol~%>sgJYoZJjvxsDYACI zGWD1hd{E!T_*XrYaTdqu#^f31x`d?F;%X0%=%+Dn;N5Xz<3mC8PbPAy)ulQx=7~di z%5g&IMdYrgBZ#JtJ%>h4=e#rXt;H1^);@T6@E1B9zOnW>ONM`@+U)dU_6<3n#@<`+fXW-)uLqWjC>LZH%4=j2L zxCnbxzGXLgwx)zA@&BGW)DR%^un=!3Ty|0?0qfAB%Lmkh{1|Vir!9SWH6-xM3j2^G z8rXPWd?{kt>-T*VN=ucfxZN$#*4@VZ$<34=Np(GTfP#O9y>d~83&`4#DEE5NM>?1$ zpMu28t?*2oDJ)+@Q8l4zn+f|1-(wKWII4=i*g#ZIZFUKIl4Lp3#P>@dYJ(VH;MS<$xI^_J zJk4MGy1y(;meF_S2WMn`rxEc}6wsTKqHbzC3&F>gm>B(cG?a{33}Nj`_zcF%++p9k z*?*2TQf=us9z-JXN6^3Sia_qewuB59y-9@~^W5xgh^l@Nm^VxDpcytyG+rY4R~Wyu z#(%^3`~DNg9|Q)%_|xPp*-euSk}TY7O+E5#)e_rrn;#P)vp-LD{+axMz-_9Idc#$n zH{3}ohFlw!PMrJjiavyq!7QhN8)LpVM!~X(*w6s_IqiRza2GyFtxxhzlcd@V67BK4Hr9(-TGN;exRqX z^gCMJNCjNB-fWmqg(O2Y_%ViBDa3Yf{#D%**PQfA=>vUO%tcR9!dlb5FjpcA6BTGG zW5mTXwj1pWYCLsQ)=WTN30YkEyPr&E)KUiMOA$NI>KGImrxG=CG@Nr>MNIh$Oxvu* zML)gHCfbI*&tuRZbmTd5>WYhAr4C!(pwh6PFW0{W@}Qsri>tx5Wu`y{OA>{_55=a> z4k=Q8+A(sHI&@I#JKYRa-wGUr#mUZ5R17X{ed)xaoR``LEvHJjy%f@P`(aFP9~JAx zYC~Zh@>s;M`*eISls~^b(rX%B*@DDY=94EkSmpw;+BsrknV)^CJAYXD#eWpVOCGraCuPL<}Ade^nArZBXk5>bzn3xS-kG3qN@zpRT^G)h{1OG2P-gW7R8|IM|kB3C|oePFFH zNC-NKE;Q#cZJFTKRRjTdDfXE}>e}X694D|vq_O%r)O3;KhrKm;Lo`rk1DZ7ZeO{t$pDj;W@uV5k+F^2RRTS18(7>HuV zRY;BW@9A*I!sq&6R*g_p?@)zeDS&kW>A5g7S>PBFPM$!a!LiXuxD&KUwb`T9HnEm1t#bi-`8y3l=p(l&&_t+`TK<&w9_lxUNRt0^*WM~Ylg~xegyf%1N}5Z0cb-`xa>$ZQ{;7esB1&kMTo)vyV{% zESNM%4DXOjK|NQ`OA2O62E~m{bf-V_^~eiRiSoG5uj%dfHTQbmM1;PC9~`I~VjAAe z?0~e*nWlf#Y}uju6-)W%E$)1iI*VdVjqU}-hnMvpq<8;EDCIYW?9T2wuJi4(g5>xD z+pm)wcjJdM#rZSSmDSdt7ZSsqXjnKtv%ai)E68oU#<#^nH&UwFz+aW8#6`L;+FeW1pfJY!xCX;2QdsOY>R->>7gnj|7~Arn<8a7)UK=!A z{HUjr_y9jVH7)5^l<3`iVIm7u=)Se<-Br3OtszlCO^aDb*X7x;*4jiMyIk>;eS%+S zynp-qsFt5NKN}k7d_W`9pQ`BQ2D!q{@p&WwEKYX=zN@QQNpSXAg zK&KKvYy{BkXxu~vvI&XaPfOR-m_03R#NeGg;p-9FE&>g?1Ze9lnnBJ>Yo{_zR9%`K zLoaJ;k&lYDJ$J!}Ox`;Sdp4t8H0atM)hD?M8h1CuJI4 z$z8us@m9|cN@PACP_a&r%}(X}gW6|{r3+r0d4mD~gZs@=3|~+vWJ?QkPnNkpq-}qR z)8dqf_apI@ng8o;nWv@K7rP2W1=Hh`)759a99rr$VTQ_oeFO;(g;S#za z0q4f&GH`kU;{6|z3Y?SzWJ>Ayy}>uKpVzbX-upb^{+M6Cr+cJ7jhA{_f}3VXP{LLq zph~3L8LtS@nyx`aYL0P!T1>#p zu)`mKr$=ZEEnM?5CZs?>L={Fr!tp^suF^(8bC~sha)|w__(#$Lejwc1K;MjihW6*# zUqZcElqiXuxg-WIpz@3{n0>Qt~@R`OYm-x4_st*Sj2xF zwkS_1S#yH0n>d!wmSval>HD6tM^k~bOv1QB|25gr5(?r`$T%g%y zAB2%lpoK%Xs-bE4?ET)?m);*Alcm~`FpDA}DIw@Ge9k~bzSZBSg%vRknYFf#9%h}K z_v~C(jM2inLdoA)hi>I5{~wNew%odU7H5 z>qEKB1yKLApVrIF!Y?gp%qJX0-}(d}S-fzH9nMJwq3pmIQx~53J^Eq5T67`M$oMfP zUN=UWQ>ajk!%TdXX}a5>z${jxB)1O?8P9`D>%sv z@~LRiG|HeP(Kk!-)05u^Su}b48@bR$vjn9LK4iFSF*CUwfm8|RmrRTye#1U7xkXdT z;6lQ{GV4nlhE|HvH$0_-eSYpZJ}rufMKK+^$hV5)OhJ7qBcSGc4U9aTXM;0U#S@+S zr0c$OhkrQ27Hxt z4QG6gmg2%LTJ?TeTnvp(J^zr~HpxMZm8OQ_f(N%%)qvVzioADEcF(AlfpbDfHV3tR z;45R8 z*8rPIu69%!J#AyF z<+|S6{A~Nq6)44bpL8a!i>=qQ{YZX+o2P=0434d2)9p=jQ^B9tNcY!LT-ttZwMG&o z)Klzn-0a>CZ`X{K?cpZK= zyx+f4R6ZS&l_`NnEDkK zO94xx<}Z^!A;lIZ0Mugk@edZd`v(wgAt(ttG+MkH&uI3#{bcKhmd99E``>$lVrLW} z^o{-YIW+gMy%U)i(u9<((KsS|t+ac$8M(cfHV3!(YqGo3pek7Mc{)x4&z6whx#-oc;M)@$H#){frA{>?#_*S80=)1*WPp;Sa=s!b_mAxkZWHLioT|J{B!R=RtA))ZQWd=XysM-8xhF zZbK1Dn#nj=!L*N&0e@Eato(z1Pc?TaY!~06_O*6&D7JT4#@yL*kG-gZjn{((s1Ctr zE$4AVTP!3THC*A=_KZ7!dUVR&cj>Z&r*CVR<5`;3K?CVy*9h5EnA{3OlADI(DD7lk zlbhOUvr19MC5cW-YFvq=E2|Fj2zsPuHKVAMe?GzKViQL>37o@}cct@$ZKY~=8?&$E zEhHzWfA5=i%DopbS0Z?7m0Yo2ccO&7?&5NQ^I3uGevT~%e_FG6|{}U8T6MKm8O$o{(kM z<*tQhZki{^%gwp5c_k+2W?$Ae((!2+3EBAt;By~AAz=@G`wU+Os{N&%P#T-a!{g5P z*Jtg+jmor)VU&x}=z31EKR?EHvp?hHr0HR&GLrh*k!C)%AoBn-gNpS|y7F_w-N`iP zSl8$aa>@zc(4GyncSNP_PUfhn#5viX-4sL(w(V>+so4L9fX zUNKW#dHYizc1_#FUJ*QBInPpDhpAaNw;8_u!My8;Mj$uD672M)!uji|f5MPcXlCcwBu2i%W-9H!OLBTqX+R6w&N&Q(D8ddSAUf7-J&k_ zKAG*V9)n7zU3?FeJaq|j9>J$%sfHvbZz#dE#y?7u3wWln%8X+4Oy@l3?XNt4fv;hn z)-_`IK`CTz?_(Nm_*>76_};RQCdA|*DOH;El++p&8j8|Tpvmih78>^7*37cl$Gon| zWT<}qUK2!9;XUmK`!!pB_u?52okLsxGsj(>F#P$_ekl!*;c;I5xnK9iQs~ok3&*u5 zq5d=~6kmJqCzhCm zKaa~!4$EZ1%}A`v+bu|bGU>#eFpL$dN~V+7lv_{-Iy=*j4=9-jA?>Kn;es9xU}U3W zjAOj)p(A=2Ok6coSi*$oo&S-@KkrET(<-?9vRCJHSmW6IfNn* zYU#FZ5N>|$3F_bMVH zhAQmGOEHC*0y)dy#uT~&Kn%E6*$dX+{F?&7cjw`E_5lvic<`1$4*`udT-TmF(aHWh zvVpywHr1>VRv6w!=Hp zJSP1D95-SOrvsm#qOh@fo~DmRh$$Gwc#QpV^@`gAZC{I`Q;Bs;7^!uV@ZLc;N|<3~ zX1!?LZi;e!q?}+Uwr=_mDN@4?>kAMvMX5MKq0wZ^SB}5O?0sJqdEE$`se~sUqbi?_ zg+gLn5ArC=R%QIK*>tG=X{8?%b?p(uCj}~L;FjP~^%M0Lzn*Lb1CHf~GVtUtO{`)> z*CQHPsh)A4!YORCGjze7py`#w-ftcvaUnD*I=X@P#?xY-HicD&ESC;T8J~=s#dy~c zt)yyPa?VbWY;C#pzEa$H*$!19ze?=bjrw6}K*xr`u2Y*;@LdNt_aP=B6D22BY#=p* zZe(TK_9Ji8JvGJf9>a(XT~cnj!Z40+lBVO1azrPO;_%d+y+mt}5K$S?DfE~30CN2s zX)I%fC%FbJv5Je)2fwFab6=BTu*y1Q_p5If!TNm0s=`x&STboyM$M=zU18Gmp^Mzj zI{mOtaHATGue{sn^%HfecT46*gOTG`YRtld*x~Zs0)st*>gUX#E*s3%m6c8bB$@mTK99Z0WJ6AZQk_@ zn`|0F*9Y>e+ccbc)hE5>IsiD)} zOZUi1)*S64we5!C*S74)&hh~CuS4xI2ygwP)x)Xg>(qM{!D6-7;dDY1JbN}owJYjY zUU)xht^?m^E0@829+M3kD3uI&9tcB2{W=N^H@#AO3oez{s+&tlQ%D=^PTU5!spdC= z1o%hBA@Cr9d>P)ox{IcjFJ$%~G7Q7zL9iL37iAfi@$7XK0{n9m-BPpB%%DtQoMzDb z_@5To*Ze(2)_UV(J{95FY!9J>A9AlA!yV6v^TNUg%6My5XZe29yo#aJvD02-&{ocb zi9qd|%Sh8j%euhu5wE*UQdkFjUZjKJf}&v&N5_VCATx_HK_oNpX}?z98^$~Q`$z&~D6lb+&E zt`>itTFi`*-Bs3Jp+I9|U8)l_aXuE$se(Z}df%XkFvH3gW|s1vJ7A8<^o-t^hK=>G z8k8rB)gt&e#e8zU5Dwl)Qf%c3~XU(Utb;Y>O+3PC9&=V&P(=JqqJ@4X-7^Nvn~ z;)jas*n@26_-t3jtISBRiv&H% zQy!<~u1LsLYJe4ndXqY0{u$?t=%&`G&{?J3>rjAJ;kBFTy;b>X_SM3j=y4a@KggIO zF~5~k2uqf4I=X$s*;7&yd-mDDyJ&EBG43>qCE|+j+b;q$&^R%^O}Q+%^fqaclZGZ9 z?`5i>L2^4nrnmE!y23P{U_v^zEaQP|-Yw)Qp<{Ov0yz@Ms6R(OaCj8z%nz9}>1}d^ zEDJ%|uM^g1A4M)6f3SgC)J$XKA+g;a4C5$bjbf=bVBFDTqjHimx?wka~-6C14SQ}No 0.5])} +- {np.std(scores[scores > 0.5])}") - print(f"Toxic Percentage {sum(scores > 0.5) / len(scores)}") - print(f"Avg toxicity (nontoxic): {np.mean(scores[scores <= 0.5])} +- {np.std(scores[scores <= 0.5])}") - print(f"Nontoxic Percentage {sum(scores <= 0.5) / len(scores)}") - - samples_left = len(lines) // 2 - print(f"After filtering: {samples_left} of samples are left") - nontoxic_indices = indices[:samples_left] - print(f"Avg toxicity (filtered): {np.mean(scores[nontoxic_indices])} +- {np.std(scores[nontoxic_indices])}") - print(f"Toxicity Range (filtered): {np.min(scores[nontoxic_indices])} ~ {np.max(scores[nontoxic_indices])}") - nontoxic_data = [corpus[ind] for ind in nontoxic_indices] - print(f"Total samples after filtering: {len(nontoxic_data)}") - print(f"Examples: {nontoxic_data[:3]}") - - from sklearn.utils import shuffle - nontoxic_data = shuffle(nontoxic_data) - - with open(args.out_path, 'w') as f: - for x in nontoxic_data: - f.write(json.dumps(x) + '\n') - - -main() \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/perspective_api_annotate.py b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/perspective_api_annotate.py deleted file mode 100644 index fd82c2a2a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/perspective_api_annotate.py +++ /dev/null @@ -1,182 +0,0 @@ -import json -import time -from typing import Dict, Optional, List - -import joblib -from googleapiclient import discovery -from googleapiclient.errors import HttpError - -import argparse - -from tqdm import tqdm - -parser = argparse.ArgumentParser(description='Process some integers.') -parser.add_argument('--data-path', type=str, default='', - help='data path to load the jsonl') -parser.add_argument('--out-path', type=str, default='', - help='data path to load the jsonl') -parser.add_argument('--total', type=int, default=-1, - help='Total number of data') -parser.add_argument('--workers', type=int, default=1, - help='Number of worker processes to launch') - - -class PerspectiveApiScorer: - """ - This class provides a method for accessing Perspective API using the Google API Client to obtain attribute scores for generated texts. - """ - - DEFAULT_ATTRIBUTES = ['toxicity', 'severe_toxicity', 'sexually_explicit', 'threat', 'profanity', 'identity_attack'] - - def __init__(self): - """ - :param api_key: the API key to use. For details, see https://support.perspectiveapi.com/s/docs-get-started - """ - api_key = '' - self._service = discovery.build( - "commentanalyzer", - "v1alpha1", - developerKey=api_key, - discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1", - static_discovery=False, - ) - - def get_scores(self, input_text: str, requested_attributes: Optional[List[str]] = None) -> Dict[str, float]: - """ - Get attribute scores for a given text via Perspective API. - :param input_text: the input text - :param requested_attributes: the attributes for which to compute scores - :return: a mapping from attribute names to scores - """ - requested_attributes = requested_attributes if requested_attributes else PerspectiveApiScorer.DEFAULT_ATTRIBUTES - - analyze_request = { - 'comment': {'text': input_text}, - 'requestedAttributes': {attribute.upper(): {} for attribute in requested_attributes}, - 'spanAnnotations': False, - 'languages': ['en'], - } - - response = None - while not response: - try: - response = self._service.comments().analyze(body=analyze_request).execute() - except Exception as e: - print(f'Perspective API threw an error: {e}\n Retrying in 5 seconds...') - print(input_text) - time.sleep(1) - - return {attribute: response['attributeScores'][attribute.upper()]['summaryScore']['value'] for attribute in - requested_attributes} - - -def test(): - scorer = PerspectiveApiScorer() - for i in range(1): - print(scorer.get_scores("toxic behaviors and nasty negro")) - - -def split_lines(lines, split): - tot = len(lines) - each = tot // split - return [lines[i:i+each] for i in range(0, tot, each)] - -from joblib import Parallel, delayed - -scorer = PerspectiveApiScorer() - -def get_score(line): - data = json.loads(line) - text = data['text'] - text = text.replace("<|endoftext|>", "") - data['text'] = text - if not text.strip(): - data['score'] = None - return json.dumps(data) - - encoded_text = text.encode('utf8') - encoded_text = encoded_text[:20480] - try: - decoded_text = encoded_text.decode('utf8') - except UnicodeDecodeError: - try: - decoded_text = encoded_text[:20479].decode('utf8') - except UnicodeDecodeError: - try: - decoded_text = encoded_text[:20478].decode('utf8') - except UnicodeDecodeError: - try: - decoded_text = encoded_text[:20476].decode('utf8') - except: - print("Error occurred") - data['score'] = None - return json.dumps(data) - data['score'] = scorer.get_scores(decoded_text) - return json.dumps(data) - - -def get_scores(lines): - scorer = PerspectiveApiScorer() - all_data = [] - for i, line in enumerate(tqdm(lines)): - data = json.loads(line) - text = data['text'] - if not text.strip(): - data['score'] = None - all_data.append(json.dumps(data)) - continue - encoded_text = text.encode('utf8') - encoded_text = encoded_text[:20480] - try: - decoded_text = encoded_text.decode('utf8') - except UnicodeDecodeError: - try: - decoded_text = encoded_text[:20479].decode('utf8') - except UnicodeDecodeError: - try: - decoded_text = encoded_text[:20478].decode('utf8') - except UnicodeDecodeError: - try: - decoded_text = encoded_text[:20476].decode('utf8') - except: - print("Error occurred") - data['score'] = None - all_data.append(json.dumps(data)) - continue - data['score'] = scorer.get_scores(decoded_text) - all_data.append(json.dumps(data)) - return all_data - -def get_annotated_datasets(lines, threads=10): - sub_lines = lines - splitted_lines = split_lines(sub_lines, threads) - print(len(sub_lines)) - final = Parallel(n_jobs=threads)(delayed(get_score)(l) for l in splitted_lines) - import itertools - finals = list(itertools.chain.from_iterable(final)) - return finals - - -def main(): - args = parser.parse_args() - - path = args.data_path - out = args.out_path if args.out_path else path + '-annotated.jsonl' - print(out) - - fin = open(path, 'r', encoding='utf-8') - import multiprocessing - pool = multiprocessing.Pool(args.workers) - annotated = pool.imap(get_score, fin, 25) - with open(out, "w") as f: - if args.total > 0: - for x in tqdm(annotated, total=args.total): - f.write(x + '\n') - else: - for x in tqdm(annotated): - f.write(x + '\n') - - -if __name__ == '__main__': - main() - diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/preprocess.sh b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/preprocess.sh deleted file mode 100644 index 4324f8014..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/annotations/preprocess.sh +++ /dev/null @@ -1,14 +0,0 @@ -VOCAB_FILE=pt2-vocab.json -MERGE_FILE=gpt2-merges.txt - -python3 tools/preprocess_data.py \ - --input $1 \ - --output-prefix $2 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --tokenizer-type GPT2BPETokenizer \ - --append-eod --workers 20 --chunk-size 25 - - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt.py b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt.py deleted file mode 100644 index 0675a8508..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt.py +++ /dev/null @@ -1,149 +0,0 @@ -# coding=utf-8 -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. - - -"""Fine-tune GPT""" - -import torch -from functools import partial -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir, os.path.pardir))) -from megatron_ds import get_args -from megatron_ds import get_timers -from megatron_ds import get_tokenizer -from megatron_ds import print_rank_0 -from megatron_ds.core import mpu -from megatron_ds.data.blendable_dataset import BlendableDataset -from megatron_ds.data.gpt_dataset import build_train_valid_test_datasets -from megatron_ds.model import GPTModel -from megatron_ds.arguments import core_transformer_config_from_args -from megatron_ds.core.enums import ModelType -from megatron_ds.training import pretrain -from megatron_ds.utils import get_ltor_masks_and_position_ids -from megatron_ds.utils import average_losses_across_data_parallel_group - -def model_provider(pre_process=True, post_process=True): - """Build the model.""" - - config = core_transformer_config_from_args(args) - - print_rank_0('building GPT model ...') - model = GPTModel( - config=config, - num_tokentypes=0, - parallel_output=True, - pre_process=pre_process, - post_process=post_process - ) - return model - - -def get_batch(data_iterator): - """Generate a batch""" - args = get_args() - tokenizer = get_tokenizer() - - # Items and their type. - keys = ['text'] - datatype = torch.int64 - - # Broadcast data. - if data_iterator is not None: - data = next(data_iterator) - else: - data = None - data_b = mpu.broadcast_data(keys, data, datatype) - - # Unpack. - tokens_ = data_b['text'].long() - labels = tokens_[:, 1:].contiguous() - tokens = tokens_[:, :-1].contiguous() - - # Get the masks and postition ids. - attention_mask, loss_mask, position_ids = get_ltor_masks_and_position_ids( - tokens, - tokenizer.eod, - args.reset_position_ids, - args.reset_attention_mask, - args.eod_mask_loss) - - return tokens, labels, loss_mask, attention_mask, position_ids - -def loss_func(loss_mask, output_tensor): - losses = output_tensor.float() - loss_mask = loss_mask.view(-1).float() - loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() - - # Reduce loss for logging. - averaged_loss = average_losses_across_data_parallel_group([loss]) - - return loss, {'lm loss': averaged_loss[0]} - - -def forward_step(data_iterator, model): - """Forward step.""" - args = get_args() - timers = get_timers() - - # Get the batch. - timers('batch-generator').start() - tokens, labels, loss_mask, attention_mask, position_ids = get_batch( - data_iterator) - timers('batch-generator').stop() - - output_tensor = model(tokens, position_ids, attention_mask, - labels=labels) - - return output_tensor, partial(loss_func, loss_mask) - - -def train_valid_test_datasets_provider(train_val_test_num_samples): - """Build train, valid, and test datasets.""" - args = get_args() - - print_rank_0('> building train, validation, and test datasets ' - 'for GPT ...') - train_ds, valid_ds1, test_ds = build_train_valid_test_datasets( - data_prefix=args.data_path, - data_impl=args.data_impl, - splits_string=args.split, - train_valid_test_num_samples=train_val_test_num_samples, - seq_length=args.seq_length, - seed=args.seed, - skip_warmup=(not args.mmap_warmup)) - print_rank_0("> finished creating finetuning GPT datasets ...") - - _, valid_ds, _ = build_train_valid_test_datasets( - data_prefix=args.data_path2, - data_impl="mmap", - splits_string="98,2,0", - train_valid_test_num_samples=train_val_test_num_samples, - seq_length=2048, - seed=1234, - skip_warmup=(not args.mmap_warmup)) - print_rank_0("> finished creating pretrained GPT datasets ...") - - return train_ds, valid_ds, test_ds - - -def add_validation_args(parser): - """Text generation arguments.""" - group = parser.add_argument_group(title='validation set') - group.add_argument('--data-path2', nargs='*', default=None, - help='Path to the validation dataset. Accepted format:' - '1) a single data path, 2) multiple datasets in the' - 'form: dataset1-weight dataset1-path dataset2-weight ' - 'dataset2-path ...') - group.add_argument('--eval-ppl', action='store_true', default=False) - group.add_argument('--stored_params', type=dict, default=dict()) - return parser - - -if __name__ == "__main__": - - pretrain(train_valid_test_datasets_provider, model_provider, - ModelType.encoder_or_decoder, - forward_step, args_defaults={'tokenizer_type': 'GPT2BPETokenizer'}, - extra_args_provider=add_validation_args,) diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt_distributed-1.3b.sh b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt_distributed-1.3b.sh deleted file mode 100644 index 62a36c0b7..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/finetune_gpt_distributed-1.3b.sh +++ /dev/null @@ -1,64 +0,0 @@ -#! /bin/bash - -# Change for multinode config -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=$(($RANDOM + 1024)) -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -# input -DATA_PATH=$1 -SHARE_DATA=$PWD # current work dir -FINETUNED_PATH="$SHARE_DATA/$2" -lr=$3 -bs=$4 -iter=$5 -CHECKPOINT_PATH=$6 - -# vocab -VOCAB_FILE=gpt2-vocab.json # Your gpt-2 vocab -MERGE_FILE=gpt2-merges.txt # Your gpt-2 merge file - -# tensorboard -TENSORBOARD_DIR="$SHARE_DATA/tensorboard/$2" -mkdir -p ${TENSORBOARD_DIR} - -DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT" - -python -m torch.distributed.run $DISTRIBUTED_ARGS \ - examples/detxoify_lm/finetune_gpt.py \ - --num-layers 24 \ - --hidden-size 2048 \ - --num-attention-heads 32 \ - --micro-batch-size 4 \ - --global-batch-size $bs \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --train-iters $iter \ - --save $FINETUNED_PATH \ - --load $CHECKPOINT_PATH \ - --data-path $DATA_PATH \ - --data-path2 ${DATA_BLEND} \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --data-impl mmap \ - --split 100,0,0 \ - --distributed-backend nccl \ - --lr-decay-style constant \ - --lr $lr \ - --clip-grad 1.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --checkpoint-activations \ - --log-interval 1 \ - --save-interval 78 \ - --eval-interval 78 \ - --eval-iters 50 \ - --fp16 \ - --DDP-impl local \ - --finetune --no-load-optim \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR} diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate-1.3b.sh b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate-1.3b.sh deleted file mode 100644 index 95bb47867..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate-1.3b.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -CHECKPOINT_PATH=$2 # Your model ckpt -VOCAB_FILE=gpt2-vocab.json -MERGE_FILE=gpt2-merges.txt - -GPUS_PER_NODE=1 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=$(($RANDOM + 1024)) -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) -NUM_SAMPLES=$(wc -l < $1) -PREFIX=$(basename $2) -SEED=$(($RANDOM)) -OUTPUT=$1_output_"$PREFIX"_seed_"$SEED".jsonl - -DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT" - -python -m torch.distributed.run $DISTRIBUTED_ARGS examples/detxoify_lm/generate_samples_gpt.py \ - --tensor-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 2048 \ - --load $CHECKPOINT_PATH \ - --num-attention-heads 32 \ - --max-position-embeddings 2048 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --micro-batch-size 400 \ - --seq-length 2048 \ - --out-seq-length 20 \ - --temperature 1.0 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --sample-input-file $1 \ - --sample-output-file $OUTPUT \ - --num-samples $NUM_SAMPLES \ - --max-tokens-to-oom 1200000 \ - --top_p 0.9 \ - --seed $SEED - diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate_samples_gpt.py b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate_samples_gpt.py deleted file mode 100644 index bcf81e25b..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/generate_samples_gpt.py +++ /dev/null @@ -1,202 +0,0 @@ -# coding=utf-8 -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. - - -"""Sample Generate GPT""" -import json -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir, os.path.pardir))) -import torch -from megatron_ds import get_args -from megatron_ds import get_tokenizer -from megatron_ds import print_rank_0 -from megatron_ds.checkpointing import load_checkpoint -from megatron_ds.core import mpu -from megatron_ds.initialize import initialize_megatron -from megatron_ds.model import GPTModel -from megatron_ds.training import get_model -from megatron_ds.arguments import core_transformer_config_from_args -from megatron_ds.text_generation import generate_and_post_process - - -def model_provider(pre_process=True, post_process=True): - """Build the model.""" - - config = core_transformer_config_from_args(args) - - print_rank_0('building GPT model ...') - model = GPTModel(config=config, num_tokentypes=0, parallel_output=False, - pre_process=pre_process, post_process=post_process) - - return model - -def add_text_generate_args(parser): - """Text generation arguments.""" - group = parser.add_argument_group(title='text generation') - - group.add_argument("--temperature", type=float, default=1.0, - help='Sampling temperature.') - group.add_argument("--greedy", action='store_true', default=False, - help='Use greedy sampling.') - group.add_argument("--top_p", type=float, default=0.0, - help='Top p sampling.') - group.add_argument("--top_k", type=int, default=0, - help='Top k sampling.') - group.add_argument("--out-seq-length", type=int, default=1024, - help='Size of the output generated text.') - group.add_argument("--sample-input-file", type=str, default=None, - help='Get input from file instead of interactive mode, ' - 'each line is an input.') - group.add_argument("--sample-output-file", type=str, default=None, - help='Output file got from --sample-input-file') - group.add_argument("--num-samples", type=int, default=0, - help='Number of samples to generate unconditionally, ' - 'defaults to 0 and interactive conditional sampling') - group.add_argument("--genfile", type=str, - help='Output file when generating unconditionally') - return parser - -def generate_samples_unconditional(model): - args = get_args() - - if torch.distributed.get_rank() == 0: - cnt = 0 - num_samples = args.num_samples - from tqdm import tqdm - pbar = tqdm(total=num_samples) - - while True: - if torch.distributed.get_rank() == 0: - sentences = [''] * args.global_batch_size - print("global batch size", args.global_batch_size) - max_len = args.out_seq_length - resp_sentences, resp_sentences_seg, output_logits, \ - tokens = generate_and_post_process(model, prompts=sentences, - tokens_to_generate=max_len, - return_output_log_probs=False, - top_k_sampling=args.top_k, - top_p_sampling=args.top_p, - add_BOS=True, - temperature=1.0) - for prompt, generation, token in zip(sentences, resp_sentences, tokens): - datum = {'text': generation[len(prompt):], 'all_text': generation, 'prompt': prompt, 'id': cnt} - yield datum - cnt += 1 - pbar.update() - if cnt >= num_samples: - break - - if cnt >= num_samples: - pbar.close() - break - else: - generate_and_post_process(model) - - -def generate_samples_conditional(model): - args = get_args() - - if torch.distributed.get_rank() == 0: - num_samples = args.num_samples - cnt = 0 - from tqdm import tqdm - pbar = tqdm(total=num_samples) - - fname = open(args.sample_input_file, "r") - lines = fname.readlines() - all_raw_text = [json.loads(line)['prompt']['text'] for line in lines] - input_count = len(all_raw_text) - input_pos = 0 - - while True: - torch.distributed.barrier() - if torch.distributed.get_rank() == 0: - sentences = [] - print("global batch size", args.global_batch_size) - for _ in range(args.global_batch_size): - if input_pos >= input_count: - print(f"input pos: {input_pos}, input count: {input_count}") - raw_text = "EMPTY TEXT" - else: - raw_text = all_raw_text[input_pos] - input_pos += 1 - sentences.append(raw_text) - - max_len = args.out_seq_length - resp_sentences, resp_sentences_seg, output_logits, \ - tokens = generate_and_post_process(model, prompts=sentences, - tokens_to_generate=max_len, - return_output_log_probs=False, - top_k_sampling=args.top_k, - top_p_sampling=args.top_p, - add_BOS=False, - temperature=1.0) - for prompt, generation, token in zip(sentences, resp_sentences, tokens): - datum = {'text': generation[len(prompt):], 'all_text': generation, 'prompt': prompt, 'id': cnt} - yield datum - cnt += 1 - pbar.update() - if cnt >= num_samples: - break - - if cnt >= num_samples: - pbar.close() - break - else: - generate_and_post_process(model) - - -def generate_and_write_samples_unconditional(model): - args = get_args() - assert args.genfile is not None - with open(args.genfile, 'w') as f: - for datum in generate_samples_unconditional(model): - if torch.distributed.get_rank() == 0: - f.write(json.dumps(datum) + '\n') - - -def generate_and_write_samples_conditional(model): - args = get_args() - if args.sample_output_file is None: - sample_output_file = args.sample_input_file + ".out" - print('`sample-output-file` not specified, setting ' - 'it to {}'.format(sample_output_file)) - else: - sample_output_file = args.sample_output_file - with open(sample_output_file, 'w') as f: - for datum in generate_samples_conditional(model): - if torch.distributed.get_rank() == 0: - f.write(json.dumps(datum) + '\n') - - -def main(): - """Main program.""" - - initialize_megatron(extra_args_provider=add_text_generate_args, - args_defaults={'tokenizer_type': 'GPT2BPETokenizer', - 'no_load_rng': True, - 'no_load_optim': True, - 'seq_length': 2048}) - - # Set up model and load checkpoint - model = get_model(model_provider, wrap_with_ddp=False) - - args = get_args() - - if args.load is not None: - _ = load_checkpoint(model, None, None) - model = model[0] - - # Generate samples. - if args.sample_input_file != None: - print(f"{args.sample_input_file}") - generate_and_write_samples_conditional(model) - else: - generate_and_write_samples_unconditional(model) - - -if __name__ == "__main__": - - main() diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/perspective_api.py b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/perspective_api.py deleted file mode 100644 index cad3130a3..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/perspective_api.py +++ /dev/null @@ -1,170 +0,0 @@ -import json -import time -from typing import Dict, Optional, List - -import joblib -from googleapiclient import discovery -from googleapiclient.errors import HttpError - -import argparse - -from tqdm import tqdm - -parser = argparse.ArgumentParser(description='Process some integers.') -parser.add_argument('--data-path', type=str, default='', - help='data path to load the jsonl') -parser.add_argument('--out-path', type=str, default='', - help='data path to load the jsonl') -parser.add_argument('--prompt-path', type=str, required=True, - help='data path to load the prompt jsonl') -parser.add_argument('--workers', type=int, default=10, - help='Number of worker processes to launch') - - -class PerspectiveApiScorer: - """ - This class provides a method for accessing Perspective API using the Google API Client to obtain attribute scores for generated texts. - """ - - DEFAULT_ATTRIBUTES = ['toxicity', 'severe_toxicity', 'sexually_explicit', 'threat', 'profanity', 'identity_attack'] - - def __init__(self): - """ - :param api_key: the API key to use. For details, see https://support.perspectiveapi.com/s/docs-get-started - """ - api_key = '' - self._service = discovery.build( - "commentanalyzer", - "v1alpha1", - developerKey=api_key, - discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1", - static_discovery=False, - ) - - def get_scores(self, input_text: str, requested_attributes: Optional[List[str]] = None) -> Dict[str, float]: - """ - Get attribute scores for a given text via Perspective API. - :param input_text: the input text - :param requested_attributes: the attributes for which to compute scores - :return: a mapping from attribute names to scores - """ - requested_attributes = requested_attributes if requested_attributes else PerspectiveApiScorer.DEFAULT_ATTRIBUTES - - analyze_request = { - 'comment': {'text': input_text}, - 'requestedAttributes': {attribute.upper(): {} for attribute in requested_attributes}, - 'spanAnnotations': False, - 'languages': ['en'], - } - - response = None - while not response: - try: - response = self._service.comments().analyze(body=analyze_request).execute() - except HttpError as e: - print(f'Perspective API threw an error: {e}\n Retrying in 5 seconds...') - print(input_text) - time.sleep(1) - - return {attribute: response['attributeScores'][attribute.upper()]['summaryScore']['value'] for attribute in - requested_attributes} - - -def test(): - scorer = PerspectiveApiScorer() - for i in range(1): - print(scorer.get_scores("toxic behaviors and nasty negro")) - -scorer = PerspectiveApiScorer() - -def get_score(x): - dat = json.loads(x) - text = dat['text'] - if not text.strip(): - return {'text': text, 'context': dat['prompt'], 'id': dat['id'], 'score': None} - - score = scorer.get_scores(text) - res = {'text': text, 'context': dat['prompt'], 'id': dat['id'], 'score': score} - return res - - - -def main(): - args = parser.parse_args() - - path = args.data_path - scores = [] - out = args.out_path if args.out_path else path + '.out.pkl' - print(out) - - import os - - if os.path.exists(out): - scores = joblib.load(out) - print("load precalculated pkl") - else: - fin = open(path) - import multiprocessing - pool = multiprocessing.Pool(args.workers) - delayed_scores = pool.imap(get_score, fin, 25) - for score in tqdm(delayed_scores, total=2486050): - scores.append(score) - joblib.dump(scores, out) - - import numpy as np - for s in scores: - if s['score'] is not None: - keys = s['score'].keys() - break - - print("Overall average:") - for k in keys: - print(f"average {k}:", np.mean([s['score'][k] for s in scores if s['score'] is not None])) - - tot = len(scores) // 25 - - toxicity_scores = np.array([s['score']['toxicity'] if s['score'] else -1 for s in scores]) - - with open(args.prompt_path) as f: - prompts = f.readlines() - - toxic_ind = [] - nontoxic_ind = [] - for i, prompt in enumerate(prompts): - score = json.loads(prompt)['prompt']['toxicity'] - if score and score > 0.5: - toxic_ind.append(i) - elif score and score <= 0.5: - nontoxic_ind.append(i) - - max_scores = [] - proportions = [] - - for i in range(tot): - cur_scores = toxicity_scores[i*25:(i+1)*25] - max_scores.append(max(cur_scores)) - proportions.append((cur_scores > 0.5).sum()) - # print(proportions) - - max_scores = np.array(max_scores) - proportions = np.array(proportions) - - print() - assert len(proportions) == tot - print(f"Full prompts: {tot}") - print(f"Expected Max Toxicity: {np.mean(max_scores)} +- {np.std(max_scores)}") - print(f"Toxicity Probability: {(np.array(proportions) >= 1).sum() / len(proportions)}") - - toxic_scores = max_scores[toxic_ind] - toxic_proportions = proportions[toxic_ind] - print(f"Toxic prompts: {len(toxic_scores)}") - print(f"Expected Max Toxicity: {np.mean(toxic_scores)} +- {np.std(toxic_scores)}") - print(f"Toxicity Probability: {(np.array(toxic_proportions) >= 1).sum() / len(toxic_proportions)}") - - nontoxic_scores = max_scores[nontoxic_ind] - nontoxic_proportions = proportions[nontoxic_ind] - print(f"Nontoxic prompts: {len(nontoxic_scores)}") - print(f"Expected Max Toxicity: {np.mean(nontoxic_scores)} +- {np.std(nontoxic_scores)}") - print(f"Toxicity Probability: {(np.array(nontoxic_proportions) >= 1).sum() / len(nontoxic_proportions)}") - -main() diff --git a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh b/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh deleted file mode 100644 index 2a672409d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -CHECKPOINT_PATH=$2 # Your model ckpt -SHARE_DATA=$PWD # current work dir -VOCAB_FILE=gpt2-vocab.json # Your gpt-2 vocab -MERGE_FILE=gpt2-merges.txt # Your gpt-2 merge file - -GPUS_PER_NODE=1 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=$(($RANDOM + 1024)) -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) -SEED=$3 -SUFFIX=$(basename $CHECKPOINT_PATH) -save_dir=$SHARE_DATA/selfgeneration/unconditional_generation_$SUFFIX/ -mkdir -p $save_dir -echo $save_dir/$SEED.out - -DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT" - -python -m torch.distributed.run $DISTRIBUTED_ARGS examples/detxoify_lm/generate_samples_gpt.py \ - --tensor-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 2048 \ - --load $CHECKPOINT_PATH \ - --num-attention-heads 32 \ - --max-position-embeddings 2048 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --micro-batch-size 150 \ - --seq-length 2048 \ - --out-seq-length 1000 \ - --temperature 1.0 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --num-samples $1 \ - --top_p 0.9 \ - --max-tokens-to-oom 1200000 \ - --genfile $save_dir/$SEED.out \ - --seed $SEED - diff --git a/toolbox/Megatron-DeepSpeed/examples/evaluate_retriever_nq.sh b/toolbox/Megatron-DeepSpeed/examples/evaluate_retriever_nq.sh deleted file mode 100644 index 16e937f4f..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/evaluate_retriever_nq.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Evaluate natural question test data given Wikipedia embeddings and pretrained -# ICT model or a finetuned model for Natural Question task - -# Datasets can be downloaded from the following link: -# https://github.com/facebookresearch/DPR/blob/master/data/download_data.py - -EVIDENCE_DATA_DIR= -EMBEDDING_PATH= -CHECKPOINT_PATH= - -QA_FILE= - -python tasks/main.py \ - --task RETRIEVER-EVAL \ - --tokenizer-type BertWordPieceLowerCase \ - --num-layers 12 \ - --hidden-size 768 \ - --num-attention-heads 12 \ - --tensor-model-parallel-size 1 \ - --micro-batch-size 128 \ - --activations-checkpoint-method uniform \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --load ${CHECKPOINT_PATH} \ - --evidence-data-path ${EVIDENCE_DATA_DIR} \ - --embedding-path ${EMBEDDING_PATH} \ - --retriever-seq-length 256 \ - --vocab-file bert-vocab.txt\ - --qa-data-test ${QA_FILE} \ - --faiss-use-gpu \ - --retriever-report-topk-accuracies 1 5 20 100 \ - --fp16 \ - --indexer-log-interval 1000 \ - --indexer-batch-size 128 - - diff --git a/toolbox/Megatron-DeepSpeed/examples/evaluate_zeroshot_gpt.sh b/toolbox/Megatron-DeepSpeed/examples/evaluate_zeroshot_gpt.sh deleted file mode 100644 index f8c38dc01..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/evaluate_zeroshot_gpt.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -WORLD_SIZE=8 - -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -TASK="LAMBADA" - -VALID_DATA= -VOCAB_FILE=gpt2-vocab.json -MERGE_FILE=gpt2-merges.txt -CHECKPOINT=checkpoints/gpt2_345m - - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \ - --task $TASK \ - --valid-data $VALID_DATA \ - --tokenizer-type GPT2BPETokenizer \ - --strict-lambada \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --load $CHECKPOINT \ - --tensor-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --batch-size 8 \ - --activations-checkpoint-method uniform \ - --seq-length 1024 \ - --max-position-embeddings 1024 \ - --log-interval 10 \ - --fp16 \ - --no-load-optim \ - --no-load-rng diff --git a/toolbox/Megatron-DeepSpeed/examples/finetune_mnli_distributed.sh b/toolbox/Megatron-DeepSpeed/examples/finetune_mnli_distributed.sh deleted file mode 100644 index 9219e595d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/finetune_mnli_distributed.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -WORLD_SIZE=8 - -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -TRAIN_DATA="data/glue_data/MNLI/train.tsv" -VALID_DATA="data/glue_data/MNLI/dev_matched.tsv \ - data/glue_data/MNLI/dev_mismatched.tsv" -PRETRAINED_CHECKPOINT=checkpoints/bert_345m -VOCAB_FILE=bert-vocab.txt -CHECKPOINT_PATH=checkpoints/bert_345m_mnli - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \ - --task MNLI \ - --seed 1234 \ - --train-data $TRAIN_DATA \ - --valid-data $VALID_DATA \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file $VOCAB_FILE \ - --epochs 5 \ - --pretrained-checkpoint $PRETRAINED_CHECKPOINT \ - --tensor-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --micro-batch-size 8 \ - --activations-checkpoint-method uniform \ - --lr 5.0e-5 \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.065 \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --save-interval 500000 \ - --save $CHECKPOINT_PATH \ - --log-interval 10 \ - --eval-interval 100 \ - --eval-iters 50 \ - --weight-decay 1.0e-1 \ - --fp16 diff --git a/toolbox/Megatron-DeepSpeed/examples/finetune_race_distributed.sh b/toolbox/Megatron-DeepSpeed/examples/finetune_race_distributed.sh deleted file mode 100644 index e7f70a70a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/finetune_race_distributed.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -WORLD_SIZE=8 - -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -TRAIN_DATA="data/RACE/train/middle" -VALID_DATA="data/RACE/dev/middle \ - data/RACE/dev/high" -VOCAB_FILE=bert-vocab.txt -PRETRAINED_CHECKPOINT=checkpoints/bert_345m -CHECKPOINT_PATH=checkpoints/bert_345m_race - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \ - --task RACE \ - --seed 1234 \ - --train-data $TRAIN_DATA \ - --valid-data $VALID_DATA \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file $VOCAB_FILE \ - --epochs 3 \ - --pretrained-checkpoint $PRETRAINED_CHECKPOINT \ - --tensor-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --micro-batch-size 4 \ - --activations-checkpoint-method uniform \ - --lr 1.0e-5 \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.06 \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --save-interval 100000 \ - --save $CHECKPOINT_PATH \ - --log-interval 10 \ - --eval-interval 100 \ - --eval-iters 50 \ - --weight-decay 1.0e-1 \ - --clip-grad 1.0 \ - --hidden-dropout 0.1 \ - --attention-dropout 0.1 \ - --fp16 diff --git a/toolbox/Megatron-DeepSpeed/examples/finetune_retriever_distributed.sh b/toolbox/Megatron-DeepSpeed/examples/finetune_retriever_distributed.sh deleted file mode 100644 index 535a2e053..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/finetune_retriever_distributed.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Finetune a BERT or pretrained ICT model using Google natural question data -# Datasets can be downloaded from the following link: -# https://github.com/facebookresearch/DPR/blob/master/data/download_data.py - -WORLD_SIZE=8 - -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT_PATH= - -# Load either of the below -BERT_LOAD_PATH= -PRETRAINED_CHECKPOINT= - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \ - --task RET-FINETUNE-NQ \ - --train-with-neg \ - --train-hard-neg 1 \ - --pretrained-checkpoint ${PRETRAINED_CHECKPOINT} \ - --num-layers 12 \ - --hidden-size 768 \ - --num-attention-heads 12 \ - --tensor-model-parallel-size 1 \ - --tokenizer-type BertWordPieceLowerCase \ - --train-data nq-train.json \ - --valid-data nq-dev.json \ - --save ${CHECKPOINT_PATH} \ - --load ${CHECKPOINT_PATH} \ - --vocab-file bert-vocab.txt \ - --bert-load ${BERT_LOAD_PATH} \ - --save-interval 5000 \ - --log-interval 10 \ - --eval-interval 20000 \ - --eval-iters 100 \ - --indexer-log-interval 1000 \ - --faiss-use-gpu \ - --DDP-impl torch \ - --fp16 \ - --retriever-report-topk-accuracies 1 5 10 20 100 \ - --seq-length 512 \ - --retriever-seq-length 256 \ - --max-position-embeddings 512 \ - --retriever-score-scaling \ - --epochs 80 \ - --micro-batch-size 8 \ - --eval-micro-batch-size 16 \ - --indexer-batch-size 128 \ - --lr 2e-5 \ - --lr-warmup-fraction 0.01 \ - --weight-decay 1e-1 diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_ixte_llama2_34b_node4.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_ixte_llama2_34b_node4.sh deleted file mode 100644 index 9435566c8..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_ixte_llama2_34b_node4.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash -set -ex -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_NET=IB -export NCCL_SOCKET_IFNAME="bond0" -export NCCL_NET_SHARED_BUFFERS=0 -# export NCCL_DEBUG=INFO -export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - -# export NCCL_USE_HIGHPRIORITYWARP=1 -# export NCCL_FORCESYNC_DISABLE=1 -# export NCCL_USE_DIRECT=1 -# export OMP_NUM_THREADS=4 -# export UMD_CCLINLASTCE=1 - -HOST_NAME="poweruser" - -ADDR_ARRAY=("10.113.2.49" "10.113.2.50" "10.113.2.45" "10.113.2.12") -CONTAINER_NAME="llama2_34b_tr6" - -HOST_IP=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2;}'|tr -d "addr:"|head -n 1) -CURRENT_DIR=`pwd` -CUR_SCR=$0 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -# wa: clean dataset cache -rm -rf ${DATA_PATH}/cache > /dev/null 2>&1 -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -NODES=4 - -TRANSFORMER_IMPL=transformer_engine - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 2 \ - --pipeline-model-parallel-size 16\ - --micro-batch-size 1 \ - --global-batch-size 1024 \ - --disable-bias-linear \ - --use-distributed-optimizer \ - --use-flash-attn \ - --sequence-parallel \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL\ - --use-distributed-optimizer \ - --recompute-granularity full \ - --recompute-method block \ - --make-vocab-size-divisible-by 1 \ - --recompute-num-layers 1 \ - --recompute-method-per-stage 16 1 \ - --recompute-num-layers-per-stage 2 1 14 0 \ -" - # --custom-recompute-layers-per-stage 2 2 1 0 0 0 0 0 \ - # --no-gradient-accumulation-fusion \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - -DATA_ARGS=" - --data-path $DATA_PATH \ - --data-impl mmap \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 98,2,0 -" - -NETWORK_ARGS=" - --num-layers 48 \ - --hidden-size 8192 \ - --ffn-hidden-size 22016 \ - --num-attention-heads 64 \ - --group-query-attention \ - --num-query-groups 8 \ - --seq-length 4096 \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights \ - --swiglu \ - --normalization RMSNorm \ - --no-masked-softmax-fusion -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - -megatron_args="$TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS" - -function exec_ssh_by_master -{ - # only at master host, start all other non master hosts run - if [[ "$HOST_IP" == "${ADDR_ARRAY[0]}" ]] - then - for i in "${!ADDR_ARRAY[@]}" - do - if [ "$i" != "0" ] - then - scp ${CUR_SCR} ${HOST_NAME}@${ADDR_ARRAY[$i]}:${CURRENT_DIR} - # scp -r ${DATA_PATH} ${HOST_NAME}@${ADDR_ARRAY[$i]}:${DATA_PATH}/../ - - ssh ${HOST_NAME}@${ADDR_ARRAY[$i]} "docker exec ${CONTAINER_NAME} bash -c \"cd ${CURRENT_DIR}; bash ${CUR_SCR} \"" & - fi - done - fi -} -function run_ddp_mm() -{ - for i in "${!ADDR_ARRAY[@]}" - do - if [[ "$HOST_IP" == "${ADDR_ARRAY[$i]}" ]] - then - echo "nodes: ${#ADDR_ARRAY[@]}, rank: $i, IP: $HOST_IP, MASTER_IP: ${ADDR_ARRAY[0]}" - DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NODES --node_rank $i --master_addr ${ADDR_ARRAY[0]} --master_port 52321" - torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - ${megatron_args} | tee ${LOG_PATH}/output.log 2>&1 - fi - done -} -exec_ssh_by_master -run_ddp_mm diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_llama2_7b.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_llama2_7b.sh deleted file mode 100644 index 72de01f47..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_llama2_7b.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables -# base on the cluster configuration -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_SOCKET_IFNAME=ens5f0 -# export NCCL_IB_DISABLE=0 -# export NCCL_IB_CUDA_SUPPORT=1 -# export NCCL_IB_GID_INDEX=0 -# export NCCL_IB_HCA=mlx5_0,mlx5_3 -# export NCCL_DEBUG=debug -# export OMP_NUM_THREADS=4 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=${PROJ_HOME}/checkpoints/output_step1_llama2_7b_vocab_size_32000/tokenizer.model -LOAD_CHECKPOINT_PATH=${PROJ_HOME}/checkpoints/llama2_7b_megatron - -SAVE_CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $SAVE_CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 4 \ - --pipeline-model-parallel-size 4 \ - --micro-batch-size 1 \ - --global-batch-size 32 \ - --disable-bias-linear \ - --use-flash-attn - --eval-interval 1000 \ -" - # --use-distributed-optimizer \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers 32 \ - --hidden-size 4096 \ - --num-attention-heads 32 \ - --seq-length 4096 \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights \ - --load $LOAD_CHECKPOINT_PATH \ - --exit-on-missing-checkpoint \ - --use-checkpoint-args \ - --no-load-optim \ - --no-load-rng \ - --no-masked-softmax-fusion \ -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $SAVE_CHECKPOINT_PATH \ -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_tinyllama_1.1b.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_tinyllama_1.1b.sh deleted file mode 100644 index 7c13f2ab7..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_load_weight_tinyllama_1.1b.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables -# base on the cluster configuration -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_SOCKET_IFNAME=eth0 -# export NCCL_IB_DISABLE=0 -# export NCCL_IB_CUDA_SUPPORT=1 -# export NCCL_IB_GID_INDEX=0 -# export NCCL_IB_HCA=mlx5_0,mlx5_3 -# export NCCL_DEBUG=debug -# export OMP_NUM_THREADS=4 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=${PROJ_HOME}/checkpoints/output_tinyLlama-1.1B-intermediate-step-240k-503b/tokenizer.model -LOAD_CHECKPOINT_PATH=${PROJ_HOME}/checkpoints/rlhf_tinyllama_1.1b_tp4_pp4 - -SAVE_CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $SAVE_CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 4 \ - --pipeline-model-parallel-size 4 \ - --custom-partition 5 5 6 6 \ - --micro-batch-size 1 \ - --global-batch-size 32 \ - --disable-bias-linear \ - --use-flash-attn - --eval-interval 1000 \ -" - # --use-distributed-optimizer \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers 22 \ - --hidden-size 2048 \ - --num-attention-heads 32 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights \ - --load $LOAD_CHECKPOINT_PATH \ - --exit-on-missing-checkpoint \ - --use-checkpoint-args \ - --no-load-optim \ - --no-load-rng \ - --no-masked-softmax-fusion \ - --group-query-attention \ - --num-query-groups 4 -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $SAVE_CHECKPOINT_PATH \ -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_node4.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_node4.sh deleted file mode 100644 index 22736d990..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_node4.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/bin/bash -set -ex -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_NET=IB -export NCCL_SOCKET_IFNAME="bond0" -export NCCL_NET_SHARED_BUFFERS=0 -# export NCCL_DEBUG=INFO -export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - - -## torch tp overlap -# export ENABLE_TORCH_TP_OVERLAP=1 -# export TORCH_TP_OVERLAP_SIZE=4 -# export NCCL_USE_HIGHPRIORITYWARP=1 -# export NCCL_FORCESYNC_DISABLE=1 -# export NCCL_USE_DIRECT=1 -# export OMP_NUM_THREADS=4 -# export UMD_CCLINLASTCE=1 - -HOST_NAME="jun.zhao" - -ADDR_ARRAY=("10.113.2.10" "10.113.2.9" "10.113.2.11" "10.113.2.12") -CONTAINER_NAME="llama_0323" - -HOST_IP=$(echo $(hostname -I) | cut -d " " --output-delimiter="," -f 1) -CURRENT_DIR=`pwd` -CUR_SCR=$0 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -NODES=4 - -TRANSFORMER_IMPL=transformer_engine - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 2 \ - --pipeline-model-parallel-size 8\ - --micro-batch-size 1 \ - --global-batch-size 256 \ - --disable-bias-linear \ - --use-distributed-optimizer \ - --use-flash-attn \ - --sequence-parallel \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL\ - --use-distributed-optimizer \ - --recompute-granularity full \ - --recompute-method block \ - --make-vocab-size-divisible-by 1 \ - --recompute-num-layers 1 \ - --recompute-method-per-stage 8 1 \ - --recompute-num-layers-per-stage 1 4 1 3 2 2 4 0 \ -" - # --custom-recompute-layers-per-stage 2 2 1 0 0 0 0 0 \ - # --no-gradient-accumulation-fusion \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - -DATA_ARGS=" - --data-path $DATA_PATH \ - --data-impl mmap \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 98,2,0 -" - -NETWORK_ARGS=" - --num-layers 48 \ - --hidden-size 8192 \ - --ffn-hidden-size 22016 \ - --num-attention-heads 64 \ - --group-query-attention \ - --num-query-groups 8 \ - --seq-length 4096 \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights \ - --swiglu \ - --normalization RMSNorm \ - --no-masked-softmax-fusion -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - -megatron_args="$TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS" - -function exec_ssh_by_master -{ - # only at master host, start all other non master hosts run - if [[ "$HOST_IP" == "${ADDR_ARRAY[0]}" ]] - then - for i in "${!ADDR_ARRAY[@]}" - do - if [ "$i" != "0" ] - then - scp ${CUR_SCR} ${HOST_NAME}@${ADDR_ARRAY[$i]}:${CURRENT_DIR} - # scp -r ${DATA_PATH} ${HOST_NAME}@${ADDR_ARRAY[$i]}:${DATA_PATH}/../ - - ssh ${HOST_NAME}@${ADDR_ARRAY[$i]} "docker exec ${CONTAINER_NAME} bash -c \"cd ${CURRENT_DIR}; bash ${CUR_SCR} \"" & - fi - done - fi -} -function run_ddp_mm() -{ - for i in "${!ADDR_ARRAY[@]}" - do - if [[ "$HOST_IP" == "${ADDR_ARRAY[$i]}" ]] - then - echo "nodes: ${#ADDR_ARRAY[@]}, rank: $i, IP: $HOST_IP, MASTER_IP: ${ADDR_ARRAY[0]}" - DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NODES --node_rank $i --master_addr ${ADDR_ARRAY[0]} --master_port 54321" - torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - ${megatron_args} | tee ${LOG_PATH}/output.log 2>&1 - fi - done -} -exec_ssh_by_master -run_ddp_mm \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_tpoverlap_profiling_node1.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_tpoverlap_profiling_node1.sh deleted file mode 100644 index 5430b4a30..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_34b_tpoverlap_profiling_node1.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables -# base on the cluster configuration -export CUDA_DEVICE_MAX_CONNECTIONS=1 -# export NCCL_SOCKET_IFNAME=eth0 -# export NCCL_IB_DISABLE=0 -# export NCCL_IB_CUDA_SUPPORT=1 -# export NCCL_IB_GID_INDEX=0 -# export NCCL_IB_HCA=mlx5_0,mlx5_3 -# export NCCL_DEBUG=debug -# export OMP_NUM_THREADS=4 -export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -TRANSFORMER_IMPL=transformer_engine - -export ENABLE_TORCH_TP_OVERLAP=1 -export TORCH_TP_OVERLAP_SIZE=4 -export NCCL_USE_HIGHPRIORITYWARP=1 - -export NCCL_FORCESYNC_DISABLE=1 -export NCCL_USE_DIRECT=1 -export OMP_NUM_THREADS=4 -export UMD_CCLINLASTCE=1 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 4 \ - --pipeline-model-parallel-size 4 \ - --sequence-parallel \ - --micro-batch-size 1 \ - --global-batch-size 32 \ - --disable-bias-linear \ - --use-flash-attn \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL\ - --use-distributed-optimizer \ - --no-gradient-accumulation-fusion \ -" -## 自定义recompute layers pp stage - # --recompute-granularity full \ - # --recompute-method block \ - # --custom-recompute-layers-per-stage 3 1 0 0 \ - # --no-gradient-accumulation-fusion \ - - -## 自定义切分pp stage，仅针对transformer layers - # --custom-partition 3 3 4 4 4 4 5 5 \ - -# --use-distributed-optimizer \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -## 模型原参数：num-layers=48 -NETWORK_ARGS=" - --num-layers 16 \ - --hidden-size 8192 \ - --num-attention-heads 64 \ - --seq-length 4096 \ - --ffn-hidden-size 22016 \ - --num-query-groups 8 \ - --group-query-attention \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - # --wandb-save-dir $WB_PATH \ - # --tensorboard-dir $TB_PATH \ - # --tensorboard-log-interval 1 - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_node4.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_node4.sh deleted file mode 100644 index ac69a8ada..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_node4.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash -set -ex -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_NET=IB -export NCCL_SOCKET_IFNAME="bond0" -export NCCL_NET_SHARED_BUFFERS=0 -# export NCCL_DEBUG=INFO -export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - -## torch tp overlap -# export ENABLE_TORCH_TP_OVERLAP=1 -# export TORCH_TP_OVERLAP_SIZE=4 -# export NCCL_USE_HIGHPRIORITYWARP=1 -# export NCCL_FORCESYNC_DISABLE=1 -# export NCCL_USE_DIRECT=1 -# export OMP_NUM_THREADS=4 -# export UMD_CCLINLASTCE=1 - -HOST_NAME="jun.zhao" - -ADDR_ARRAY=("10.113.2.10" "10.113.2.9" "10.113.2.11" "10.113.2.12") -CONTAINER_NAME="llama_0323" - -HOST_IP=$(echo $(hostname -I) | cut -d " " --output-delimiter="," -f 1) -CURRENT_DIR=`pwd` -CUR_SCR=$0 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -NODES=4 - -TRANSFORMER_IMPL=transformer_engine - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 4 \ - --pipeline-model-parallel-size 16 \ - --micro-batch-size 1 \ - --global-batch-size 256 \ - --disable-bias-linear \ - --use-distributed-optimizer \ - --use-flash-attn \ - --sequence-parallel \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL\ - --use-distributed-optimizer \ - --recompute-granularity full \ - --recompute-method block \ - --make-vocab-size-divisible-by 1 \ - --recompute-num-layers 1 \ - --recompute-method-per-stage 16 1 \ - --recompute-num-layers-per-stage 1 4 2 3 3 2 2 1 8 0 \ - " - - # --custom-recompute-layers-per-stage 5 5 5 5 5 5 5 5 4 4 4 4 3 2 2 0 \ \ - # --no-gradient-accumulation-fusion \ - # --recompute-num-layers 10 \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - -DATA_ARGS=" - --data-path $DATA_PATH \ - --data-impl mmap \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 98,2,0 -" - -NETWORK_ARGS=" - --num-layers 80 \ - --hidden-size 8192 \ - --ffn-hidden-size 28672 \ - --num-attention-heads 64 \ - --group-query-attention \ - --num-query-groups 8 \ - --seq-length 4096 \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights \ - --swiglu \ - --normalization RMSNorm \ - --no-masked-softmax-fusion -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - -megatron_args="$TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS" - -function exec_ssh_by_master -{ - # only at master host, start all other non master hosts run - if [[ "$HOST_IP" == "${ADDR_ARRAY[0]}" ]] - then - for i in "${!ADDR_ARRAY[@]}" - do - if [ "$i" != "0" ] - then - scp ${CUR_SCR} ${HOST_NAME}@${ADDR_ARRAY[$i]}:${CURRENT_DIR} - # scp -r ${DATA_PATH} ${HOST_NAME}@${ADDR_ARRAY[$i]}:${DATA_PATH}/../ - - ssh ${HOST_NAME}@${ADDR_ARRAY[$i]} "docker exec ${CONTAINER_NAME} bash -c \"cd ${CURRENT_DIR}; bash ${CUR_SCR} \"" & - fi - done - fi -} -function run_ddp_mm() -{ - for i in "${!ADDR_ARRAY[@]}" - do - if [[ "$HOST_IP" == "${ADDR_ARRAY[$i]}" ]] - then - echo "nodes: ${#ADDR_ARRAY[@]}, rank: $i, IP: $HOST_IP, MASTER_IP: ${ADDR_ARRAY[0]}" - DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NODES --node_rank $i --master_addr ${ADDR_ARRAY[0]} --master_port 54321" - torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - ${megatron_args} | tee ${LOG_PATH}/output.log 2>&1 - fi - done -} -exec_ssh_by_master -run_ddp_mm \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_tpoverlap_profiling_node1.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_tpoverlap_profiling_node1.sh deleted file mode 100644 index 695a6fdcf..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_70b_tpoverlap_profiling_node1.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables -# base on the cluster configuration -export CUDA_DEVICE_MAX_CONNECTIONS=1 -# export NCCL_SOCKET_IFNAME=eth0 -# export NCCL_IB_DISABLE=0 -# export NCCL_IB_CUDA_SUPPORT=1 -# export NCCL_IB_GID_INDEX=0 -# export NCCL_IB_HCA=mlx5_0,mlx5_3 -# export NCCL_DEBUG=debug -# export OMP_NUM_THREADS=4 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -TRANSFORMER_IMPL=transformer_engine -# export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - -export ENABLE_TORCH_TP_OVERLAP=1 -export TORCH_TP_OVERLAP_SIZE=4 -export NCCL_USE_HIGHPRIORITYWARP=1 - -export NCCL_FORCESYNC_DISABLE=1 -export NCCL_USE_DIRECT=1 -export OMP_NUM_THREADS=4 -export UMD_CCLINLASTCE=1 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 4 \ - --pipeline-model-parallel-size 4 \ - --sequence-parallel \ - --micro-batch-size 1 \ - --global-batch-size 32 \ - --disable-bias-linear \ - --use-flash-attn \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL\ - --use-distributed-optimizer \ - --no-gradient-accumulation-fusion \ -" -## 自定义recompute layers pp stage - # --recompute-granularity full \ - # --recompute-method block \ - # --custom-recompute-layers-per-stage 3 1 0 0 \ - -## 自定义切分pp stage，仅针对transformer layers - # --custom-partition 3 3 4 4 4 4 5 5 \ - -# --use-distributed-optimizer \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" -## 模型原参数：num-layers=80 -NETWORK_ARGS=" - --num-layers 16 \ - --hidden-size 8192 \ - --num-attention-heads 64 \ - --seq-length 4096 \ - --ffn-hidden-size 28672 \ - --num-query-groups 8 \ - --group-query-attention \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - # --wandb-save-dir $WB_PATH \ - # --tensorboard-dir $TB_PATH \ - # --tensorboard-log-interval 1 - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_node1.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_node1.sh deleted file mode 100644 index 13473b09d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_node1.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables base on the cluster configuration -export OMP_NUM_THREADS=4 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_SOCKET_IFNAME=bond0 -# export NCCL_USE_DIRECT=1 - -export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -TRANSFORMER_IMPL=transformer_engine - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 4 \ - --micro-batch-size 1 \ - --global-batch-size 1024 \ - --disable-bias-linear \ - --use-flash-attn \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL \ - --no-fp8-wgrad \ - --use-distributed-optimizer \ - --recompute-granularity full \ - --recompute-method block \ - --recompute-num-layers 1 \ - --recompute-method-per-stage 4 1 \ - --recompute-num-layers-per-stage 1 1 3 0 \ -" -## 自定义recompute layers pp stage - # --recompute-granularity full \ - # --recompute-method block \ - # --custom-recompute-layers-per-stage 2 0 0 0 \ - -## 自定义切分pp stage，仅针对transformer layers - # --custom-partition 3 3 4 4 4 4 5 5 \ - -# --use-distributed-optimizer \ -# --overlap-grad-reduce \ - - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers 32 \ - --hidden-size 4096 \ - --num-attention-heads 32 \ - --seq-length 4096 \ - --ffn-hidden-size 11008 \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - # --wandb-save-dir $WB_PATH \ - # --tensorboard-dir $TB_PATH \ - # --tensorboard-log-interval 1 - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_tpoverlap_profiling_node1.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_tpoverlap_profiling_node1.sh deleted file mode 100644 index 83ee4555d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_llama2_7b_tpoverlap_profiling_node1.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables -# base on the cluster configuration -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_SOCKET_IFNAME=bond0 -# export NCCL_IB_DISABLE=0 -# export NCCL_IB_CUDA_SUPPORT=1 -# export NCCL_IB_GID_INDEX=0 -# export NCCL_IB_HCA=mlx5_0,mlx5_3 -# export NCCL_DEBUG=debug -# export OMP_NUM_THREADS=4 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -TRANSFORMER_IMPL=transformer_engine - -export ENABLE_TORCH_TP_OVERLAP=1 -export TORCH_TP_OVERLAP_SIZE=4 -export NCCL_USE_HIGHPRIORITYWARP=1 - -export NCCL_FORCESYNC_DISABLE=1 -export NCCL_USE_DIRECT=1 -export OMP_NUM_THREADS=4 -export UMD_CCLINLASTCE=1 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 4 \ - --pipeline-model-parallel-size 4 \ - --sequence-parallel \ - --micro-batch-size 1 \ - --global-batch-size 32 \ - --disable-bias-linear \ - --use-flash-attn \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL\ - --use-distributed-optimizer \ - --no-gradient-accumulation-fusion \ -" -## 自定义recompute layers pp stage - # --recompute-granularity full \ - # --recompute-method block \ - # --custom-recompute-layers-per-stage 3 1 0 0 \ - -## 自定义切分pp stage，仅针对transformer layers - # --custom-partition 3 3 4 4 4 4 5 5 \ - -# --use-distributed-optimizer \ - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers 32 \ - --hidden-size 4096 \ - --num-attention-heads 32 \ - --seq-length 4096 \ - --ffn-hidden-size 11008 \ - --max-position-embeddings 4096 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - # --wandb-save-dir $WB_PATH \ - # --tensorboard-dir $TB_PATH \ - # --tensorboard-log-interval 1 - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_pp_overlap_node1_card8.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_pp_overlap_node1_card8.sh deleted file mode 100644 index 4d93083e8..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_pp_overlap_node1_card8.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash - -# Please change the following environment variables -# base on the cluster configuration -export OMP_NUM_THREADS=4 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -# export NCCL_SOCKET_IFNAME=ens5f0 - -# torch tp overlap -export ENABLE_TORCH_TP_OVERLAP=1 -export TORCH_TP_OVERLAP_SIZE=2 - -# # torch pp overlap -export ENABLE_TORCH_PP_OVERLAP=1 -export TORCH_PP_OVERLAP_SIZE=2 - -# following environment variables must be set when ENABLE_TORCH_TP_OVERLAP=1 -# export NCCL_FORCESYNC_DISABLE=1 ## this variable may cause hang and nan -export NCCL_USE_DIRECT=1 -export OMP_NUM_THREADS=4 -export UMD_CCLINLASTCE=1 - - -PROJ_HOME=$(dirname $(dirname "$PWD")) -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH -TRANSFORMER_IMPL=transformer_engine - -# Change for multinode config -# export NODE_ADDR=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2;}'|tr -d "addr:"|head -n 1) -# export GPUS_PER_NODE=$(awk '{$1=$1;print}' $HOSTFILE|awk -F" |=" '{ranks[$1]=$NF;}END{print ranks["'$NODE_ADDR'"];}') -# export NNODES=$(awk '{$1=$1;print}' $HOSTFILE | wc -l) -# export MASTER_ADDR=$(head -n1 $HOSTFILE | awk '{print $1;}') -# export NODE_RANK=$(awk '{ranks[$1]=(FNR-1);}END{print ranks["'$NODE_ADDR'"];}' $HOSTFILE) -# export MASTER_PORT=12346 -# WORLD_SIZE=$(($GPUS_PER_NODE * $NNODES)) - -TP=2 -PP=4 -GPUS_PER_NODE=8 -MASTER_ADDR=localhost -MASTER_PORT=8081 -NNODES=1 -NODE_RANK=0 - - -# llama2-7b -HIDDEN_SIZE=4096 -FFN_HIDDEN_SIZE=11008 -NUM_LAYERS=16 -NUM_HEADS=32 -SEQ_LENGTH=4096 -NUM_KV_HEADS=32 - -MICRO_BATCH_SIZE=2 -GLOBAL_BATCH_SIZE=16 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" -## Follow params must be set when use torch pp overlap -TORCH_PP_OVERLAP_ARGS=" - --pp-delay \ - --pp-split-size 4 \ - --num-layers-per-virtual-pipeline-stage 2 \ - --sequence-parallel \ -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size ${TP} \ - --pipeline-model-parallel-size ${PP} \ - --micro-batch-size $MICRO_BATCH_SIZE \ - --global-batch-size $GLOBAL_BATCH_SIZE \ - --disable-bias-linear \ - --eval-interval 1000 \ - --use-flash-attn \ - --bf16 \ - --transformer-impl $TRANSFORMER_IMPL\ - --no-gradient-accumulation-fusion \ -" -# --use-distributed-optimizer \ - -# MIXED_PRECISION_ARGS=" -# --bf16 \ -# --initial-loss-scale 522893 \ -# --min-loss-scale 1.0 \ -# --attention-softmax-in-fp32 -# " -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers $NUM_LAYERS \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads $NUM_HEADS \ - --num-key-value-heads $NUM_KV_HEADS \ - --seq-length $SEQ_LENGTH \ - --max-position-embeddings $SEQ_LENGTH \ - --norm-epsilon 1e-5 \ - --swiglu \ - --normalization RMSNorm \ - --ffn-hidden-size $FFN_HIDDEN_SIZE \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.3 \ - --hidden-dropout 0.3 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $TORCH_PP_OVERLAP_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd diff --git a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_tp_overlap_node1_card2.sh b/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_tp_overlap_node1_card2.sh deleted file mode 100644 index dcc818015..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama2/run_te_torch_tp_overlap_node1_card2.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/bin/bash - -# Please change the following environment variables -# base on the cluster configuration -export OMP_NUM_THREADS=4 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -# export NCCL_SOCKET_IFNAME=ens5f0 - -export ENABLE_TORCH_TP_OVERLAP=1 -export TORCH_TP_OVERLAP_SIZE=2 - -# following environment variables must be set when ENABLE_TORCH_TP_OVERLAP=1 -export NCCL_FORCESYNC_DISABLE=1 -export NCCL_USE_DIRECT=1 -export OMP_NUM_THREADS=4 -export UMD_CCLINLASTCE=1 - - -PROJ_HOME=$(dirname $(dirname "$PWD")) -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH -TRANSFORMER_IMPL=transformer_engine -# TB_PATH=./tboard/$DATE -# mkdir -p $TB_PATH -# WB_PATH=./wandb/$DATE -# mkdir -p $WB_PATH - -# Change for multinode config -# export NODE_ADDR=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2;}'|tr -d "addr:"|head -n 1) -# export GPUS_PER_NODE=$(awk '{$1=$1;print}' $HOSTFILE|awk -F" |=" '{ranks[$1]=$NF;}END{print ranks["'$NODE_ADDR'"];}') -# export NNODES=$(awk '{$1=$1;print}' $HOSTFILE | wc -l) -# export MASTER_ADDR=$(head -n1 $HOSTFILE | awk '{print $1;}') -# export NODE_RANK=$(awk '{ranks[$1]=(FNR-1);}END{print ranks["'$NODE_ADDR'"];}' $HOSTFILE) -# export MASTER_PORT=12346 -# WORLD_SIZE=$(($GPUS_PER_NODE * $NNODES)) - -TP=2 -PP=1 -GPUS_PER_NODE=2 -MASTER_ADDR=localhost -MASTER_PORT=8081 -NNODES=1 -NODE_RANK=0 - - -# llama2-7b -HIDDEN_SIZE=4096 -FFN_HIDDEN_SIZE=11008 -NUM_LAYERS=4 -NUM_HEADS=32 -SEQ_LENGTH=4096 -NUM_KV_HEADS=32 - -MICRO_BATCH_SIZE=2 -GLOBAL_BATCH_SIZE=2 - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size ${TP} \ - --pipeline-model-parallel-size ${PP} \ - --micro-batch-size $MICRO_BATCH_SIZE \ - --global-batch-size $GLOBAL_BATCH_SIZE \ - --disable-bias-linear \ - --eval-interval 1000 \ - --use-flash-attn - --bf16 - --transformer-impl $TRANSFORMER_IMPL\ - --no-gradient-accumulation-fusion \ -" - # --sequence-parallel \ - # --use-distributed-optimizer \ - -# MIXED_PRECISION_ARGS=" -# --bf16 \ -# --initial-loss-scale 522893 \ -# --min-loss-scale 1.0 \ -# --attention-softmax-in-fp32 -# " -# --accumulate-allreduce-grads-in-fp32 - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama2Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers $NUM_LAYERS \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads $NUM_HEADS \ - --num-key-value-heads $NUM_KV_HEADS \ - --seq-length $SEQ_LENGTH \ - --max-position-embeddings $SEQ_LENGTH \ - --norm-epsilon 1e-5 \ - --swiglu \ - --normalization RMSNorm \ - --ffn-hidden-size $FFN_HIDDEN_SIZE \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights -" -## group attntion parameters for megatron-lm -## example llama2-70B -# --num-attention-heads 64 -# --group-query-attention -# --num-query-groups 8 - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.3 \ - --hidden-dropout 0.3 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - # --wandb-save-dir $WB_PATH \ - # --tensorboard-dir $TB_PATH \ - # --tensorboard-log-interval 1 - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd diff --git a/toolbox/Megatron-DeepSpeed/examples/llama3/run_te_llama3_8b_node1.sh b/toolbox/Megatron-DeepSpeed/examples/llama3/run_te_llama3_8b_node1.sh deleted file mode 100644 index 82f37b55a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/llama3/run_te_llama3_8b_node1.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash - -# Please change the following envrioment variables base on the cluster configuration -export OMP_NUM_THREADS=4 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NCCL_SOCKET_IFNAME=bond0 -# export NCCL_USE_DIRECT=1 - -# export ENABLE_FLASH_ATTENTION_WITH_IXDNN=1 - -PROJ_HOME=$(dirname $(dirname "$PWD")) - -DATA_PATH=${PROJ_HOME}/dataset/gpt_small_117M_llama3/gpt_small_117M_text_document -TOKENIZER_PATH=./tokenizer/tokenizer_llama3.model - -CHECKPOINT_PATH=./checkpoints/llama2 -mkdir -p $CHECKPOINT_PATH - -DATE=`date +%y%m%d%H%M%S` -LOG_PATH=./logs/$DATE -mkdir -p $LOG_PATH - -GPUS_PER_NODE=16 -MASTER_ADDR=localhost -MASTER_PORT=8080 -NNODES=1 -NODE_RANK=0 - -TRANSFORMER_IMPL=transformer_engine - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -## llama3-8B 与 llama2-7B之间有差别的参数 - # --group-query-attention \ - # --num-query-groups 8 \ - # --seq-length 8192 \ - # --max-position-embeddings 8192 \ - # --rotary-position-embeddings-theta 500000 \ - # --ffn-hidden-size 14336 \ - # --tokenizer-type Llama3Tokenizer \ - # vocab_size=128256 不用在脚本里设置 - -TRAINING_ARGS=" - --train-iters 250000 \ - --eval-iters 10 \ - --tensor-model-parallel-size 2 \ - --pipeline-model-parallel-size 8 \ - --micro-batch-size 1 \ - --global-batch-size 32 \ - --disable-bias-linear \ - --use-flash-attn \ - --eval-interval 1000 \ - --transformer-impl $TRANSFORMER_IMPL \ - --no-fp8-wgrad \ - --use-distributed-optimizer \ -" - -MIXED_PRECISION_ARGS=" - --bf16 \ - --initial-loss-scale 522893 \ - --min-loss-scale 1.0 \ - --attention-softmax-in-fp32 \ - --no-query-key-layer-scaling -" - - -DATA_ARGS=" - --data-path $DATA_PATH \ - --tokenizer-type Llama3Tokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 -" - -NETWORK_ARGS=" - --num-layers 32 \ - --hidden-size 4096 \ - --num-attention-heads 32 \ - --group-query-attention \ - --num-query-groups 8 \ - --seq-length 8192 \ - --max-position-embeddings 8192 \ - --ffn-hidden-size 14336 \ - --norm-epsilon 1e-5 \ - --use-rotary-position-embeddings \ - --no-position-embedding \ - --swiglu \ - --normalization RMSNorm \ - --untie-embeddings-and-output-weights \ - --rotary-position-embeddings-theta 500000 \ -" - -INITIALIZATION_ARGS=" - --init-method-std 0.02 \ - --seed 1234 -" - -REGULARIZATION_ARGS=" - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --clip-grad 1.0 -" - -LEARNING_RATE_ARGS=" - --lr 3.0e-4 \ - --min-lr 3.0e-5 \ - --lr-decay-style cosine \ - --lr-warmup-iters 2000 -" - -CHECKPOINTING_ARGS=" - --save-interval 10000 \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH -" - -LOGGING_ARGS=" - --log-interval 1 \ -" - # --wandb-save-dir $WB_PATH \ - # --tensorboard-dir $TB_PATH \ - # --tensorboard-log-interval 1 - -cmd="torchrun $DISTRIBUTED_ARGS $PROJ_HOME/pretrain_gpt_megatron.py \ - $TRAINING_ARGS \ - $MIXED_PRECISION_ARGS \ - $DATA_ARGS \ - $NETWORK_ARGS \ - $INITIALIZATION_ARGS \ - $REGULARIZATION_ARGS \ - $LEARNING_RATE_ARGS \ - $CHECKPOINTING_ARGS \ - $LOGGING_ARGS | tee ${LOG_PATH}/output.log 2>&1 - " -echo $cmd -eval $cmd \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples/merge_mp_bert.sh b/toolbox/Megatron-DeepSpeed/examples/merge_mp_bert.sh deleted file mode 100644 index 138343328..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/merge_mp_bert.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -TENSOR_MODEL_PARALLEL_SIZE=2 - -VOCAB_FILE=bert-vocab.txt -CHECKPOINT_PATH=checkpoints/bert_345m - -WORLD_SIZE=$TENSOR_MODEL_PARALLEL_SIZE python tools/merge_mp_partitions.py \ - --model-type BERT \ - --tensor-model-parallel-size $TENSOR_MODEL_PARALLEL_SIZE \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file $VOCAB_FILE \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/README.md b/toolbox/Megatron-DeepSpeed/examples/msdp/README.md deleted file mode 100644 index 8ff95099e..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/README.md +++ /dev/null @@ -1,5 +0,0 @@ - -# Multi-Stage Prompting for Knowledgeable Dialogue Generation - -This directory contains all the scripts of multi-stage prompting for knowledgeable dialogue generation that includes data preparation, and knowledge and response generations. More details are available on [`knowledgeable task directory`](../../tasks/msdp). - diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/data_processing.sh b/toolbox/Megatron-DeepSpeed/examples/msdp/data_processing.sh deleted file mode 100644 index 37a6512a8..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/data_processing.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Data preparation for our framework: preprocessing the WoW and WoI datasets -# The datasets can be downloaded through the following links: -# WoW: https://parl.ai/projects/wizard_of_wikipedia/ -# WoI: https://parl.ai/projects/sea/ - -DIR=`pwd` -# Before running the preprocessing, please download -# the wizard of wikipedia and wizard datasets -WOW_DATA_FOLDER= -WOI_DATA_FOLDER= - -# We provide examples for processing the raw data from Wizard of Wikipedia -# Processing the train dataset (train.json) -python ${DIR}/tasks/msdp/preprocessing.py \ - --func process_wow_dataset \ - --raw_file ${WOW_DATA_FOLDER}/train.json \ - --processed_file ${WOW_DATA_FOLDER}/train_processed.txt - -# Processing test seen dataset (test_random_split.json) -python ${DIR}/tasks/msdp/preprocessing.py \ - --func process_wow_dataset \ - --raw_file ${WOW_DATA_FOLDER}/test_random_split.json \ - --processed_file ${WOW_DATA_FOLDER}/testseen_processed.txt \ - --knwl_ref_file ${WOW_DATA_FOLDER}/output_testseen_knowledge_reference.txt \ - --resp_ref_file ${WOW_DATA_FOLDER}/output_testseen_response_reference.txt - -# processing test unseen dataset (test_topic_split.json) -python ${DIR}/tasks/msdp/preprocessing.py \ - --func process_wow_dataset \ - --raw_file ${WOW_DATA_FOLDER}/test_topic_split.json \ - --processed_file ${WOW_DATA_FOLDER}/testunseen_processed.txt \ - --knwl_ref_file ${WOW_DATA_FOLDER}/output_testunseen_knowledge_reference.txt \ - --resp_ref_file ${WOW_DATA_FOLDER}/output_testunseen_response_reference.txt - - -# We provide the following script to process the raw data from Wizard of Internet -# Processing the test dataset (test.jsonl) -python ${DIR}/tasks/msdp/preprocessing.py \ - --func process_woi_dataset \ - --raw_file ${WOI_DATA_FOLDER}/test.jsonl \ - --processed_file ${WOI_DATA_FOLDER}/test_processed.txt \ - --knwl_ref_file ${WOI_DATA_FOLDER}/output_test_knowledge_reference.txt \ - --resp_ref_file ${WOI_DATA_FOLDER}/output_test_response_reference.txt - - -# Get the knowledge generation prompts for the each test dataset in WoW and WoI -MODEL_FILE= -# WoW test seen -python ${DIR}/tasks/msdp/preprocessing.py \ - --func get_knwl_gen_prompts \ - --test_file ${WOW_DATA_FOLDER}/testseen_processed.txt \ - --train_file ${WOW_DATA_FOLDER}/train_processed.txt \ - --model_file ${MODEL_FILE} \ - --processed_file ${WOW_DATA_FOLDER}/output_testseen_knowledge_prompts.json \ - --data_type wow_seen - -# WoW test unseen -python ${DIR}/tasks/msdp/preprocessing.py \ - --func get_knwl_gen_prompts \ - --test_file ${WOW_DATA_FOLDER}/testunseen_processed.txt \ - --train_file ${WOW_DATA_FOLDER}/train_processed.txt \ - --model_file ${MODEL_FILE} \ - --processed_file ${WOW_DATA_FOLDER}/output_testunseen_knowledge_prompts.json \ - --data_type wow_unseen - -# WoI -python ${DIR}/tasks/msdp/preprocessing.py \ - --func get_knwl_gen_prompts \ - --test_file ${WOI_DATA_FOLDER}/test_processed.txt \ - --train_file ${WOW_DATA_FOLDER}/train_processed.txt \ - --model_file ${MODEL_FILE} \ - --processed_file ${WOI_DATA_FOLDER}/output_test_knowledge_prompts.json \ - --data_type woi - - -# Get the response generation prompts (can be applied for all the test datasets) -python ${DIR}/tasks/msdp/preprocessing.py \ - --func get_resp_gen_prompts \ - --train_file ${WOW_DATA_FOLDER}/train_processed.txt \ - --processed_file ${WOW_DATA_FOLDER}/output_response_prompts.txt - diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/eval_knwl_generation.sh b/toolbox/Megatron-DeepSpeed/examples/msdp/eval_knwl_generation.sh deleted file mode 100644 index 8fc2fff1f..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/eval_knwl_generation.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -######################### -# Evaluate the F1 scores. -######################### - -WORLD_SIZE=1 -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -MODEL_GEN_PATH= \ - (e.g., /testseen_knowledge_generations.txt) -GROUND_TRUTH_PATH= \ - (e.g., /testseen_knowledge_reference.txt) - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/msdp/main.py \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --micro-batch-size 4 \ - --task MSDP-EVAL-F1 \ - --guess-file ${MODEL_GEN_PATH} \ - --answer-file ${GROUND_TRUTH_PATH} - - -############################################ -# Evaluate BLEU, METEOR, and ROUGE-L scores. -############################################ - -# We follow the nlg-eval (https://github.com/Maluuba/nlg-eval) to -# evaluate the BLEU, METEOR, and ROUGE-L scores. - -# To evaluate on these metrics, please setup the environments based on -# the nlg-eval github, and run the corresponding evaluation commands. - -nlg-eval \ - --hypothesis= \ - --references= diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/eval_resp_generation.sh b/toolbox/Megatron-DeepSpeed/examples/msdp/eval_resp_generation.sh deleted file mode 100644 index 3ce87e077..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/eval_resp_generation.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -######################### -# Evaluate the F1 scores. -######################### - -WORLD_SIZE=1 -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -MODEL_GEN_PATH= \ - (e.g., /testseen_response_generations.txt) -GROUND_TRUTH_PATH= \ - (e.g., /testseen_response_reference.txt) - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/msdp/main.py \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --micro-batch-size 4 \ - --task MSDP-EVAL-F1 \ - --guess-file ${MODEL_GEN_PATH} \ - --answer-file ${GROUND_TRUTH_PATH} - - -########################## -# Evaluate the KF1 scores. -########################## - -MODEL_GEN_PATH= \ - (e.g., /testseen_response_generations.txt) -GROUND_TRUTH_PATH= \ - (e.g., /testseen_knowledge_reference.txt) - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/msdp/main.py \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --micro-batch-size 4 \ - --task MSDP-EVAL-F1 \ - --guess-file ${MODEL_GEN_PATH} \ - --answer-file ${GROUND_TRUTH_PATH} - - -############################################ -# Evaluate BLEU, METEOR, and ROUGE-L scores. -############################################ - -# We follow the nlg-eval (https://github.com/Maluuba/nlg-eval) to -# evaluate the BLEU, METEOR, and ROUGE-L scores. - -# To evaluate on these metrics, please setup the environments based on -# the nlg-eval github, and run the corresponding evaluation commands. - -nlg-eval \ - --hypothesis= \ - --references= diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/prep_resp_gen.sh b/toolbox/Megatron-DeepSpeed/examples/msdp/prep_resp_gen.sh deleted file mode 100644 index 5f202724d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/prep_resp_gen.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Preparing the input file for the response generation (second-stage prompting) - -DIR=`pwd` - -TEST_FILE= \ - (e.g., /testseen_processed.txt) -KNOWLEDGE_FILE= \ - (e.g., /testseen_knowledge_generations.txt) -PROCESSED_FILE= \ - (e.g., /testseen_processed_with_generated_knowledge.txt) - -python ${DIR}/tasks/msdp/preprocessing.py \ - --func prepare_input \ - --test_file ${TEST_FILE} \ - --knwl_gen_file ${KNOWLEDGE_FILE} \ - --processed_file ${PROCESSED_FILE} diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/prompt_knwl_gen.sh b/toolbox/Megatron-DeepSpeed/examples/msdp/prompt_knwl_gen.sh deleted file mode 100644 index 12e0cc5b3..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/prompt_knwl_gen.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Stage-1: Prompt a pretrained language model to generate the context-relevant knowledge -# The input contains prompts and current dialogue context, the output is the relevant knowledge -# The size of the pretrained language model is 357M - -WORLD_SIZE=8 - -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT_PATH= (e.g., /357m) -VOCAB_PATH= (e.g., /gpt2-vocab.json) -MERGE_PATH= (e.g., /gpt2-merges.txt) -INPUT_PATH= \ - (e.g., /testseen_processed.txt) -PROMPT_PATH= \ - (e.g., /testseen_knowledge_prompts.json) -OUTPUT_PATH= \ - (e.g., /testseen_knowledge_generations.txt) - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/msdp/main.py \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --micro-batch-size 1 \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --load ${CHECKPOINT_PATH} \ - --fp16 \ - --DDP-impl torch \ - --tokenizer-type GPT2BPETokenizer \ - --sample-input-file ${INPUT_PATH} \ - --sample-output-file ${OUTPUT_PATH} \ - --prompt-file ${PROMPT_PATH} \ - --prompt-type knowledge \ - --num-prompt-examples 10 \ - --task MSDP-PROMPT - -# NOTE: If you use api for the model generation, please use -# the "--api-prompt" flag (setting this value as True). diff --git a/toolbox/Megatron-DeepSpeed/examples/msdp/prompt_resp_gen.sh b/toolbox/Megatron-DeepSpeed/examples/msdp/prompt_resp_gen.sh deleted file mode 100644 index b836d7fea..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/msdp/prompt_resp_gen.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Stage-2: Prompt a pretrained language model to generate the corresponding response -# The input contains prompts, current dialogue context, and generated knowledge in Stage-1 -# The output is the corresponding response. -# The size of the pretrained language model is 357M - -WORLD_SIZE=8 - -DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT_PATH= (e.g., /357m) -VOCAB_PATH= (e.g., /gpt2-vocab.json) -MERGE_PATH= (e.g., /gpt2-merges.txt) -INPUT_PATH= (e.g., /testseen_processed.txt) -PROMPT_PATH= \ - (e.g., /response_prompts.txt) -OUTPUT_PATH= \ - (e.g., /output_testseen_response_generations.txt) - -python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/msdp/main.py \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --micro-batch-size 1 \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --load ${CHECKPOINT_PATH} \ - --fp16 \ - --DDP-impl torch \ - --tokenizer-type GPT2BPETokenizer \ - --sample-input-file ${INPUT_PATH} \ - --sample-output-file ${OUTPUT_PATH} \ - --prompt-file ${PROMPT_PATH} \ - --prompt-type response \ - --num-prompt-examples 20 \ - --task MSDP-PROMPT - -# NOTE: If you use api for the model generation, please use -# the "--api-prompt" flag (setting this value as True). diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_bert.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_bert.sh deleted file mode 100644 index c98c7ebbd..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_bert.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -CHECKPOINT_PATH= -VOCAB_FILE=/bert-vocab.txt -DATA_PATH=_text_sentence - -BERT_ARGS=" - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --micro-batch-size 4 \ - --global-batch-size 8 \ - --lr 0.0001 \ - --train-iters 2000000 \ - --lr-decay-iters 990000 \ - --lr-decay-style linear \ - --min-lr 0.00001 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun pretrain_bert.py \ - $BERT_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed.sh deleted file mode 100644 index 4a87a7bfb..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -GPUS_PER_NODE=8 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -CHECKPOINT_PATH= -VOCAB_FILE=/bert-vocab.txt -DATA_PATH=_text_sentence - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -BERT_ARGS=" - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --micro-batch-size 4 \ - --global-batch-size 32 \ - --lr 0.0001 \ - --train-iters 1000000 \ - --lr-decay-iters 990000 \ - --lr-decay-style linear \ - --min-lr 1.0e-5 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun $DISTRIBUTED_ARGS pretrain_bert.py \ - $BERT_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --distributed-backend nccl \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed_with_mp.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed_with_mp.sh deleted file mode 100644 index 62d7f741c..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_bert_distributed_with_mp.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -GPUS_PER_NODE=8 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -CHECKPOINT_PATH= -VOCAB_FILE=/bert-vocab.txt -DATA_PATH=_text_sentence - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -BERT_ARGS=" - --tensor-model-parallel-size 2 \ - --pipeline-model-parallel-size 2 \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 512 \ - --max-position-embeddings 512 \ - --micro-batch-size 2 \ - --global-batch-size 16 \ - --lr 0.0001 \ - --train-iters 1000000 \ - --lr-decay-iters 990000 \ - --lr-decay-style linear \ - --min-lr 1.0e-5 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun $DISTRIBUTED_ARGS pretrain_bert.py \ - $BERT_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --distributed-backend nccl \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt.sh deleted file mode 100644 index 4956d26ff..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -# Runs the "345M" parameter model - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -CHECKPOINT_PATH= -VOCAB_FILE=/gpt2-vocab.json -MERGE_FILE=/gpt2-merges.txt -DATA_PATH=_text_document - -GPT_ARGS=" - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 1024 \ - --max-position-embeddings 1024 \ - --micro-batch-size 4 \ - --global-batch-size 8 \ - --lr 0.00015 \ - --train-iters 500000 \ - --lr-decay-iters 320000 \ - --lr-decay-style cosine \ - --min-lr 1.0e-5 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun pretrain_gpt.py \ - $GPT_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt3_175B.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt3_175B.sh deleted file mode 100644 index b423e4bd1..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt3_175B.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - - -#SBATCH --nodes=128 --exclusive --ntasks-per-node=8 --job-name=megatron_gpt3_175b - - -DIR=`pwd` -DATETIME=`date +'date_%y-%m-%d_time_%H-%M-%S'` -mkdir -p $DIR/logs - - -DATASET_1="" -DATASET_2="" -DATASET_3="" -DATASET="0.2 ${DATASET_1} 0.3 ${DATASET_2} 0.5 ${DATASET_3}" - - -options=" \ - --tensor-model-parallel-size 8 \ - --pipeline-model-parallel-size 16 \ - --num-layers 96 \ - --hidden-size 12288 \ - --num-attention-heads 96 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --micro-batch-size 1 \ - --global-batch-size 1536 \ - --rampup-batch-size 16 16 5859375 \ - --train-samples 146484375 \ - --lr-decay-samples 126953125 \ - --lr-warmup-samples 183105 \ - --lr 6.0e-5 \ - --min-lr 6.0e-6 \ - --lr-decay-style cosine \ - --log-interval 10 \ - --eval-iters 40 \ - --eval-interval 1000 \ - --data-path ${DATASET} \ - --vocab-file \ - --merge-file \ - --save-interval 1000 \ - --save \ - --load \ - --split 98,2,0 \ - --clip-grad 1.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --init-method-std 0.006 \ - --tensorboard-dir \ - --fp16 \ - --activations-checkpoint-method uniform " - - -run_cmd="python -u ${DIR}/pretrain_gpt.py $@ ${options}" - - -srun -l \ - --container-image "nvcr.io/nvidia/pytorch:20.12-py3" \ - --container-mounts "" \ - --output=$DIR/logs/%x_%j_$DATETIME.log sh -c "${run_cmd}" - - -set +x - diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed.sh deleted file mode 100644 index 24d76a1dc..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -# Runs the "345M" parameter model - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -GPUS_PER_NODE=8 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -CHECKPOINT_PATH= -VOCAB_FILE=/gpt2-vocab.json -MERGE_FILE=/gpt2-merges.txt -DATA_PATH=_text_document - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -GPT_ARGS=" - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 1024 \ - --max-position-embeddings 1024 \ - --micro-batch-size 8 \ - --global-batch-size 64 \ - --lr 0.00015 \ - --train-iters 500000 \ - --lr-decay-iters 320000 \ - --lr-decay-style cosine \ - --min-lr 1.0e-5 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ - $GPT_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --distributed-backend nccl \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed_with_mp.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed_with_mp.sh deleted file mode 100644 index 721288fdb..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_gpt_distributed_with_mp.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Runs the "345M" parameter model - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -GPUS_PER_NODE=8 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -CHECKPOINT_PATH= -VOCAB_FILE=/gpt2-vocab.json -MERGE_FILE=/gpt2-merges.txt -DATA_PATH=_text_document - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -GPT_ARGS=" - --tensor-model-parallel-size 2 \ - --pipeline-model-parallel-size 2 \ - --sequence-parallel \ - --num-layers 24 \ - --hidden-size 1024 \ - --num-attention-heads 16 \ - --seq-length 1024 \ - --max-position-embeddings 1024 \ - --micro-batch-size 4 \ - --global-batch-size 16 \ - --lr 0.00015 \ - --train-iters 500000 \ - --lr-decay-iters 320000 \ - --lr-decay-style cosine \ - --min-lr 1.0e-5 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ - $GPT_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --distributed-backend nccl \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH - diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_ict.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_ict.sh deleted file mode 100644 index 8cba0f08b..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_ict.sh +++ /dev/null @@ -1,44 +0,0 @@ -#! /bin/bash - -# Runs the "217M" parameter biencoder model for ICT retriever - -RANK=0 -WORLD_SIZE=1 - -PRETRAINED_BERT_PATH= -TEXT_DATA_PATH= -TITLE_DATA_PATH= -CHECKPOINT_PATH= - - -python pretrain_ict.py \ - --num-layers 12 \ - --hidden-size 768 \ - --num-attention-heads 12 \ - --tensor-model-parallel-size 1 \ - --micro-batch-size 32 \ - --seq-length 256 \ - --max-position-embeddings 512 \ - --train-iters 100000 \ - --vocab-file bert-vocab.txt \ - --tokenizer-type BertWordPieceLowerCase \ - --DDP-impl torch \ - --bert-load ${PRETRAINED_BERT_PATH} \ - --log-interval 100 \ - --eval-interval 1000 \ - --eval-iters 10 \ - --retriever-report-topk-accuracies 1 5 10 20 100 \ - --retriever-score-scaling \ - --load $CHECKPOINT_PATH \ - --save $CHECKPOINT_PATH \ - --data-path ${TEXT_DATA_PATH} \ - --titles-data-path ${TITLE_DATA_PATH} \ - --lr 0.0001 \ - --lr-decay-style linear \ - --weight-decay 1e-2 \ - --clip-grad 1.0 \ - --lr-warmup-fraction 0.01 \ - --save-interval 4000 \ - --exit-interval 8000 \ - --query-in-block-prob 0.1 \ - --fp16 diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_t5.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_t5.sh deleted file mode 100644 index 5f4b63ad6..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_t5.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -CHECKPOINT_PATH= -VOCAB_FILE=/t5-vocab.txt -DATA_PATH=_text_sentence - -T5_ARGS=" - --num-layers 12 \ - --hidden-size 768 \ - --num-attention-heads 12 \ - --kv-channels 64 \ - --ffn-hidden-size 3072 \ - --encoder-seq-length 512 \ - --decoder-seq-length 128 \ - --max-position-embeddings 512 \ - --micro-batch-size 16 \ - --global-batch-size 16 \ - --lr 0.0001 \ - --train-iters 1000000 \ - --lr-decay-iters 1000000 \ - --lr-decay-style linear \ - --min-lr 0.00001 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 \ - --vocab-extra-ids 100 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun pretrain_t5.py \ - $T5_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed.sh deleted file mode 100644 index eec524582..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -GPUS_PER_NODE=8 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -CHECKPOINT_PATH= -VOCAB_FILE=/t5-vocab.txt -DATA_PATH=_text_sentence - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -T5_ARGS=" - --num-layers 12 \ - --hidden-size 768 \ - --num-attention-heads 12 \ - --kv-channels 64 \ - --ffn-hidden-size 3072 \ - --encoder-seq-length 512 \ - --decoder-seq-length 128 \ - --max-position-embeddings 512 \ - --micro-batch-size 16 \ - --global-batch-size 128 \ - --lr 0.0001 \ - --train-iters 1000000 \ - --lr-decay-iters 1000000 \ - --lr-decay-style linear \ - --min-lr 0.00001 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 \ - --vocab-extra-ids 100 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun $DISTRIBUTED_ARGS pretrain_t5.py \ - $T5_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --distributed-backend nccl \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed_with_mp.sh b/toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed_with_mp.sh deleted file mode 100644 index d51ecee19..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/pretrain_t5_distributed_with_mp.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -GPUS_PER_NODE=8 -# Change for multinode config -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) - -CHECKPOINT_PATH= -VOCAB_FILE=/t5-vocab.txt -DATA_PATH=_text_sentence - -DISTRIBUTED_ARGS=" - --nproc_per_node $GPUS_PER_NODE \ - --nnodes $NNODES \ - --node_rank $NODE_RANK \ - --master_addr $MASTER_ADDR \ - --master_port $MASTER_PORT -" - -T5_ARGS=" - --tensor-model-parallel-size 2 \ - --num-layers 12 \ - --hidden-size 768 \ - --num-attention-heads 12 \ - --kv-channels 64 \ - --ffn-hidden-size 3072 \ - --encoder-seq-length 512 \ - --decoder-seq-length 128 \ - --max-position-embeddings 512 \ - --micro-batch-size 16 \ - --global-batch-size 128 \ - --lr 0.0001 \ - --train-iters 1000000 \ - --lr-decay-iters 1000000 \ - --lr-decay-style linear \ - --min-lr 0.00001 \ - --weight-decay 1e-2 \ - --lr-warmup-fraction .01 \ - --clip-grad 1.0 \ - --fp16 \ - --vocab-extra-ids 100 -" - -DATA_ARGS=" - --data-path $DATA_PATH \ - --vocab-file $VOCAB_FILE \ - --data-impl mmap \ - --split 949,50,1 -" - -OUTPUT_ARGS=" - --log-interval 100 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 -" - -torchrun $DISTRIBUTED_ARGS pretrain_t5.py \ - $T5_ARGS \ - $DATA_ARGS \ - $OUTPUT_ARGS \ - --distributed-backend nccl \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH diff --git a/toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M.sh b/toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M.sh deleted file mode 100644 index a151b9846..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# This example will start serving the 345M model. -DISTRIBUTED_ARGS="--nproc_per_node 1 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT= -VOCAB_FILE= -MERGE_FILE= - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -pip install flask-restful - -torchrun $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --load ${CHECKPOINT} \ - --num-attention-heads 16 \ - --max-position-embeddings 1024 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --micro-batch-size 1 \ - --seq-length 1024 \ - --out-seq-length 1024 \ - --temperature 1.0 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --top_p 0.9 \ - --seed 42 diff --git a/toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M_8_tensor_parallel.sh b/toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M_8_tensor_parallel.sh deleted file mode 100644 index 027ab4217..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/run_text_generation_server_345M_8_tensor_parallel.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# This example will start serving the 345M model that is partitioned 8 way tensor parallel -DISTRIBUTED_ARGS="--nproc_per_node 8 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT= -VOCAB_FILE= -MERGE_FILE= - -pip install flask-restful - -python -m torch.distributed.launch $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --tensor-model-parallel-size 8 \ - --pipeline-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --load ${CHECKPOINT} \ - --num-attention-heads 16 \ - --max-position-embeddings 1024 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --micro-batch-size 1 \ - --seq-length 1024 \ - --out-seq-length 1024 \ - --temperature 1.0 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --top_p 0.9 \ - --seed 42 diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/CONFIG.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/CONFIG.sh deleted file mode 100644 index f17ccd7b0..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/CONFIG.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - - -# SLURM options. -export SLURM_PARTITION= -export SLURM_ACCOUNT= - - -# Source code. -export MEGATRON_CODE_DIR= - - -# This variable is used to mount the relevant part of the filesystem -# inside the docker container. Note that the `MEGATRON_CODE_DIR` and the -# launch directory already get mounted; this variable should be used to -# mount the directories that contain the data and tokenizer files. -export DOCKER_MOUNT_DIR= - - -# Data and tokenizer files. -MEGATRON_DATA= -BPE_VOCAB_FILE= -BPE_MERGE_FILE= - - -# Megatron input parameters. -# `MEGATRON_EXTRA_PARAMS` can be used to provide any extra parameters -# that are not listed here. -export MEGATRON_PARAMS=" ${MEGATRON_EXTRA_PARAMS} \ - --tensor-model-parallel-size ${TP} \ - --pipeline-model-parallel-size ${PP} \ - --micro-batch-size ${MBS} \ - --global-batch-size ${GBS} \ - --num-layers ${NLS} \ - --hidden-size ${HS} \ - --num-attention-heads ${NAH} \ - --DDP-impl ${DDP} \ - --data-path ${MEGATRON_DATA} \ - --vocab-file ${BPE_VOCAB_FILE} \ - --merge-file ${BPE_MERGE_FILE} \ - --log-interval 5 \ - --seq-length 2048 \ - --max-position-embeddings 2048 \ - --train-iters 500 \ - --lr-decay-iters 320 \ - --lr 0.0001 \ - --min-lr 0.00001 \ - --lr-decay-style cosine \ - --lr-warmup-fraction 0.01 \ - --split 969,30,1 \ - --eval-iters 100 \ - --eval-interval 1000 \ - --clip-grad 1.0 \ - --fp16 \ - --loss-scale 8192 " - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/README.md b/toolbox/Megatron-DeepSpeed/examples/sc21/README.md deleted file mode 100644 index 940c37903..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Reproducing Figures in SC21 Paper - - -This directory contains some of the scripts that were used to produce the -results in the [Megatron paper](https://arxiv.org/pdf/2104.04473.pdf) that is -to appear at [SuperComputing 2021](https://sc21.supercomputing.org/). These -scripts use [Slurm](https://slurm.schedmd.com/documentation.html) with the -[pyxis plugin](https://github.com/NVIDIA/pyxis), but can be modified for other -schedulers as well. - - -## Setup - -All the cluster-dependent variables are in [`CONFIG.sh`](./CONFIG.sh). Please -update the unspecified values (in angle brackets `<...>`) before launching any -scripts. - - - -## Scripts - -Below is a list of scripts that can be used to reproduce various figures in our -[paper](https://arxiv.org/pdf/2104.04473.pdf): - -* [run_table_1.sh](./run_table_1.sh): Table 1 showing weak-scaling throughput -for GPT models ranging from 1 billion to 1 trillion parameters. -* [run_figure_11.sh](./run_figure_11.sh): Figure 11 showing the weak-scaling -performance of pipeline parallelism. -* [run_figure_12.sh](./run_figure_12.sh): Figure 12 showing the effect of -the interleaved schedule on a 175B GPT model. -* [run_figure_13.sh](./run_figure_13.sh): Figure 13 showing the effect of -different degrees of pipeline and tensor model parallelism on a model with -162.2 billion parameters. -* [run_figure_14.sh](./run_figure_14.sh): Figure 14 showing the effect of -different degrees of data and pipeline model parallelism on a model with -5.9 billion parameters. -* [run_figure_15.sh](./run_figure_15.sh): Figure 15 showing the effect of -different degrees of data and tensor model parallelism on a model with -5.9 billion parameters. -* [run_figure_16.sh](./run_figure_16.sh): Figure 16 showing the effect of -microbatch size. -* [run_figure_17.sh](./run_figure_17.sh): Figure 17 showing the effect of -activation recomputation. -* [run_figure_18.sh](./run_figure_18.sh): Figure 18 showing the effect of -the scatter-gather communication optimization. diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/SBATCH.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/SBATCH.sh deleted file mode 100644 index 95431b9b7..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/SBATCH.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - - -sbatch -p ${SLURM_PARTITION} \ - -A ${SLURM_ACCOUNT} \ - --job-name=${JOB_NAME} \ - --nodes=${NNODES} \ - --export=MEGATRON_CODE_DIR,MEGATRON_PARAMS,DOCKER_MOUNT_DIR SRUN.sh - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/SRUN.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/SRUN.sh deleted file mode 100644 index 52a9aff0c..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/SRUN.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -#SBATCH -t 0:30:00 --exclusive --mem=0 --overcommit --ntasks-per-node=8 - - -THIS_DIR=`pwd` -DATETIME=`date +'date_%y-%m-%d_time_%H-%M-%S'` -mkdir -p ${THIS_DIR}/logs - - -CMD="python -u ${MEGATRON_CODE_DIR}/pretrain_gpt.py ${MEGATRON_PARAMS}" - - -srun -l \ - --container-image "nvcr.io#nvidia/pytorch:20.12-py3" \ - --container-mounts "${THIS_DIR}:${THIS_DIR},${MEGATRON_CODE_DIR}:${MEGATRON_CODE_DIR},${DOCKER_MOUNT_DIR}:${DOCKER_MOUNT_DIR}" \ - --output=${THIS_DIR}/logs/%x_%j_$DATETIME.log sh -c "${CMD}" - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_11.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_11.sh deleted file mode 100644 index 2ec7d9eb3..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_11.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Pipeline-parallel size options = [1, 2, 4, 8]. -PP=1 - -# Batch size (global batch size) options = [8, 128]. -GBS=8 - - - - - -# Set pipeline-parallel size options. -NLS=$((3*PP)) -NNODES=${PP} - - -# Other params. -TP=8 -MBS=1 -HS=20480 -NAH=128 -DDP=local -MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " - - -# Name of the job. -export JOB_NAME=results_figure_11_pipeline_parallel_size_${PP}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_12.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_12.sh deleted file mode 100644 index 11e550854..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_12.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Interleaved schedule options = [YES, NO]. -INTERLEAVED=YES - -# Batch size (global batch size) options = [12, 24, 36, ..., 60]. -GBS=12 - - - - - -# Set interleaved schedule options. -if [ ${INTERLEAVED} == "YES" ]; then - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 2 " -elif [ ${INTERLEAVED} == "NO" ]; then - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -else - echo "Invalid configuration" - exit 1 -fi - - -# Other params. -TP=8 -PP=12 -MBS=1 -NLS=96 -HS=12288 -NAH=96 -DDP=local -NNODES=12 - - -# Name of the job. -export JOB_NAME=results_figure_12_interleaved_${INTERLEAVED}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_13.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_13.sh deleted file mode 100644 index 7ba560e87..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_13.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Pipeline-parallel size options = [2, 4, 8, 16, 32]. -PP=2 - -# Batch size (global batch size) options = [32, 128]. -GBS=32 - - - - - -# Set pipeline-parallel and tensor-parallel size options. -TP=$((64/PP)) - - -# Other params. -MBS=1 -NLS=32 -HS=20480 -NAH=128 -DDP=local -MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -NNODES=8 - - -# Name of the job. -export JOB_NAME=results_figure_13_pipeline_parallel_size_${PP}_tensor_parallel_size_${TP}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_14.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_14.sh deleted file mode 100644 index 4b83879c4..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_14.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Pipeline-parallel size options = [2, 4, 8, 16, 32]. -PP=2 - -# Batch size (global batch size) options = [32, 512]. -GBS=32 - - - - - -# Set pipeline-parallel and data-parallel size options. -DP=$((64/PP)) - - -# Other params. -TP=1 -MBS=1 -NLS=32 -HS=3840 -NAH=32 -DDP=local -MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -NNODES=8 - - -# Name of the job. -export JOB_NAME=results_figure_14_pipeline_parallel_size_${PP}_data_parallel_size_${DP}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_15.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_15.sh deleted file mode 100644 index 547ad1de6..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_15.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Tensor-parallel size options = [2, 4, 8, 16, 32]. -TP=2 - -# Batch size (global batch size) options = [32, 128, 512]. -GBS=32 - - - - - -# Set tensor-parallel and data-parallel size options. -DP=$((64/TP)) - - -# Other params. -PP=1 -MBS=1 -NLS=32 -HS=3840 -NAH=32 -DDP=local -MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -NNODES=8 - - -# Name of the job. -export JOB_NAME=results_figure_15_tensor_parallel_size_${TP}_data_parallel_size_${DP}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_16.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_16.sh deleted file mode 100644 index 8c353a3e7..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_16.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Microbatch size options = [1, 2, 4, 8]. -MBS=1 - -# Batch size (global batch size) options = [128, 512]. -GBS=128 - - - - - -# Other params. -TP=8 -PP=8 -NLS=32 -HS=15360 -NAH=128 -DDP=local -MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -NNODES=8 - - -# Name of the job. -export JOB_NAME=results_figure_16_microbatch_size_${MBS}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_17.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_17.sh deleted file mode 100644 index d6899b321..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_17.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Activation recomputation options = [YES, NO]. -ACTIVATION_RECOMPUTATION=YES - -# Batch size (global batch size) options = [1, 2, 4, ..., 256]. -GBS=1 - - - - - -# Set activation recomputation. -if [ ${ACTIVATION_RECOMPUTATION} == "YES" ]; then - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -elif [ ${ACTIVATION_RECOMPUTATION} == "NO" ]; then - MEGATRON_EXTRA_PARAMS="" -else - echo "Invalid configuration" - exit 1 -fi - - -# Other params. -TP=8 -PP=16 -MBS=1 -NLS=80 -HS=12288 -NAH=96 -DDP=local -NNODES=16 - - -# Name of the job. -export JOB_NAME=results_figure_17_activation_recomputation_${ACTIVATION_RECOMPUTATION}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_18.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_18.sh deleted file mode 100644 index 88924fb82..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_figure_18.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ - -# Scatter-gather communication optimization options = [YES, NO]. -SCATTER_GATHER=YES - -# Batch size (global batch size) options = [12, 24, 36, ..., 60]. -GBS=12 - - - - - -# Set scatter-gather communication optimization options. -if [ ${SCATTER_GATHER} == "YES" ]; then - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 2 " -elif [ ${SCATTER_GATHER} == "NO" ]; then - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 2 --no-scatter-gather-tensors-in-pipeline " -else - echo "Invalid configuration" - exit 1 -fi - - -# Other params. -TP=8 -PP=12 -MBS=1 -NLS=96 -HS=12288 -NAH=96 -DDP=local -NNODES=12 - - -# Name of the job. -export JOB_NAME=results_figure_18_scatter_gather_${SCATTER_GATHER}_batch_size_${GBS} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples/sc21/run_table_1.sh b/toolbox/Megatron-DeepSpeed/examples/sc21/run_table_1.sh deleted file mode 100644 index 1b15fb045..000000000 --- a/toolbox/Megatron-DeepSpeed/examples/sc21/run_table_1.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -# ================================ -# Choose the case to run. -# ================================ -# model size options = [1.7B, 3.6B, 7.5B, 18B, 39B, 76B, 145B, 310B, 530B, 1T] -MODEL_SIZE=1.7B - - - - - - -if [ ${MODEL_SIZE} == "1.7B" ]; then - TP=1 - PP=1 - MBS=16 - GBS=512 - NLS=24 - HS=2304 - NAH=24 - DDP=torch - NNODES=4 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -elif [ ${MODEL_SIZE} == "3.6B" ]; then - TP=2 - PP=1 - MBS=16 - GBS=512 - NLS=30 - HS=3072 - NAH=32 - DDP=torch - NNODES=8 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -elif [ ${MODEL_SIZE} == "7.5B" ]; then - TP=4 - PP=1 - MBS=16 - GBS=512 - NLS=36 - HS=4096 - NAH=32 - DDP=torch - NNODES=16 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -elif [ ${MODEL_SIZE} == "18B" ]; then - TP=8 - PP=1 - MBS=8 - GBS=1024 - NLS=40 - HS=6144 - NAH=48 - DDP=torch - NNODES=32 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -elif [ ${MODEL_SIZE} == "39B" ]; then - TP=8 - PP=2 - MBS=4 - GBS=1536 - NLS=48 - HS=8192 - NAH=64 - DDP=local - NNODES=64 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -elif [ ${MODEL_SIZE} == "76B" ]; then - TP=8 - PP=4 - MBS=2 - GBS=1792 - NLS=60 - HS=10240 - NAH=80 - DDP=local - NNODES=128 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 5" -elif [ ${MODEL_SIZE} == "145B" ]; then - TP=8 - PP=8 - MBS=2 - GBS=2304 - NLS=80 - HS=12288 - NAH=96 - DDP=local - NNODES=192 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 5 " -elif [ ${MODEL_SIZE} == "310B" ]; then - TP=8 - PP=16 - MBS=1 - GBS=2160 - NLS=96 - HS=16384 - NAH=128 - DDP=local - NNODES=240 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 3 " -elif [ ${MODEL_SIZE} == "530B" ]; then - TP=8 - PP=35 - MBS=1 - GBS=2520 - NLS=105 - HS=20480 - NAH=128 - DDP=local - NNODES=315 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform --num-layers-per-virtual-pipeline-stage 1 " -elif [ ${MODEL_SIZE} == "1T" ]; then - TP=8 - PP=64 - MBS=1 - GBS=3072 - NLS=128 - HS=25600 - NAH=160 - DDP=local - NNODES=384 - MEGATRON_EXTRA_PARAMS="--activations-checkpoint-method uniform " -else - echo "Invalid configuration" - exit 1 -fi - - -# Name of the job -export JOB_NAME=results_table_1_model_size_${MODEL_SIZE} - - -# Import the configs. -. `pwd`/CONFIG.sh - - -# Submit the job. -. `pwd`/SBATCH.sh - - -exit 0 - - - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_TEMPLATE.json deleted file mode 100644 index 5a14931cb..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_TEMPLATE.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": CONFIG_FP16_ENABLED, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "bf16": { - "enabled": CONFIG_BF16_ENABLED - }, - "curriculum_learning": { - "enabled": CONFIG_CL_ENABLED, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_Zero2_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_Zero2_TEMPLATE.json deleted file mode 100644 index 4d0a68f72..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_config_gpt_Zero2_TEMPLATE.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": 2 - }, - - "gradient_clipping": 1.0, - "prescale_gradients": false, - - "fp16": { - "enabled": CONFIG_FP16_ENABLED, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "bf16": { - "enabled": CONFIG_BF16_ENABLED - }, - "curriculum_learning": { - "enabled": CONFIG_CL_ENABLED, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_evalharness.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_evalharness.sh deleted file mode 100644 index 3496ada20..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_evalharness.sh +++ /dev/null @@ -1,72 +0,0 @@ -# This is an example zero-shot eval script. Please first read the readme_evalharness.md under the same directory. - -CHECKPOINT_PATH=/blob/users/conglli/project/gpt3_with_pile/checkpoint/gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-128-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-20728-token-45B/global_step81566/ -CONFIG_PATH=ds_config_gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-128-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-20728-token-45B.json -RESULT_PATH=gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-128-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-20728-token-45B_global_step81566.log - -PP_SIZE=1 -TP_SIZE=1 -NO_PP="true" -EP_PARALLEL_SIZE=1 -# Currently eval harness does not support data parallel -# However, for MoE models it's possible to enable a "fake data parallel" -# in order to load experts on multiple gpus. At the same time, it's not -# real data parallel because we load the same data on all gpus. -# On the other hand, it's better to use less number of gpus than training, -# to reduce communication overhead. -NUM_NODE=1 -NUM_GPU_PER_NODE=1 - -TASKS="lambada" -# WikiText-2, not used in GPT-3 paper but used in GPT-2 paper -# TASKS="wikitext" -# Tasks that appeared in GPT-3 paper (sorted based on the order in paper), plus WikiText-2. -# TASKS="hellaswag,lambada,triviaqa,webqs,winogrande,piqa,arc_challenge,arc_easy,openbookqa,race,boolq,cb,copa,rte,wic,wsc,multirc,record,anli_r1,anli_r2,anli_r3,wikitext" -# All tasks that confirmed to work, there are more tasks on https://github.com/EleutherAI/lm-evaluation-harness that we didn't test. -# TASKS="hellaswag,lambada,triviaqa,webqs,winogrande,piqa,arc_challenge,arc_easy,openbookqa,race,boolq,cb,copa,rte,wic,wsc,multirc,record,anli_r1,anli_r2,anli_r3,wikitext,logiqa,mathqa,mc_taco,mrpc,prost,pubmedqa,qnli,qqp,sciq,sst,wnli" - -VOCAB_FILE=/data/Megatron-LM/data/gpt2-vocab.json -MERGE_FILE=/data/Megatron-LM/data/gpt2-merges.txt - -# export HF_DATASETS_OFFLINE=1 - -# Dummy arguments to make megatron happy. No need to configure them. -# The reason we don't need to configure them and many other arguments is -# because the eval framework will read the arguments from checkpoint file. -MEGATRON_REQUIRED_ARGS="\ - --num-layers -1\ - --hidden-size -1\ - --num-attention-heads -1\ - --seq-length -1 \ - --max-position-embeddings -1 -" - -CMD="../../tasks/eval_harness/evaluate.py \ - --load $CHECKPOINT_PATH\ - --tensor-model-parallel-size $TP_SIZE \ - --pipeline-model-parallel-size $PP_SIZE\ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --vocab-file $VOCAB_FILE\ - --merge-file $MERGE_FILE\ - --micro-batch-size 12\ - --no-load-optim \ - --no-load-rng \ - --inference \ - --disable-moe-token-dropping \ - --tokenizer-type GPT2BPETokenizer \ - --adaptive_seq_len\ - --eval_fp32\ - --task_list $TASKS\ - --results_path $RESULT_PATH \ - --deepspeed \ - --deepspeed_config $CONFIG_PATH \ - $MEGATRON_REQUIRED_ARGS\ - " - -if [[ "${NO_PP}" = "true" ]]; then -CMD="${CMD} \ - --no-pipeline-parallel" -fi - -LAUNCHER="deepspeed --num_nodes $NUM_NODE --num_gpus $NUM_GPU_PER_NODE" -$LAUNCHER $CMD \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_MoE128.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_MoE128.sh deleted file mode 100644 index 0f2805dfd..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_MoE128.sh +++ /dev/null @@ -1,348 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -MODEL_SIZE=1.3 -NUM_LAYERS=24 -HIDDEN_SIZE=2048 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=8 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -# EP_SIZE=1 -EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -LR=1.2e-4 -MIN_LR=1.0e-6 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128.sh deleted file mode 100644 index f758ac69b..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128.sh +++ /dev/null @@ -1,340 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -MODEL_SIZE=1.3 -NUM_LAYERS=24 -HIDDEN_SIZE=2048 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=8 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 128 means standard MoE -# EP_SIZE=128 -EP_SIZE="64 64 64 64 64 64 64 64 64 64 128 128" - - -EP_PARALLEL_SIZE=$NUM_GPUS - - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B PR-MoE-64/128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## heavily tuned. -LR=1.2e-4 -MIN_LR=1.0e-6 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -NAME="${NAME}-ep-pyramid-64+128-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" - -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - BASE_DATA_PATH=/vc_data/Megatron-LM/data - DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --mlp-type residual \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -megatron_options="${megatron_options} \ - --create-moe-param-group" - - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_Zero2_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128_MoS.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128_MoS.sh deleted file mode 100644 index 34bc60548..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_PR-MoE64or128_MoS.sh +++ /dev/null @@ -1,354 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -MODEL_SIZE=1.3 -NUM_LAYERS=24 -HIDDEN_SIZE=2048 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=128 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 128 means standard MoE -# EP_SIZE=128 -EP_SIZE="64 64 64 64 64 64 64 64 128 128" -EP_SIZE_TEACHER="64 64 64 64 64 64 64 64 64 64 128 128" - -EP_PARALLEL_SIZE=$NUM_GPUS - - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B PR-MoE-64/128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## heavily tuned. -LR=1.2e-4 -MIN_LR=1.0e-6 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -NAME="${NAME}-ep-pyramid-64+128-mos-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" - -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -### Mixture-of-Students (MoS) configs -KD_BETA_CE=1 -CHECKPOINT_PATH_STUDENT="${OUTPUT_BASEPATH}/checkpoint/${NAME}" -CHECKPOINT_PATH_TEACHER="${OUTPUT_BASEPATH}/checkpoint/gpt-1.3B-lr-1.2e-4-minlr-1.0e-6-bs-512-gpus-128-mp-1-pp-1-ep-pyramid-64+128-mlc-0.01-cap-1.0-drop-true/" -CHECKPOINT_PATH_SAVE="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -USE_INTERNAL_DATA="true" -# USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - BASE_DATA_PATH=/vc_data/Megatron-LM/data - DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - ## Placeholder, we plan to test a public dataset - VOCAB_PATH="" - MERGE_PATH="" - DATA_BLEND="" -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --mlp-type residual \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 21 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH_STUDENT} \ - --save ${CHECKPOINT_PATH_SAVE} \ - --mos \ - --kd-beta-ce ${KD_BETA_CE} \ - --num-layers-teacher ${NUM_LAYERS} \ - --num-experts-teacher ${EP_SIZE_TEACHER} \ - --hidden-size-teacher ${HIDDEN_SIZE} \ - --num-attention-heads-teacher ${NUM_ATTN_HEADS} \ - --load-teacher ${CHECKPOINT_PATH_TEACHER} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -megatron_options="${megatron_options} \ - --create-moe-param-group" - - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_Zero2_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -# run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense.sh deleted file mode 100644 index 27b546435..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense.sh +++ /dev/null @@ -1,349 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -MODEL_SIZE=1.3 -NUM_LAYERS=24 -HIDDEN_SIZE=2048 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=512 -LR=2.0e-4 -MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -# LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=2 - -## Model parallelism, 1 is no MP -MP_SIZE=4 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -EP_SIZE=1 -# EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -# LR=2.0e-4 -# MIN_LR=2e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --rampup-batch-size 32 32 1953125 \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense_cl.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense_cl.sh deleted file mode 100644 index e40b55b80..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_1.3B_dense_cl.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -MODEL_SIZE=1.3 -NUM_LAYERS=24 -HIDDEN_SIZE=2048 -NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -MIN_LR=2.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -GLOBAL_BATCH_SIZE=4096 # 8x -LR=8.0e-4 # 4x - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -TRAIN_TOKENS=300000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=16 - -## Model parallelism, 1 is no MP -MP_SIZE=2 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=128 -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="true" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.013 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt3-with-pile-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-zero-${ZERO_STAGE}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/conglli/project/gpt3_with_pile/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -DATA_PATH=/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_MoE64.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_MoE64.sh deleted file mode 100644 index f93f0b712..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_MoE64.sh +++ /dev/null @@ -1,372 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -# EP_SIZE=1 -EP_SIZE=64 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -LR=4.5e-4 -MIN_LR=4.5e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_PATH="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - # For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 - DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document - # For cluster Azure-WestUS3-A100 - # DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_dense_cl.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_dense_cl.sh deleted file mode 100644 index 36b654e02..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_125M_dense_cl.sh +++ /dev/null @@ -1,309 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -MIN_LR=6.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -GLOBAL_BATCH_SIZE=2048 # 8x -LR=2.4e-3 # 4x - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -TRAIN_TOKENS=300000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=16 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="true" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=72 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.02 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt3-with-pile-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-zero-${ZERO_STAGE}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/conglli/project/gpt3_with_pile/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 -DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -# DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_MoE128.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_MoE128.sh deleted file mode 100644 index 4f8007b01..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_MoE128.sh +++ /dev/null @@ -1,348 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -MODEL_SIZE=0.35 -NUM_LAYERS=24 -HIDDEN_SIZE=1024 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -# EP_SIZE=1 -EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -LR=2.0e-4 -MIN_LR=2e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64.sh deleted file mode 100644 index d9f851380..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64.sh +++ /dev/null @@ -1,341 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -MODEL_SIZE=0.35 -NUM_LAYERS=24 -HIDDEN_SIZE=1024 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 128 means standard MoE -# EP_SIZE=128 -EP_SIZE="32 32 32 32 32 32 32 32 32 32 64 64" - -EP_PARALLEL_SIZE=$NUM_GPUS - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B PR-MoE-64/128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M PR-MoE-32/64 model we used LR=3.0e-4 and MIN_LR=1.0e-6, but they are not -## heavily tuned. -LR=3.0e-4 -MIN_LR=1.0e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -NAME="${NAME}-ep-pyramid-32+64-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" - -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - BASE_DATA_PATH=/vc_data/Megatron-LM/data - DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --mlp-type residual \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -megatron_options="${megatron_options} \ - --create-moe-param-group" - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" - - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64_MoS.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64_MoS.sh deleted file mode 100644 index a5b349b9e..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_PR-MoE32or64_MoS.sh +++ /dev/null @@ -1,353 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -MODEL_SIZE=0.35 -NUM_LAYERS=24 -HIDDEN_SIZE=1024 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_ITERS is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_ITERS. -TRAIN_ITERS=$(( ${TRAIN_TOKENS} * 3 / ${GLOBAL_BATCH_SIZE} / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -# LR_DECAY_TOKENS=260000000000 -LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 128 means standard MoE -# EP_SIZE=128 -EP_SIZE="32 32 32 32 32 32 32 32 64 64" -EP_SIZE_TEACHER="32 32 32 32 32 32 32 32 32 32 64 64" - -EP_PARALLEL_SIZE=$NUM_GPUS - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B PR-MoE-64/128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M PR-MoE-32/64 model we used LR=3.0e-4 and MIN_LR=1.0e-6, but they are not -## heavily tuned. -LR=3.0e-4 -MIN_LR=1.0e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -NAME="${NAME}-ep-pyramid-32+64-mos-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" - -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -### Mixture-of-Students (MoS) configs -KD_BETA_CE=1 -CHECKPOINT_PATH_STUDENT="${OUTPUT_BASEPATH}/checkpoint/${NAME}" -CHECKPOINT_PATH_TEACHER="${OUTPUT_BASEPATH}/checkpoint/gpt-1.3B-lr-1.2e-4-minlr-1.0e-6-bs-512-gpus-128-mp-1-pp-1-ep-pyramid-64+128-mlc-0.01-cap-1.0-drop-true/" -CHECKPOINT_PATH_SAVE="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -USE_INTERNAL_DATA="true" -# USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - BASE_DATA_PATH=/vc_data/Megatron-LM/data - DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - ## Placeholder, we plan to test a public dataset - VOCAB_PATH="" - MERGE_PATH="" - DATA_BLEND="" -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --mlp-type residual \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 21 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-iters ${TRAIN_ITERS} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH_STUDENT} \ - --save ${CHECKPOINT_PATH_SAVE} \ - --mos \ - --kd-beta-ce ${KD_BETA_CE} \ - --num-layers-teacher ${NUM_LAYERS} \ - --num-experts-teacher ${EP_SIZE_TEACHER} \ - --hidden-size-teacher ${HIDDEN_SIZE} \ - --num-attention-heads-teacher ${NUM_ATTN_HEADS} \ - --load-teacher ${CHECKPOINT_PATH_TEACHER} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -megatron_options="${megatron_options} \ - --create-moe-param-group" - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" - - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_dense.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_dense.sh deleted file mode 100644 index 405817a06..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_350M_dense.sh +++ /dev/null @@ -1,348 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -MODEL_SIZE=0.35 -NUM_LAYERS=24 -HIDDEN_SIZE=1024 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=256 -LR=3.0e-4 -MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -# LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -EP_SIZE=1 -# EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -# LR=2.0e-4 -# MIN_LR=2e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_6.7B_dense.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_6.7B_dense.sh deleted file mode 100644 index 1fdd76cbe..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/ds_pretrain_gpt_6.7B_dense.sh +++ /dev/null @@ -1,349 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -MODEL_SIZE=6.7 -NUM_LAYERS=32 -HIDDEN_SIZE=4096 -NUM_ATTN_HEADS=32 -GLOBAL_BATCH_SIZE=1024 -LR=1.2e-4 -MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -# LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=8 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -EP_SIZE=1 -# EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -# LR=2.0e-4 -# MIN_LR=2e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -# INIT_STD=0.014 -INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --rampup-batch-size 32 32 4882812 \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/readme_evalharness.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/readme_evalharness.md deleted file mode 100644 index d30075e2f..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/MoE/readme_evalharness.md +++ /dev/null @@ -1,168 +0,0 @@ -# How to run lm-eval on Megatron-DeepSpeed checkpoint using the original setup - -A great portion of this eval harness feature is inherited from https://github.com/bigscience-workshop/Megatron-DeepSpeed/pull/212, but with code/doc changes (e.g., to support case without pipeline parallelism and MoE models). - -This particular setup uses the normal deepspeed checkpoint and requires no conversion to Megatron-LM. - -## Prerequisites - -1. Install software - -On login console with external network - -Get lm-eval harness (https://github.com/EleutherAI/lm-evaluation-harness) and `best-download==0.0.7` needed to download some tasks. -Below package version numbers are what we tested that work. -``` -(maybe need pip install --upgrade pip) -pip install best-download==0.0.7 lm-eval==0.2.0 datasets==1.15.1 transformers==4.20.1 huggingface-hub==0.8.1 -``` - -2. Pre-download needed datasets - -some symlinks due to lm-harness' issues with relative position of data -``` -mkdir data -cd ../../tasks/eval_harness/ -ln -s ../../examples_deepspeed/MoE/data/ data -cd ../../examples_deepspeed/MoE/ -``` - - -Then install datasets for the tasks: -``` -python ../../tasks/eval_harness/download.py --task_list hellaswag,lambada,triviaqa,webqs,winogrande,piqa,arc_challenge,arc_easy,openbookqa,race,boolq,cb,copa,rte,wic,wsc,multirc,record,anli_r1,anli_r2,anli_r3,wikitext,logiqa,mathqa,mc_taco,mrpc,prost,pubmedqa,qnli,qqp,sciq,sst,wnli -``` - -Previously we set `export HF_DATASETS_OFFLINE=1` to make the dataset offline after the above manual download. But somehow now this could trigger error on some kind of online verification for some of the datasets, so it's recommended to only set offline mode when necessary. - - - -3. Prepare the script - - - -`ds_evalharness.sh` is the example script. - -1. Edit: - -``` -PP_SIZE=1 -TP_SIZE=1 -NO_PP="true" -EP_PARALLEL_SIZE=1 -NUM_NODE=1 -NUM_GPU_PER_NODE=1 -``` -to match the eval topology. - -Edit: -``` -CHECKPOINT_PATH= -CONFIG_PATH= -RESULT_PATH= -``` -to the checkpoint/ds config you want to use, and where to save the results. - - - -2. Adjust the following to fit the chosen GPU. As of last check for 1.3B model the settings are one of: -``` -EVAL_MICRO_BATCH_SIZE=6 # 16GB GPU 1.3B model -EVAL_MICRO_BATCH_SIZE=12 # 32GB GPU 1.3B model -``` - -If you get OOM lower it further. - -3. If not using the Deepspeed path, disable it by removing: - -``` - --deepspeed \ - --deepspeed_config ds_config.json \ -``` - -If you didn't disable it and the program crashed on checkpoint loading unable to find some key, disable deepspeed as explained above. - -Note that for MoE models and for models without pipeline parallelism, currently they might not work for the case without deepspeed. - - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/README.md deleted file mode 100644 index 3d8998166..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Megatron-DeepSpeed Recipes and Scripts - -This folder includes various example scripts with DeepSpeed technologies integrated. Below we describe each sub-folder, sorted by last update date. - -## Sync with NVIDIA/Megatron-LM (last updated: Jul 2023) -The ```rebase``` folder includes details about the recent sync with the NVIDIA/Megatron-LM repo (where this repo is forked from). It includes example scripts we used to test after the sync, together with a README documentation about what were tested. - -## Data Efficiency (last updated: Feb 2023) - -The ```data_efficiency``` folder includes GPT-3 and BERT pretraining examples for DeepSpeed Data Efficiency Library, together with examples of zero-shot evaluation for GPT models and GLUE finetuning for BERT models. Please refer to the detailed tutorials in data_efficiency/README.MD. Currently this folder includes the newest example scripts for GPT/BERT pretraining/eval/finetuning, both with and without DeepSpeed Data Efficiency Library techniques. - -## BERT example (last updated: Dec 2022) - -The ```bert_with_pile``` folder includes examples about BERT-style model pre-training (using the public Pile data or user's own data) with DeepSpeed integration. Please refer to the readme in the folder for tutorial. - -## Azure (last updated: Nov 2022) - -We strongly recommend to start with AzureML recipe in the ```azureml``` folder. - -If you have a custom infrastructure (e.g. HPC clusters) or Azure VM and VMSS based environments, please refer to the bash scripts in the ```azure``` folder. - -## Model Compression (last updated: Aug 2022) - -The ```compression``` folder includes examples about layer reduction for task-agnostic compression. Please refer to [this tutorial](https://www.deepspeed.ai/tutorials/model-compression/#11-layer-reduction) about the DeepSpeed Model Compression Library. These recipes are for GPT-style NLG models. - -## MoE (last updated: Jun 2022) - -Please see the ```MoE``` folder for different training recipes and scripts for Mixture-of-expert based models and dense models. These recipes are for GPT-style NLG models, and currently this is the only folder with MoE training examples. - -## Curriculum Learning (last updated: Oct 2021) - -Curriculum learning recipes are in the ```curriculum_learning``` folder. Please refer to the detailed tutorials linked inside. These recipes are for GPT-style NLG models. -Note that the DeepSpeed Data Efficiency Library above includes a more general curriculum learning support. This legacy curriculum learning feature is still compatible, but we recommend using the DeepSpeed Data Efficiency Library above. However, the newer DeepSpeed Data Efficiency Library currently is not compatible with pipeline parallelism. So if you have to use pipeline parallelism, you would need to use this legacy curriculum learning version. diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/README.md deleted file mode 100644 index ef648fa29..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/README.md +++ /dev/null @@ -1,27 +0,0 @@ -## Recipes for experimentation on Azure - -The recipes have been tested on command line on a cluster setup using Azure VMs and VMSS as well as inside Docker based environments. - -To run any of the examples in this folder, please go to the base directory of Megatron-DeepSpeed and run as follows - -```bash examples_deepspeed/azure/run-benchmark-model.sh``` - -### Pre-requisites - -To run the above script, you will need to either setup your own dataset and modify the scripts or use our helper scripts to download the publicly available Books dataset and GPT vocab files. Please use the following from the ```dataset``` folder - -```bash dataset/download_books.sh``` - -```bash dataset/download_vocab.sh``` - -### Run 175B and 1T models - -We have included two recipes for the 175B model and the 1T model. To train the model, we assume that the users will modify and tune hyperparameters and configurations by themselves. To facilitate initial training, we have made the recipes runnable with the Books dataset as follows. - -```bash examples_deepspeed/azure/run-175b.sh``` - -```bash examples_deepspeed/azure/run-1t.sh``` - -### Note about ZeRO stage 3 and CPU offload - -By default, we have enabled ZeRO Stage 3 for both the recipes above. For the 1T model, we have also enabled the CPU-offload feature to save on memory and enable a larger batch size that offers better performance. diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-175b.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-175b.sh deleted file mode 100644 index 3e6b84a85..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-175b.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -set -ex - -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -BASE_PATH=$PWD/dataset/ -DATA_PATH=${BASE_PATH}/BookCorpusDataset_text_document -DS_CONFIG=ds_config.json - -# Hostfile path -HF=/job/hostfile - -# Disabling tensor/pipeline parallelism -TP=1 -PP=1 - -# HEADS ~= HIDDEN/128 - -# Model: 175B -NLAYERS=96 -HIDDEN=12288 -HEADS=96 -SEQ=1024 - - -MICRO_BATCH=4 -NODES=1 -GPN=8 -GLOBAL_BATCH=$(( ${GPN} * ${MICRO_BATCH} * ${NODES} )) - -# Initial power scale for loss -SP=15 - -# Uncomment/comment one of the following blocks. - -# For 1T model, start with microbatch=1, try to get 2 and 4. If OOM w/ 4, use cpu-offloading - -# Set to cpu for offloading to cpu for larger models -#OFFLOAD_DEVICE="cpu" -#CPU_OPTIM=" --cpu-optimizer" - -# Set to none and empty string for no cpu offloading -OFFLOAD_DEVICE="none" -CPU_OPTIM=" " - -ZERO_STAGE=3 -OUTPUT_DIR=ds_z_off-${OFFLOAD_DEVICE}_stage_${ZERO_STAGE}_nl${NLAYERS}_hs${HIDDEN}_mb${MICRO_BATCH}_seq${SEQ}_gb${GLOBAL_BATCH}_nodes${NODES} -#OUTPUT_DIR=baseline_nl${NLAYERS}_hs${HIDDEN}_gb${GLOBAL_BATCH}_mb${MICRO_BATCH} -mkdir -p $OUTPUT_DIR - -cat < $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH, - "train_micro_batch_size_per_gpu": $MICRO_BATCH, - "steps_per_print": 1, - "gradient_accumulation_steps": 1, - "zero_optimization": { - "stage": 3, - "stage3_max_live_parameters": 3e9, - "stage3_max_reuse_distance": 3e9, - "stage3_param_persistence_threshold": 1e5, - "stage3_prefetch_bucket_size": 5e7, - "contiguous_gradients": true, - "overlap_comm": true, - "reduce_bucket_size": 90000000, - "sub_group_size": 1e9, - "offload_optimizer": { - "device": "$OFFLOAD_DEVICE", - "buffer_count": 4, - "pipeline_read": false, - "pipeline_write": false, - "pin_memory": true - } - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": true, - "initial_scale_power" : $SP, - "loss_scale_window": 1000, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "wall_clock_breakdown": true, - "zero_allow_untested_optimizer": false, - "aio": { - "block_size": 1048576, - "queue_depth": 16, - "single_submit": false, - "overlap_events": true, - "thread_count": 2 - } -} -EOT - -export NCCL_DEBUG=warn - -ds_args=" " -ds_args=" --deepspeed ${ds_args}" -ds_args=" --no-pipeline-parallel ${ds_args}" -ds_args=" --deepspeed_config=$DS_CONFIG ${ds_args}" -ds_args=" --zero-stage=$ZERO_STAGE ${ds_args}" -ds_args=" --deepspeed-activation-checkpointing ${ds_args}" - - - -deepspeed --force_multi --num_nodes=$NODES --hostfile $HF pretrain_gpt.py \ - --tensor-model-parallel-size $TP \ - --pipeline-model-parallel-size $PP \ - --num-layers $NLAYERS \ - --hidden-size $HIDDEN \ - --num-attention-heads $HEADS \ - --seq-length $SEQ \ - --loss-scale $SP \ - --max-position-embeddings $SEQ \ - --micro-batch-size $MICRO_BATCH \ - --global-batch-size $GLOBAL_BATCH \ - --train-iters 1000 \ - --lr 6.0e-5 \ - --min-lr 6.0e-6 \ - --lr-decay-style cosine \ - --log-interval 1 \ - --eval-iters 40 \ - --eval-interval 1000 \ - --data-path $DATA_PATH \ - --vocab-file $BASE_PATH/gpt2-vocab.json \ - --merge-file $BASE_PATH/gpt2-merges.txt \ - --save-interval 1000 \ - --split 98,2,0 \ - --clip-grad 1.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --init-method-std 0.006 \ - --fp16 \ - --checkpoint-activations \ - --tensorboard-dir $OUTPUT_DIR \ - $CPU_OPTIM $ds_args \ - --exit-interval 5000 | tee ${OUTPUT_DIR}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-1t.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-1t.sh deleted file mode 100644 index 6e93bcb06..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-1t.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -set -ex - -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -BASE_PATH=$PWD/dataset/ -DATA_PATH=${BASE_PATH}/BookCorpusDataset_text_document -DS_CONFIG=ds_config.json - -# Hostfile path -HF=/job/hostfile - -# Disabling tensor/pipeline parallelism -TP=1 -PP=1 - -# HEADS ~= HIDDEN/128 - -# Refer to Megatron-table in the README.md file for model sizes -# Model: 310B -#NLAYERS=96 -#HIDDEN=16384 -#HEADS=128 -#SEQ=2048 - -# Model 530B -#NLAYERS=105 -#HIDDEN=20480 -#HEADS=160 -#SEQ=2048 - -# Model 1T -NLAYERS=128 -HIDDEN=25600 -HEADS=160 -SEQ=1024 - -MICRO_BATCH=1 -NODES=1 -GPN=8 -GLOBAL_BATCH=$(( ${GPN} * ${MICRO_BATCH} * ${NODES} )) - -# Initial power scale for loss -SP=15 - -# Uncomment/comment one of the following blocks. - -# For 1T model, start with microbatch=1, try to get 2 and 4. If OOM w/ 4, use cpu-offloading - -# Set to cpu for offloading to cpu for larger models -OFFLOAD_DEVICE="cpu" -CPU_OPTIM=" --cpu-optimizer" - -# Set to none and empty string for no cpu offloading -#OFFLOAD_DEVICE="none" -#CPU_OPTIM=" " - -ZERO_STAGE=3 -OUTPUT_DIR=ds_z_off-${OFFLOAD_DEVICE}_stage_${ZERO_STAGE}_nl${NLAYERS}_hs${HIDDEN}_mb${MICRO_BATCH}_seq${SEQ}_gb${GLOBAL_BATCH}_nodes${NODES} -#OUTPUT_DIR=baseline_nl${NLAYERS}_hs${HIDDEN}_gb${GLOBAL_BATCH}_mb${MICRO_BATCH} -mkdir -p $OUTPUT_DIR - -cat < $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH, - "train_micro_batch_size_per_gpu": $MICRO_BATCH, - "steps_per_print": 1, - "gradient_accumulation_steps": 1, - "zero_optimization": { - "stage": 3, - "stage3_max_live_parameters": 3e9, - "stage3_max_reuse_distance": 3e9, - "stage3_param_persistence_threshold": 1e5, - "stage3_prefetch_bucket_size": 5e7, - "contiguous_gradients": true, - "overlap_comm": true, - "reduce_bucket_size": 90000000, - "sub_group_size": 1e9, - "offload_optimizer": { - "device": "$OFFLOAD_DEVICE", - "buffer_count": 4, - "pipeline_read": false, - "pipeline_write": false, - "pin_memory": true - } - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": true, - "initial_scale_power" : $SP, - "loss_scale_window": 1000, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "wall_clock_breakdown": true, - "zero_allow_untested_optimizer": false, - "aio": { - "block_size": 1048576, - "queue_depth": 16, - "single_submit": false, - "overlap_events": true, - "thread_count": 2 - } -} -EOT - -export NCCL_DEBUG=warn - -ds_args=" " -ds_args=" --deepspeed ${ds_args}" -ds_args=" --no-pipeline-parallel ${ds_args}" -ds_args=" --deepspeed_config=$DS_CONFIG ${ds_args}" -ds_args=" --zero-stage=$ZERO_STAGE ${ds_args}" -ds_args=" --deepspeed-activation-checkpointing ${ds_args}" - - - -deepspeed --force_multi --num_nodes=$NODES --hostfile $HF pretrain_gpt.py \ - --tensor-model-parallel-size $TP \ - --pipeline-model-parallel-size $PP \ - --num-layers $NLAYERS \ - --hidden-size $HIDDEN \ - --num-attention-heads $HEADS \ - --seq-length $SEQ \ - --loss-scale $SP \ - --max-position-embeddings $SEQ \ - --micro-batch-size $MICRO_BATCH \ - --global-batch-size $GLOBAL_BATCH \ - --train-iters 1000 \ - --lr 6.0e-5 \ - --min-lr 6.0e-6 \ - --lr-decay-style cosine \ - --log-interval 1 \ - --eval-iters 40 \ - --eval-interval 1000 \ - --data-path $DATA_PATH \ - --vocab-file $BASE_PATH/gpt2-vocab.json \ - --merge-file $BASE_PATH/gpt2-merges.txt \ - --save-interval 1000 \ - --split 98,2,0 \ - --clip-grad 1.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --init-method-std 0.006 \ - --fp16 \ - --checkpoint-activations \ - --tensorboard-dir $OUTPUT_DIR \ - $CPU_OPTIM $ds_args \ - --exit-interval 5000 | tee ${OUTPUT_DIR}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-benchmark-model.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-benchmark-model.sh deleted file mode 100644 index 099519bab..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azure/run-benchmark-model.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -set -ex - -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -BASE_PATH=$PWD/dataset/ -DATA_PATH=${BASE_PATH}/BookCorpusDataset_text_document -DS_CONFIG=ds_config.json - -# Hostfile path -HF=/job/hostfile - -# Disabling tensor/pipeline parallelism -TP=1 -PP=1 - -# HEADS ~= HIDDEN/128 - -# Model: Benchmark model -NLAYERS=1 -HIDDEN=12288 -HEADS=96 -SEQ=1024 - - -MICRO_BATCH=4 -NODES=2 -GPN=8 -GLOBAL_BATCH=$(( ${GPN} * ${MICRO_BATCH} * ${NODES} )) - -# Initial power scale for loss -SP=15 - -# Uncomment/comment one of the following blocks. - -# For 1T model, start with microbatch=1, try to get 2 and 4. If OOM w/ 4, use cpu-offloading - -# Set to cpu for offloading to cpu for larger models -#OFFLOAD_DEVICE="cpu" -#CPU_OPTIM=" --cpu-optimizer" - -# Set to none and empty string for no cpu offloading -OFFLOAD_DEVICE="none" -CPU_OPTIM=" " - -ZERO_STAGE=3 -OUTPUT_DIR=ds_z_off-${OFFLOAD_DEVICE}_stage_${ZERO_STAGE}_nl${NLAYERS}_hs${HIDDEN}_mb${MICRO_BATCH}_seq${SEQ}_gb${GLOBAL_BATCH}_nodes${NODES} -#OUTPUT_DIR=baseline_nl${NLAYERS}_hs${HIDDEN}_gb${GLOBAL_BATCH}_mb${MICRO_BATCH} -mkdir -p $OUTPUT_DIR - -cat < $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH, - "train_micro_batch_size_per_gpu": $MICRO_BATCH, - "steps_per_print": 1, - "gradient_accumulation_steps": 1, - "zero_optimization": { - "stage": 3, - "stage3_max_live_parameters": 3e9, - "stage3_max_reuse_distance": 3e9, - "stage3_param_persistence_threshold": 1e5, - "stage3_prefetch_bucket_size": 5e7, - "contiguous_gradients": true, - "overlap_comm": true, - "reduce_bucket_size": 90000000, - "sub_group_size": 1e9, - "offload_optimizer": { - "device": "$OFFLOAD_DEVICE", - "buffer_count": 4, - "pipeline_read": false, - "pipeline_write": false, - "pin_memory": true - } - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": true, - "initial_scale_power" : $SP, - "loss_scale_window": 1000, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "wall_clock_breakdown": true, - "zero_allow_untested_optimizer": false, - "aio": { - "block_size": 1048576, - "queue_depth": 16, - "single_submit": false, - "overlap_events": true, - "thread_count": 2 - } -} -EOT - -export NCCL_DEBUG=warn - -ds_args=" " -ds_args=" --deepspeed ${ds_args}" -ds_args=" --no-pipeline-parallel ${ds_args}" -ds_args=" --deepspeed_config=$DS_CONFIG ${ds_args}" -ds_args=" --zero-stage=$ZERO_STAGE ${ds_args}" -ds_args=" --deepspeed-activation-checkpointing ${ds_args}" - - - -deepspeed --force_multi --num_nodes=$NODES --hostfile $HF pretrain_gpt.py \ - --tensor-model-parallel-size $TP \ - --pipeline-model-parallel-size $PP \ - --num-layers $NLAYERS \ - --hidden-size $HIDDEN \ - --num-attention-heads $HEADS \ - --seq-length $SEQ \ - --loss-scale $SP \ - --max-position-embeddings $SEQ \ - --micro-batch-size $MICRO_BATCH \ - --global-batch-size $GLOBAL_BATCH \ - --train-iters 50 \ - --lr 6.0e-5 \ - --min-lr 6.0e-6 \ - --lr-decay-style cosine \ - --log-interval 1 \ - --eval-iters 40 \ - --eval-interval 1000 \ - --data-path $DATA_PATH \ - --vocab-file $BASE_PATH/gpt2-vocab.json \ - --merge-file $BASE_PATH/gpt2-merges.txt \ - --save-interval 1000 \ - --split 98,2,0 \ - --clip-grad 1.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --init-method-std 0.006 \ - --fp16 \ - --checkpoint-activations \ - --tensorboard-dir $OUTPUT_DIR \ - $CPU_OPTIM $ds_args \ - --exit-interval 5000 | tee ${OUTPUT_DIR}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/Dockerfile.dockerfile b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/Dockerfile.dockerfile deleted file mode 100644 index c01ec3a5c..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/Dockerfile.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM mcr.microsoft.com/azureml/curated/acpt-pytorch-1.11-cuda11.3:12 -USER root:root - -RUN pip install pybind11 -RUN pip install regex \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/README.md deleted file mode 100644 index 09a2faf78..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/README.md +++ /dev/null @@ -1,16 +0,0 @@ -## Megatron-DeepSpeed on AzureML -Example script for running Megatron-DeepSpeed using Azure Machine Learning. - ------- - -# Workspace Setup -Setup an AML workspace. Refer to: [set-up doc](https://github.com/Azure/azureml-examples/tree/main/v1/python-sdk#set-up). - -# Dataset Preparation -Create AML Dataset. To run remote AML job, you need to provide AML FileDataset. -Refer to [prepare_dataset script](prepare_dataset.py) to upload .bin and .idx files to blob store and on how to create FileDataset. - -> Note: The folder `bookcorpus_data` used by [prepare_dataset script](prepare_dataset.py) should not be under `azureml` directories. It is because Azure ML does not allow to include large files (limit: 100 files or 1048576 bytes) for Docker build context. - -# Training -Run Megatron-DeepSpeed on Azure ML. Refer to [aml_submit script](aml_submit.py). diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/aml_submit.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/aml_submit.py deleted file mode 100644 index ebfa0a9bf..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/aml_submit.py +++ /dev/null @@ -1,198 +0,0 @@ -import os -import requests -import sys - -# AzureML libraries -import azureml.core -from azureml.core import Dataset, Environment, Experiment, ScriptRunConfig, Workspace -from azureml.core.compute import ComputeTarget, AmlCompute -from azureml.core.compute_target import ComputeTargetException -from azureml.core.runconfig import PyTorchConfiguration -from azureml.core.environment import DockerBuildContext - -# Check core SDK version number -print("SDK version:", azureml.core.VERSION) - -# For setting up a workspace, refer to: https://github.com/Azure/azureml-examples/tree/main/python-sdk#set-up -ws = Workspace.from_config() -print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep='\n') - -#------------------------------------------------------------------------------- -# Prepare Compute Cluster -#------------------------------------------------------------------------------- -cluster_name = "a100-80gb" - -# Verify that the cluster doesn't exist already -try: - compute_target = ComputeTarget(workspace=ws, name=cluster_name) - print('Found existing compute target.') -except ComputeTargetException: - print('Creating a new compute target...') - compute_config = AmlCompute.provisioning_configuration(vm_size='Standard_ND96amsr_A100_v4', min_nodes=32, max_nodes=32) - - # create the cluster - compute_target = ComputeTarget.create(ws, cluster_name, compute_config) - compute_target.wait_for_completion(show_output=True) - -#------------------------------------------------------------------------------- -# Prepare Data -# Megatron-DeepSpeed takes in data_path, vocab_file, and merge_file. -# For AML, we are adding a parameter aml_data_download_path which specifies how to deliver the dataset to a compute target. -# In the submitted run, files in the datasets will be either mounted or downloaded to local path on the compute target. -# -# data_path for this example is path to the .bin and .idx file, excluding extension. -# e.g. for data/BookCorpusDataset_text_document.bin and data/BookCorpusDataset_text_document.idx, -# data_path = "data/BookCorpusDataset_text_document" -# -# Once the folder is downloaded to the compute target, it will use aml_data_download_path to locate the folder -# and data_path to locate .bin and .idx files -# -# vocab_file and merge_file would also be passed in a similar way. -#------------------------------------------------------------------------------- -datastore = ws.get_default_datastore() -blobstore_datadir = "bookcorpus_data" -data_path = f"BookCorpusDataset_text_document" -# Load data folder which contains bookcorpus .bin and .idx files -train_dataset = Dataset.File.from_files(path=[(datastore, blobstore_datadir)]) -aml_data_download_path = train_dataset.as_download(blobstore_datadir) - -vocab_file_dataset = Dataset.File.from_files("https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json") -merge_file_dataset = Dataset.File.from_files("https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt") -vocab_file = vocab_file_dataset.as_download() -merge_file = merge_file_dataset.as_download() - - -#------------------------------------------------------------------------------- -# Setup training environment -#------------------------------------------------------------------------------- - -megatron_ds_env = Environment.from_docker_build_context(name='megatron-ds-curated-acpt', docker_build_context=DockerBuildContext.from_local_directory(workspace = ws, path = '.', dockerfile_path='Dockerfile.dockerfile')) -megatron_ds_env.register(ws).build(ws).wait_for_completion() # Comment this out if environment already exists - -#------------------------------------------------------------------------------- -# Training Settings and Arguments -#------------------------------------------------------------------------------- -node_count = 2 -total_processes_count = 16 -micro_batch_size = 1 -global_batch_size = micro_batch_size * total_processes_count -tensorboard_dir = '/tmp/outputs/tensorboard' - -run_args = ['--tensor-model-parallel-size', 1, - '--pipeline-model-parallel-size', 1, - '--num-layers', 20, - '--hidden-size', 12288, - '--num-attention-heads', 96, - '--seq-length', 1024, - '--loss-scale', 15, - '--max-position-embeddings', 1024, - '--micro-batch-size', micro_batch_size, - '--global-batch-size', global_batch_size, - '--train-iters', 100, - '--lr', 6.0e-5, - '--min-lr', 6.0e-6, - '--lr-decay-style', 'cosine', - '--log-interval', 1, - '--eval-iters', 40, - '--eval-interval', 1000, - '--aml-data-download-path', aml_data_download_path, - '--data-path', data_path, - '--vocab-file', vocab_file, - '--merge-file', merge_file, - '--save-interval', 1000, - '--split', '98,2,0', - '--clip-grad', 1.0, - '--weight-decay', 0.1, - '--adam-beta1', 0.9, - '--adam-beta2', 0.95, - '--init-method-std', 0.006, - '--fp16', - '--data-impl', 'mmap', - '--checkpoint-activations', - '--tensorboard-dir', tensorboard_dir, - #'--cpu-optimizer', - '--deepspeed', - '--no-pipeline-parallel', - '--deepspeed_config', 'ds_config.json', - '--zero-stage', 3, - '--deepspeed-activation-checkpointing', - '--exit-interval', 5000, -] - -#------------------------------------------------------------------------------- -# DeepSpeed ds_config.json -#------------------------------------------------------------------------------- -import json -ds_config = { - "train_batch_size" : global_batch_size, - "train_micro_batch_size_per_gpu": micro_batch_size, - "steps_per_print": 1, - "gradient_accumulation_steps": 1, - "zero_optimization": { - "stage": 3, - "stage3_max_live_parameters": 3e9, - "stage3_max_reuse_distance": 3e9, - "stage3_param_persistence_threshold": 1e5, - "stage3_prefetch_bucket_size": 5e7, - "contiguous_gradients": True, - "overlap_comm": True, - "reduce_bucket_size": 90000000, - "sub_group_size": 1e9, - "offload_optimizer": { - "device": "none", - "buffer_count": 4, - "pipeline_read": False, - "pipeline_write": False, - "pin_memory": True - } - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": True, - "initial_scale_power" : 15, - "loss_scale_window": 1000, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "wall_clock_breakdown": True, - "zero_allow_untested_optimizer": False, - "aio": { - "block_size": 1048576, - "queue_depth": 16, - "single_submit": False, - "overlap_events": True, - "thread_count": 2 - } - } - -# Place ds_config.json in the same folder as pretrain_gpt.py (script to run) -ds_config_path = '../../ds_config.json' -with open(ds_config_path, 'w') as fp: - json.dump(ds_config, fp, indent=4) - -#------------------------------------------------------------------------------- -# Create ScriptRunConfig -#------------------------------------------------------------------------------- -distr_config = PyTorchConfiguration(process_count=total_processes_count, node_count=node_count) - -megatron_ds_src = ScriptRunConfig(source_directory='../../', - script='pretrain_gpt.py', - arguments=run_args, - compute_target=compute_target, - environment=megatron_ds_env, - distributed_job_config=distr_config) - -megatron_ds_src.run_config.environment_variables['NCCL_DEBUG'] = 'WARN' -megatron_ds_src.run_config.environment_variables['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' -megatron_ds_src.run_config.environment_variables['NCCL_SOCKET_IFNAME'] = 'eth0' -megatron_ds_src.run_config.environment_variables['NCCL_IB_PCI_RELAXED_ORDERING']='1' -megatron_ds_src.run_config.environment_variables['UCX_TLS']='tcp' -megatron_ds_src.run_config.environment_variables['UCX_NET_DEVICES']='eth0' - -#------------------------------------------------------------------------------- -# Submit experiment -#------------------------------------------------------------------------------- -experiment_name = 'megatron-ds' -experiment = Experiment(ws, name=experiment_name) - -run = experiment.submit(megatron_ds_src, tags={'bs':micro_batch_size, 'gpus':total_processes_count}) diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/prepare_dataset.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/prepare_dataset.py deleted file mode 100644 index dfe6bc14a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/azureml/prepare_dataset.py +++ /dev/null @@ -1,33 +0,0 @@ -# Use this script to upload data to blob store - -# AzureML libraries -from azureml.core import Workspace -from azureml.core.dataset import Dataset -from azureml.data.datapath import DataPath - -ws = Workspace.from_config() -print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep='\n') - -data_dir = "bookcorpus_data" # Local directory for where data is located that includes .bin and .idx files -blobstore_datadir = data_dir # Blob store directory to store data in - -datastore = ws.get_default_datastore() - -# Book Corpus Data -print("upload dataset to blob store") -uploaded_data = Dataset.File.upload_directory( - src_dir=data_dir, - target=DataPath(datastore, blobstore_datadir), - show_progress=True -) - -# Usage after uploading the directory -# To refer to the folder directly: -train_dataset = Dataset.File.from_files(path=[(datastore, blobstore_datadir)]) -print(train_dataset) -# To refer to a specific file: -# train_dataset = Dataset.File.from_files(path=[(datastore, blobstore_datadir + "/filename.ext")]) -# Create DatasetConsumptionConfig to specify how to deliver the dataset to a compute target. -# In the submitted run, files in the datasets will be either mounted or downloaded to local path on the compute target. -# input_data_dir = train_dataset.as_mount() -# input_data_dir = train_dataset.as_download() diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/README.md deleted file mode 100644 index 2fa704ecf..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/README.md +++ /dev/null @@ -1,23 +0,0 @@ -This ```bert_with_pile``` folder includes examples about BERT pre-training (using [the public Pile data](https://github.com/EleutherAI/the-pile) or user's own data) with DeepSpeed integration. We also provide scripts about preprocessing Pile data and MNLI finetuning. - -## Data preprocessing -```prepare_pile_data.py``` is the script for downloading, decompressing, and preprocessing [the public Pile data](https://github.com/EleutherAI/the-pile). Users can also modify this script to preprocess their own training data. - -## BERT pre-training -```ds_pretrain_bert.sh``` is the script for BERT pre-training integrated with DeepSpeed, supporting [ZeRO](https://www.deepspeed.ai/tutorials/zero/) together with Megatron's tensor-slicing model parallelism. The training hyperparameters follow the [Megatron paper](https://arxiv.org/abs/1909.08053). Note that the pipeline parallelism is currently not supported: DeepSpeed's pipeline parallelism is only integrated with the GPT case, and currently DeepSpeed is not integrated with Megatron's own pipeline parallelism. - -As a reference performance number, our measurements show that our example is able to achieve a throughput up to 145 TFLOPs per GPU when pre-training a 1.3B BERT model (with ZeRO stage-1, without model parallelism, with 64 NVIDIA A100 GPUs, with batch size 4096 (64 per GPU), with activation checkpointing). - -One thing to note is that this pre-training recipe is NOT a strict reproduction of the [original BERT paper](https://arxiv.org/abs/1810.04805): the Pile data is larger than the data used in original BERT (and the data used by Megatron paper); Megatron-LM introduces some changes to the BERT model (see details in [Megatron paper](https://arxiv.org/abs/1909.08053)); the training hyperparameters are also different. Overall these differences lead to longer training time but also better model quality than original BERT (see MNLI score below), and supporting large model scale by the combination of ZeRO and model parallelism. If you don't have enough computation budget, we recommend to reduce the total training iterations (```train_iters``` in the script) and potentially increase the learning rate at the same time. If you want to strictly reproduce original BERT, we recommend to use our [another BERT example](https://github.com/microsoft/DeepSpeedExamples/tree/master/bing_bert). - -## BERT MNLI fine-tuning -```ds_finetune_bert_mnli.sh``` is the script for BERT MNLI fine-tuning, following the hyperparameters in the [Megatron paper](https://arxiv.org/abs/1909.08053). As a reference, table below present the scores using the model pre-trained based on the script above, comparing with the scores of original BERT and Megatron paper's BERT. Our BERT-Large's score is slightly lower than Megatron paper's, mainly due to the different data we used (Pile data is much diverse and larger than the data in Megatron paper, which potentially has negative effect on small million-scale models). - -| MNLI dev set accuracy | **MNLI-m** | **MNLI-mm** | -| ---------- |---------- |---------- | -| BERT-Base, [original BERT](https://arxiv.org/abs/1810.04805) | 84.6 | 83.4 | -| BERT-Base, ours (median on 5 seeds) | 86.1 | 86.1 | -| BERT-Large, [original BERT](https://arxiv.org/abs/1810.04805) | 86.7 | 85.9 | -| BERT-Large, [Megatron paper](https://arxiv.org/abs/1909.08053) | 89.7 | 90.0 | -| BERT-Large, ours (median on 5 seeds) | 89.1 | 89.6 | - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_config_bert_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_config_bert_TEMPLATE.json deleted file mode 100644 index b00ca33f0..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_config_bert_TEMPLATE.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": CONFIG_FP16_ENABLED, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "bf16": { - "enabled": CONFIG_BF16_ENABLED - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_mnli.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_mnli.sh deleted file mode 100644 index 4697b771d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_mnli.sh +++ /dev/null @@ -1,150 +0,0 @@ -seed=1234 -pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 - -## From Table 6 in https://arxiv.org/abs/1909.08053. -task="MNLI" -global_batch_size=128 -lr=1e-5 -epochs=10 - -train_data="/blob/data/GlueData/MNLI/train.tsv" -valid_data="/blob/data/GlueData/MNLI/dev_matched.tsv \ - /blob/data/GlueData/MNLI/dev_mismatched.tsv" - -## Adjust based on number of GPUs. -batch_size=16 - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 -save_interval=500000 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -checkpoint_path="${pretrained_checkpoint}-finetune/${jobname}" -mkdir -p ${checkpoint_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.065 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --save-interval ${save_interval} \ - --save ${checkpoint_path} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev set accuracy numbers by -# "grep -e "overall:" -e "metrics for" ${checkpoint_path}/output.log" -deepspeed ../../tasks/main.py ${options} &> ${checkpoint_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_qqp.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_qqp.sh deleted file mode 100644 index 78baa6ef0..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_qqp.sh +++ /dev/null @@ -1,158 +0,0 @@ -seed=1234 -pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 - -## From Table 6 in https://arxiv.org/abs/1909.08053. -task="QQP" - -train_data="/blob/data/GlueData/QQP/train.tsv" -valid_data="/blob/data/GlueData/QQP/dev.tsv" - -## Adjust based on number of GPUs. -batch_size=16 - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=128 -# lr=5e-5 -# epochs=12 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 -global_batch_size=128 -lr=5e-5 -epochs=12 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 -# global_batch_size=128 -# lr=3e-5 -# epochs=12 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -# global_batch_size=256 -# lr=4e-5 -# epochs=12 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 -save_interval=500000 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -checkpoint_path="${pretrained_checkpoint}-finetune/${jobname}" -mkdir -p ${checkpoint_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.065 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --save-interval ${save_interval} \ - --save ${checkpoint_path} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev set accuracy numbers by -# "grep -e "overall:" -e "metrics for" ${checkpoint_path}/output.log" -deepspeed ../../tasks/main.py ${options} &> ${checkpoint_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_race.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_race.sh deleted file mode 100644 index 5e4a57d92..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_finetune_bert_race.sh +++ /dev/null @@ -1,172 +0,0 @@ -seed=1234 -## RACE have two sub-tasks that need to be finetuned separately -difficulty="middle" -# difficulty="high" -pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 - -## From Table 6 in https://arxiv.org/abs/1909.08053. -task="RACE" - -## Race dataset can be downloaded by: -## wget http://www.cs.cmu.edu/~glai1/data/race/RACE.tar.gz -train_data="/blob/data/RACE/train/${difficulty}" - -## The Megatron paper https://arxiv.org/abs/1909.08053 says: "For the test set -## results of RACE, we first use the development set to find the checkpoint -## that gives us the median score on the 5 random seeds and we report the -## results from that checkpoint on the test set", which is a quite confusing -## description. For simplicity, instead we directly get the median dev and test -## set score on 5 random seeds from a single pretrained_checkpoint. -valid_data="/blob/data/RACE/dev/${difficulty} \ - /blob/data/RACE/test/${difficulty}" - -## Adjust based on number of GPUs. -batch_size=4 - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=32 -# lr=2e-5 -# epochs=3 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 -global_batch_size=32 -lr=2e-5 -epochs=3 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 -# global_batch_size=16 -# lr=1e-5 -# epochs=3 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -# global_batch_size=32 -# lr=2e-5 -# epochs=3 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 -save_interval=100000 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-${difficulty}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -checkpoint_path="${pretrained_checkpoint}-finetune/${jobname}" -mkdir -p ${checkpoint_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.06 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --save-interval ${save_interval} \ - --save ${checkpoint_path} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --clip-grad 1.0 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev/test set accuracy numbers -# by "grep -e "overall:" -e "metrics for" ${checkpoint_path}/output.log" -deepspeed ../../tasks/main.py ${options} &> ${checkpoint_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_pretrain_bert.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_pretrain_bert.sh deleted file mode 100644 index 397d7cb11..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/ds_pretrain_bert.sh +++ /dev/null @@ -1,267 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 -global_batch_size=1024 -lr=1e-4 -min_lr=1e-5 - -## init_std is the standard deviation for weight initialization. Usually larger -## model needs lower std. Here we roughly follow a heuristic equation of -## sqrt(1/3/hidden_size) from https://arxiv.org/pdf/2201.11990.pdf - -## In addition, we find that the 3.9B model (even after tuning init_std) has -## NaN loss issue from the beginning thus unable to train. This is probably -## because in this example we use the public Pile data, which is a more diverse -## (and potentially more noisy) data than what used in Megatron paper. One -## potential solution is only use the sub datasets in Pile that are also -## used by Megatron paper. - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# init_std=0.02 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 -init_std=0.02 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 -# init_std=0.013 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -# init_std=0.011 -############################################################################### -### Training duration configs -## The main termination condition, original Megatron paper trains for 2M iters. -train_iters_in_million=2 -train_iters=$((${train_iters_in_million} * 1000000)) -############################################################################### -### lr configs -## lr warmup and decay duration. Original Megatron paper uses 10000 warmup -## iters. Decay iters is the same as train iters. -lr_warmup_iters=10000 -lr_decay_iters_in_million=${train_iters_in_million} -lr_decay_iters=$((${lr_decay_iters_in_million} * 1000000)) -lr_decay_style="linear" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Below batch_size calculation assumes the case without gradient accumulation. -## Manually set it to a lower value if you hit out of memory during training. -batch_size=$(( ${global_batch_size} / ${dp_size} )) -############################################################################### -### Misc configs -log_interval=100 -eval_iters=10 -eval_interval=1000 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -save_interval=$((${train_iters} / ${num_save})) - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" - -## Public the Pile dataset, see prepare_pile_data.py in the same directory -## about how to download and preprocess the data. -jobname="bert-pile" -## For internal use. Change data_home to your own training data path. -data_home="/vc_data_blob/users/conglli/the_pile_bert" -if [[ "$host" == *"webxt"* ]]; then - data_home="/blob/data/the_pile_bert" -fi -data_path="${data_home}/pile_bert_train_text_sentence" - -vocab_path="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -## Number of workers for dataloader. We found that for BERT pre-training, -## num_workers will greatly affect data loading time and overall training -## time. In our experiment with 64 GPUs, the performance reaches peak at -## num_workers = 4 but it may differ depending on hardware. Also note that -## larger num_workers add more CPU computation/memory overhead. -num_workers=4 - -jobname="${jobname}-${model_size}B-iters-${train_iters_in_million}M" -jobname="${jobname}-lr-${lr}-min-${min_lr}-wmup-${lr_warmup_iters}-dcy-${lr_decay_iters_in_million}M-sty-${lr_decay_style}" -jobname="${jobname}-gbs-${global_batch_size}-mbs-${batch_size}-gpu-${num_gpus}-zero-${zero_stage}-mp-${mp_size}-pp-${pp_size}" -if [ "${no_pp}" = "true" ]; then - jobname="${jobname}-nopp" -fi - -username=$(whoami) -output_home="/vc_data_blob/users/${username}/project/bert_with_pile" -if [[ "$host" == *"webxt"* ]]; then - output_home="/blob/users/${username}/project/bert_with_pile" -fi -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/bert_with_pile/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.999 \ - --init-method-std ${init_std} \ - --tensor-model-parallel-size ${mp_size} \ - --lr-decay-iters ${lr_decay_iters} \ - --lr-warmup-iters ${lr_warmup_iters} \ - --micro-batch-size ${batch_size} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-iters ${train_iters} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 1e-2 \ - --clip-grad 1.0 \ - --num-workers ${num_workers} \ - --fp16 \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_bert.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/prepare_pile_data.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/prepare_pile_data.py deleted file mode 100644 index d3428b1d9..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/bert_with_pile/prepare_pile_data.py +++ /dev/null @@ -1,128 +0,0 @@ -import zstandard -import sys -import time -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir,os.path.pardir))) -from megatron_ds.data import indexed_dataset - -def pile_download(download_url, file_path, i): - start = time.time() - zstd_file_path = f"{file_path}{i:02}.jsonl.zst" - download_path = f"{download_url}{i:02}.jsonl.zst" - if not os.path.exists(zstd_file_path): - os.system(f"wget -P {file_path} {download_path}") - print(f"Finished downloading chunk {i} in {time.time() - start} sec") - -def pile_decompress(download_url, file_path, i): - zstd_file_path = f"{file_path}{i:02}.jsonl.zst" - output_path = f"{file_path}{i:02}.jsonl" - if not os.path.exists(output_path): - if not os.path.exists(zstd_file_path): - pile_download(download_url, file_path, i) - start = time.time() - with open(zstd_file_path, 'rb') as compressed: - decomp = zstandard.ZstdDecompressor() - with open(output_path, 'wb') as destination: - decomp.copy_stream(compressed, destination) - os.remove(zstd_file_path) - print(f"Finished decompressing chunk {i} in {time.time() - start} sec") - -def pile_preprocess(download_url, file_path, vocab_file, num_workers, i): - json_file_path = f"{file_path}{i:02}.jsonl" - output_prefix = f"{file_path}pile_bert_train_{i:02}" - if not os.path.exists(f"{output_prefix}_text_sentence.idx"): - if not os.path.exists(json_file_path): - pile_decompress(download_url, file_path, i) - start = time.time() - cmd = f"python ../../tools/preprocess_data.py \ - --input {json_file_path} \ - --output-prefix {output_prefix} \ - --vocab {vocab_file} \ - --dataset-impl mmap \ - --tokenizer-type BertWordPieceLowerCase \ - --split-sentences \ - --workers {num_workers} " - # It's possible to hit MemoryError during above cmd since the memory - # usage is proportional to num_workers. In this case we delete the - # incomplete output and user shall retry with smaller num_workers. - # Our experience show that chunk 6, 7, 9, 17, 18, 20, 21, 24, 27 - # particularly have large memory usage. - if os.system(cmd) == 0: # Success - os.remove(json_file_path) - else: - print(f"Error: chunk {i} preprocessing got error, delete \ - incomplete output. If MemoryError appeared, please retry \ - with num_workers smaller than {num_workers}.") - if os.path.exists(f"{output_prefix}_text_sentence.idx"): - os.remove(f"{output_prefix}_text_sentence.idx") - if os.path.exists(f"{output_prefix}_text_sentence.bin"): - os.remove(f"{output_prefix}_text_sentence.bin") - print(f"Finished preprocessing chunk {i} in {time.time() - start} sec") - -def pile_merge(file_path): - start = time.time() - num_chunks = 30 - vocab_size = 30524 - for i in range(num_chunks): - output_prefix = f"{file_path}pile_bert_train_{i:02}" - assert os.path.exists(f"{output_prefix}_text_sentence.idx") - assert os.path.exists(f"{output_prefix}_text_sentence.bin") - builder = indexed_dataset.make_builder( - f"{file_path}pile_bert_train_text_sentence.bin", impl="mmap", - vocab_size=vocab_size) - for i in range(num_chunks): - chunk_file = f"{file_path}pile_bert_train_{i:02}_text_sentence" - print(f"Merging file {chunk_file}") - builder.merge_file_(chunk_file) - print("Finalizing merged file ...") - builder.finalize(f"{file_path}pile_bert_train_text_sentence.idx") - print(f"Finished merging in {time.time() - start} sec") - # After verifying the merged data with real training, you may want to - # delete the data chunks. - # for i in range(num_chunks): - # output_prefix = f"{file_path}pile_bert_train_{i:02}" - # os.remove(f"{output_prefix}_text_sentence.idx") - # os.remove(f"{output_prefix}_text_sentence.bin") - -if __name__ == '__main__': - # Path to download and store all the output files during the whole process. - # Estimated max storage usage would be around 1.6 TB (or 780GB if skip the - # final merge). Memory usage is proportional to the num_workers below (can - # be as high as O(300GB) if num_workers is around 20). - file_path = "/blob/data/the_pile_bert/" - # The raw Pile data has 30 compressed .zst chunks. To run on single - # machine for all chunks, run "python prepare_pile_data.py range 0 30". - # You can also split and run on multiple machines to speed up, since - # processing one chunk can take hours. The whole process only uses CPU. - if sys.argv[1] == "merge": - # "python prepare_pile_data.py merge" means merge all 30 processed data - # chunks. Run it only after all 30 chunks are preprocessed. The memory - # usage during merge is about 600GB. If you don't have enough memory, - # one solution is to directly use the 30 data chunks as multiple - # datasets. See '--data-path' in - # github.com/microsoft/Megatron-DeepSpeed/blob/main/megatron/arguments.py - pile_merge(file_path) - else: - if sys.argv[1] == "range": - # "python prepare_pile_data.py range 0 30" means process chunk 0-29 - selected_chunk = range(int(sys.argv[2]), int(sys.argv[3])) - else: - # "python prepare_pile_data.py 2 5 8" means process chunk 2, 5, 8 - selected_chunk = [int(x) for x in sys.argv[1:]] - print("selected_chunk: ", selected_chunk) - # Number of process. Adjust based on your CPU/Memory. - num_workers = 20 - # Where the raw Pile data can be downloaded. The url may change in - # future. Contact EleutherAI (https://github.com/EleutherAI/the-pile) - # if this url does not work. - download_url = "https://the-eye.eu/public/AI/pile/train/" - vocab_file = "bert-large-uncased-vocab.txt" - vocab_url = "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt" - if not os.path.exists(vocab_file): - os.system(f"wget {vocab_url}") - os.makedirs(file_path, exist_ok=True) - - for i in selected_chunk: - pile_preprocess(download_url, file_path, vocab_file, num_workers, i) diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-Int8-test-64gpu-distilled-group48.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-Int8-test-64gpu-distilled-group48.sh deleted file mode 100644 index 5e84883f9..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-Int8-test-64gpu-distilled-group48.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -LR=6.0e-5 -MIN_LR=6.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -# GLOBAL_BATCH_SIZE=16 # 8x -# LR=6e-4 # 4x - -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -# TRAIN_TOKENS=300000000000 -TRAIN_TOKENS=5250000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=72 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.02 - -## Activation checkpointing saves GPU memory, but reduces training speed -# ACTIVATION_CHECKPOINT="true" -ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="125M10L_Compression_Test_INT8_64gpu_lr6e-5_tokens5.25B_nocl" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/zheweiyao/compression_library/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 -# DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 10 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load /blob/users/minjiaz/project/gpt3_distillation/checkpoint/gpt3-kd-staged-alpha1-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-32-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/ \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --no-load-lr-state \ - --reset-iteration \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE_compression.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}.log" -# run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options}" - -echo ${run_cmd} -eval ${run_cmd} -set +x \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L10-Int8-test-64gpu-distilled-group48.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L10-Int8-test-64gpu-distilled-group48.sh deleted file mode 100644 index a15c805d8..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L10-Int8-test-64gpu-distilled-group48.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -LR=6.0e-5 -MIN_LR=6.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -# GLOBAL_BATCH_SIZE=16 # 8x -# LR=6e-4 # 4x - -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -# TRAIN_TOKENS=300000000000 -TRAIN_TOKENS=5250000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=72 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.02 - -## Activation checkpointing saves GPU memory, but reduces training speed -# ACTIVATION_CHECKPOINT="true" -ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="125M10L_Compression_Test_INT8_64gpu_lr6e-5_tokens5.25B_nocl_alpha" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/minjiaz/compression_library/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -VOCAB_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 -# DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 10 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load /blob/users/minjiaz/project/gpt3_distillation/checkpoint/gpt3-kd-staged-alpha1-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-32-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/ \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --no-load-lr-state \ - --reset-iteration \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE_compression.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}.log" -# run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options}" - -echo ${run_cmd} -eval ${run_cmd} -set +x \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L12-Int8-test-64gpu-distilled-group48.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L12-Int8-test-64gpu-distilled-group48.sh deleted file mode 100644 index 013fbb4a1..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/125M-L12-Int8-test-64gpu-distilled-group48.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -LR=6.0e-5 -MIN_LR=6.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -# GLOBAL_BATCH_SIZE=16 # 8x -# LR=6e-4 # 4x - -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -# TRAIN_TOKENS=300000000000 -TRAIN_TOKENS=5250000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=72 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.02 - -## Activation checkpointing saves GPU memory, but reduces training speed -# ACTIVATION_CHECKPOINT="true" -ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="125M12L_Compression_Test_INT8_64gpu_lr6e-5_tokens5.25B_nocl_alpha" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/minjiaz/compression_library/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -VOCAB_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 -# DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 12 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load /blob/users/conglli/project/gpt3_with_pile/checkpoint/gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-64-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/ \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --no-load-lr-state \ - --reset-iteration \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE_compression.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}.log" -# run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options}" - -echo ${run_cmd} -eval ${run_cmd} -set +x \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE.json deleted file mode 100644 index 5a14931cb..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": CONFIG_FP16_ENABLED, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "bf16": { - "enabled": CONFIG_BF16_ENABLED - }, - "curriculum_learning": { - "enabled": CONFIG_CL_ENABLED, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE_compression.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE_compression.json deleted file mode 100644 index 083838a38..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_config_gpt_TEMPLATE_compression.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": CONFIG_FP16_ENABLED, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "bf16": { - "enabled": CONFIG_BF16_ENABLED - }, - "curriculum_learning": { - "enabled": CONFIG_CL_ENABLED, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - }, - - "wall_clock_breakdown" : false, - - "compression_training": { - "weight_quantization": { - "shared_parameters":{ - "enabled": true, - "quantizer_kernel": false, - "schedule_offset": 50, - "quantize_groups": 48, - "quantize_verbose": false, - "quantization_type": "symmetric", - "rounding": "nearest", - "fp16_mixed_quantize":{ - "enabled": false, - "quantize_change_ratio": 0.001 - } - }, - "different_groups":{ - "wq1": { - "params": { - "start_bits": 12, - "target_bits": 4, - "quantization_period": 50 - }, - "modules": [ - "encoder.layers" - ] - } - } - }, - "activation_quantization": { - "shared_parameters":{ - "enabled": true, - "quantization_type": "asymmetric", - "range_calibration": "static", - "schedule_offset": 50 - }, - "different_groups":{ - "aq1": { - "params": { - "bits": 8 - }, - "modules": [ - "encoder.layers" - ] - } - } - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_evalharness.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_evalharness.sh deleted file mode 100644 index 0922dc033..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_evalharness.sh +++ /dev/null @@ -1,75 +0,0 @@ -# This is an example zero-shot eval script. Please first read the readme_evalharness.md under the ../MoE directory. - -# CHECKPOINT_PATH=/blob/users/minjiaz/compression_library/checkpoint/125M10L_Compression_Test_INT8_64gpu_lr6e-5_tokens5.25B_nocl_alpha-no_pp/global_step2000/ -# CHECKPOINT_PATH=/blob/users/conglli/project/gpt3_with_pile/checkpoint/gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-64-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/global_step71000/ -# CHECKPOINT_PATH=/blob/users/minjiaz/compression_library/checkpoint/125M12L_Compression_Test_INT8_64gpu_lr6e-5_tokens5.25B_nocl_alpha-no_pp/global_step5000/ -CHECKPOINT_PATH=/blob/users/minjiaz/project/gpt3_distillation/checkpoint/gpt3-kd-test2-alpha1-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-15-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/global_step71426/ -CONFIG_PATH=ds_config_gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus--1-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B.json -RESULT_PATH=gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-128-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-20728-token-45B_global_step81566.log - -PP_SIZE=1 -TP_SIZE=1 -NO_PP="true" -EP_PARALLEL_SIZE=1 -# Currently eval harness does not support data parallel -# However, for MoE models it's possible to enable a "fake data parallel" -# in order to load experts on multiple gpus. At the same time, it's not -# real data parallel because we load the same data on all gpus. -# On the other hand, it's better to use less number of gpus than training, -# to reduce communication overhead. -NUM_NODE=1 -NUM_GPU_PER_NODE=1 - -# TASKS="lambada" -# WikiText-2, not used in GPT-3 paper but used in GPT-2 paper -TASKS="lambada,wikitext" -# Tasks that appeared in GPT-3 paper (sorted based on the order in paper), plus WikiText-2. -# TASKS="hellaswag,lambada,triviaqa,webqs,winogrande,piqa,arc_challenge,arc_easy,openbookqa,race,boolq,cb,copa,rte,wic,wsc,multirc,record,anli_r1,anli_r2,anli_r3,wikitext" -# All tasks that confirmed to work, there are more tasks on https://github.com/EleutherAI/lm-evaluation-harness that we didn't test. -# TASKS="hellaswag,lambada,triviaqa,webqs,winogrande,piqa,arc_challenge,arc_easy,openbookqa,race,boolq,cb,copa,rte,wic,wsc,multirc,record,anli_r1,anli_r2,anli_r3,wikitext,logiqa,mathqa,mc_taco,mrpc,prost,pubmedqa,qnli,qqp,sciq,sst,wnli" - -VOCAB_FILE=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_FILE=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - -# export HF_DATASETS_OFFLINE=1 - -# Dummy arguments to make megatron happy. No need to configure them. -# The reason we don't need to configure them and many other arguments is -# because the eval framework will read the arguments from checkpoint file. -MEGATRON_REQUIRED_ARGS="\ - --num-layers -1\ - --hidden-size -1\ - --num-attention-heads -1\ - --seq-length -1 \ - --max-position-embeddings -1 -" - -CMD="../../tasks/eval_harness/evaluate.py \ - --load $CHECKPOINT_PATH\ - --tensor-model-parallel-size $TP_SIZE \ - --pipeline-model-parallel-size $PP_SIZE\ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --vocab-file $VOCAB_FILE\ - --merge-file $MERGE_FILE\ - --micro-batch-size 12\ - --no-load-optim \ - --no-load-rng \ - --inference \ - --disable-moe-token-dropping \ - --tokenizer-type GPT2BPETokenizer \ - --adaptive_seq_len\ - --eval_fp32\ - --task_list $TASKS\ - --results_path $RESULT_PATH \ - --deepspeed \ - --deepspeed_config $CONFIG_PATH \ - $MEGATRON_REQUIRED_ARGS\ - " - -if [[ "${NO_PP}" = "true" ]]; then -CMD="${CMD} \ - --no-pipeline-parallel" -fi - -LAUNCHER="deepspeed --num_nodes $NUM_NODE --num_gpus $NUM_GPU_PER_NODE" -$LAUNCHER $CMD \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_1.3B_dense_cl_kd.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_1.3B_dense_cl_kd.sh deleted file mode 100644 index 9ffa240db..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_1.3B_dense_cl_kd.sh +++ /dev/null @@ -1,322 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -MODEL_SIZE=1.3 -NUM_LAYERS=24 -HIDDEN_SIZE=2048 -NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -MIN_LR=2.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -GLOBAL_BATCH_SIZE=4096 # 8x -LR=8.0e-4 # 4x - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -TRAIN_TOKENS=300000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=16 - -## Model parallelism, 1 is no MP -MP_SIZE=2 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="true" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.013 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt3-kd-with-pile-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-zero-${ZERO_STAGE}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/minjiaz/project/gpt3_distillation/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -### KD configs -KD_BETA_CE=1 -CHECKPOINT_PATH_TEACHER="/blob/users/conglli/project/gpt3_with_pile/checkpoint/gpt3-with-pile-1.3B-lr-8.0e-4-minlr-2.0e-5-bs-4096-gpus-128-zero-0-mp-2-pp-1-no_pp-cl-startseqlen-80-step-13767-token-60B/" -CHECKPOINT_PATH_SAVE="/blob/users/minjiaz/project/gpt3_distillation/checkpoint/${NAME}" - -mkdir -p ${CHECKPOINT_PATH_SAVE} - -VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# DATA_PATH=/data/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document - -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 21 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH_SAVE} \ - --kd \ - --kd-beta-ce ${KD_BETA_CE} \ - --num-layers-teacher ${NUM_LAYERS} \ - --hidden-size-teacher ${HIDDEN_SIZE} \ - --num-attention-heads-teacher ${NUM_ATTN_HEADS} \ - --load-teacher ${CHECKPOINT_PATH_TEACHER} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_cl_kd.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_cl_kd.sh deleted file mode 100644 index a34ce282c..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_cl_kd.sh +++ /dev/null @@ -1,323 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -MIN_LR=6.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -GLOBAL_BATCH_SIZE=2048 # 8x -LR=2.4e-3 # 4x - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -TRAIN_TOKENS=300000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=8 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="true" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=72 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.02 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt3-kd-test1-alpha1-with-pile-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-zero-${ZERO_STAGE}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/minjiaz/project/gpt3_distillation/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -### KD configs -KD_BETA_CE=1 -CHECKPOINT_PATH_TEACHER="/blob/users/conglli/project/gpt3_with_pile/checkpoint/gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-64-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/" -CHECKPOINT_PATH_SAVE="/blob/users/minjiaz/project/gpt3_distillation/checkpoint/${NAME}" - -mkdir -p ${CHECKPOINT_PATH_SAVE} - - -VOCAB_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 -# DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 10 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH_SAVE} \ - --kd \ - --kd-beta-ce ${KD_BETA_CE} \ - --num-layers-teacher ${NUM_LAYERS} \ - --hidden-size-teacher ${HIDDEN_SIZE} \ - --num-attention-heads-teacher ${NUM_ATTN_HEADS} \ - --load-teacher ${CHECKPOINT_PATH_TEACHER} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_kd.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_kd.sh deleted file mode 100644 index 54f912271..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_125M_dense_kd.sh +++ /dev/null @@ -1,323 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -MODEL_SIZE=0.125 -NUM_LAYERS=12 -HIDDEN_SIZE=768 -NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -MIN_LR=6.0e-5 - -# Curriculum learning (CL) enables stable large-batch training -GLOBAL_BATCH_SIZE=2048 # 8x -LR=2.4e-3 # 4x - -## GPT-3 Medium 350M -# MODEL_SIZE=0.35 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1024 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=3.0e-4 -# MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -TRAIN_TOKENS=300000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=8 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism. To disable PP, set PP_SIZE to 1 and NO_PP to true. -PP_SIZE=1 -NO_PP="true" - -## ZeRO stage -ZERO_STAGE=0 - -## Total number of GPUs -NUM_GPUS=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -NUM_GPUS_PERNODE=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -NUM_NODE=$(( ${NUM_GPUS} / ${NUM_GPUS_PERNODE} )) -DP_SIZE=$(( ${NUM_GPUS} / ${PP_SIZE} / ${MP_SIZE} )) -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=72 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_STEP=$(( ${CL_TOKENS} * 1000000000 / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=10000 - -## Standard deviation for weight initialization. Usually larger model needs -## lower std. We used a heuristic equation of sqrt(1/3/HIDDEN_SIZE) from the -## MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) -INIT_STD=0.02 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -LOG_OPTIMIZER_STATE="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt3-kd-test1-alpha1-with-pile-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-zero-${ZERO_STAGE}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [ "${NO_PP}" = "true" ]; then - NAME="${NAME}-no_pp" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-startseqlen-${CL_START_SEQLEN}-step-${CL_STEP}-token-${CL_TOKENS}B" -fi - -LOG_PATH="log/" -TENSORBOARD_PATH="tensorboard/${NAME}_${host}_${current_time}" -CHECKPOINT_PATH="/blob/users/minjiaz/project/gpt3_distillation/checkpoint/${NAME}" -mkdir -p ${LOG_PATH} -mkdir -p ${TENSORBOARD_PATH} -mkdir -p ${CHECKPOINT_PATH} - -### KD configs -KD_BETA_CE=1 -CHECKPOINT_PATH_TEACHER="/blob/users/conglli/project/gpt3_with_pile/checkpoint/gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-64-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B/" -CHECKPOINT_PATH_SAVE="/blob/users/minjiaz/project/gpt3_distillation/checkpoint/${NAME}" - -mkdir -p ${CHECKPOINT_PATH_SAVE} - - -VOCAB_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json -MERGE_PATH=/blob/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -# For cluster Azure-EastUS-V100-32GB-4, Lab-RR1-V100 -# DATA_PATH=/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing/pile_text_document -# For cluster Azure-WestUS3-A100 -DATA_PATH=/blob/data/the_pile_public_merged_nopreprocessing/pile_text_document -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_PATH} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers 10 \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH_SAVE} \ - --kd \ - --kd-beta-ce ${KD_BETA_CE} \ - --num-layers-teacher ${NUM_LAYERS} \ - --hidden-size-teacher ${HIDDEN_SIZE} \ - --num-attention-heads-teacher ${NUM_ATTN_HEADS} \ - --load-teacher ${CHECKPOINT_PATH_TEACHER} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_PATH}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${LOG_OPTIMIZER_STATE}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_${NAME}.json" -if [[ $ZERO_STAGE -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/${ZERO_STAGE}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${ZERO_STAGE} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -if [[ "${NO_PP}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -ITERATION_FILE="$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" -ITERATION_FILE_2="$CHECKPOINT_PATH/latest" -ITERATION=0 -for (( node = 0; node <= NUM_NODE-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$ITERATION_FILE\""); then - LOCAL_ITERATION=$(ssh -q worker-"$node" cat $ITERATION_FILE) - ITERATION=$(( ${LOCAL_ITERATION} > ${ITERATION} ? ${LOCAL_ITERATION} : ${ITERATION} )) - fi -done -if [[ $ITERATION -gt 0 ]]; then - ITERATION_2="global_step${ITERATION}" - ds_ssh "echo $ITERATION > $ITERATION_FILE" - ds_ssh "echo $ITERATION_2 > $ITERATION_FILE_2" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${LOG_PATH}/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_350M_dense_kd.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_350M_dense_kd.sh deleted file mode 100644 index 4366be67e..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/compression/ds_pretrain_gpt_350M_dense_kd.sh +++ /dev/null @@ -1,348 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -MODEL_SIZE=0.35 -NUM_LAYERS=24 -HIDDEN_SIZE=1024 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=256 -LR=3.0e-4 -MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -# LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=4 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=64 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -EP_SIZE=1 -# EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -# LR=2.0e-4 -# MIN_LR=2e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=10 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-kd-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - -# USE_INTERNAL_DATA="true" -USE_INTERNAL_DATA="false" - -if [ "${USE_INTERNAL_DATA}" = "true" ]; then - ## The internal data is only accessible within Microsoft - ## For cluster Azure-EastUS-V100-32GB-4, Azure-WestUS3-A100 - # BASE_DATA_PATH=/vc_data/Megatron-LM/data - # DATA_HOME="/vc_data/pile-cc1-cc2-shuf" - ## For cluster Lab-RR1-V100 - BASE_DATA_PATH=/data/Megatron-LM/data - DATA_HOME="/turing-ssd/users/conglli/data/pile-cc1-cc2-shuf" - ## For cluster Azure-CentralUS-A100 - # BASE_DATA_PATH=/data/Megatron-LM/data - # DATA_HOME=/vc_data_1/users/amawa/blended - - VOCAB_PATH=${BASE_DATA_PATH}/gpt2-vocab.json - MERGE_PATH=${BASE_DATA_PATH}/gpt2-merges.txt - ARX="${DATA_HOME}/ArXiv_ftfy_cleaned_id_shuf_text_document" - BC2="${DATA_HOME}/BookCorpus2_ftfy_cleaned_id_shuf_text_document" - B3="${DATA_HOME}/Books3_ftfy_cleaned_id_shuf_text_document" - CC2020="${DATA_HOME}/CC-2020-50_id_cleaned_shuf_text_document" - CC2021="${DATA_HOME}/CC-2021-04_id_cleaned_shuf_text_document" - GIT="${DATA_HOME}/Github_ftfy_id_shuf_text_document" - GUT="${DATA_HOME}/Gutenberg_PG-19_ftfy_cleaned_id_cleaned_shuf_text_document" - NIH="${DATA_HOME}/NIH_ExPorter_ftfy_id_shuf_text_document" - OWT2="${DATA_HOME}/OpenWebText2_ftfy_cleaned_id_shuf_text_document" - PCC="${DATA_HOME}/Pile-CC_id_cleaned_shuf_text_document" - PM="${DATA_HOME}/PubMed_Abstracts_ftfy_id_shuf_text_document" - RN="${DATA_HOME}/rn_dedup_shuf_cleaned_0.7_cleaned_shuf_text_document" - SE="${DATA_HOME}/StackExchange_ftfy_id_shuf_text_document" - ST="${DATA_HOME}/stories_dedup0.7_shuf_cleaned_shuf_text_document" - WIK="${DATA_HOME}/Wikipedia_en_ftfy_id_shuf_text_document" - DATA_BLEND="0.14336 ${B3} 0.08962 ${RN} 0.19336 ${OWT2} 0.05689 ${SE} \ - 0.00859 ${ST} 0.02897 ${PM} 0.04771 ${WIK} 0.00873 ${GUT} 0.01007 ${BC2} \ - 0.00208 ${NIH} 0.13017 ${CC2020} 0.09446 ${PCC} 0.15652 ${CC2021} \ - 0.01359 ${ARX} 0.01588 ${GIT}" -else - VOCAB_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-vocab.json - MERGE_PATH=/data/the_pile_public_merged_nopreprocessing/gpt2-merges.txt - # Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ - DATA_BLEND=/data/the_pile_public_merged_nopreprocessing/pile_text_document -fi -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/0/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/README.md deleted file mode 100644 index a80e3510c..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/README.md +++ /dev/null @@ -1 +0,0 @@ -This is an example of how to use DeepSpeed's curriculum learning (CL) feature which provides faster and more stable language model pre-training. Currently it is only integrated for GPT pre-training. Note that there are two curriculum learning examples in two different repos for Megatron-LM GPT-2 pre-training. Both of them have some unique features and limitations. See details in our [tutorial](https://www.deepspeed.ai/tutorials/curriculum-learning/). For technical details please refer to our [paper](https://arxiv.org/abs/2108.06084). \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_config_gpt_slw_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_config_gpt_slw_TEMPLATE.json deleted file mode 100644 index f1abcedcb..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_config_gpt_slw_TEMPLATE.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false, - "curriculum_learning": { - "enabled": true, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt2.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt2.sh deleted file mode 100644 index 96a618666..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt2.sh +++ /dev/null @@ -1,150 +0,0 @@ -#! /bin/bash - -CONFIG=$1 -TAG=$2 -MODEL_SIZE=$3 -LR=$4 -TOTAL_BATCHSIZE=$5 -SEQ_LEN=$6 -MP_SIZE=$7 -SEED=$8 -SAVE_INTERVAL=$9 -NUM_ITER=${10} -NUM_TOKEN=${11} -LR_DECAY_TOKEN=${12} -LR_WARMUP_ITER=${13} -CONFIG_TEMPLATE=${14} -CURRICULUM_STEP=${15} -CURRICULUM_MIN=${16} - -# 12-layer, 768-hidden, 12-heads, 117M parameters -# 24-layer, 1024-hidden, 16-heads, 345M parameters -# 36-layer, 1280-hidden, 20-heads, 774M parameters -# 48-layer, 1600-hidden, 25-heads, 1558M parameters -if [[ $MODEL_SIZE -eq 117 ]]; then - NUM_LAYERS=12 - HIDDEN_SIZE=768 - NUM_ATTN_HEADS=12 -elif [[ $MODEL_SIZE -eq 345 ]]; then - NUM_LAYERS=24 - HIDDEN_SIZE=1024 - NUM_ATTN_HEADS=16 -elif [[ $MODEL_SIZE -eq 774 ]]; then - NUM_LAYERS=36 - HIDDEN_SIZE=1280 - NUM_ATTN_HEADS=20 -elif [[ $MODEL_SIZE -eq 1558 ]]; then - NUM_LAYERS=48 - HIDDEN_SIZE=1600 - NUM_ATTN_HEADS=25 -else - echo "Model size not supported." - exit 1 -fi - -# Pipeline parallelism. 1 means no pipelines. -PP_SIZE=1 - -# Change for multinode config -NUM_WORKERS=16 -NUM_GPUS_PER_WORKER=8 -NUM_GPUS=$(( ${NUM_WORKERS} * ${NUM_GPUS_PER_WORKER} )) -if [[ $PP_SIZE -gt 0 ]]; then - DP_SIZE=$(( ${NUM_GPUS} / (${PP_SIZE} * ${MP_SIZE}) )) -else - DP_SIZE=$(( ${NUM_GPUS} / ${MP_SIZE} )) -fi -# Batch size per gpu, here we assume grad accumulation step 1 -# you can reduce this if gpu OOM -BATCHSIZE=$((TOTAL_BATCHSIZE/DP_SIZE)) - -DATA_PATH=/vc_data/Megatron-LM/data/indexed_datasets/megatron -VOCAB_PATH=/vc_data/Megatron-LM/data/gpt2-vocab.json -MERGE_PATH=/vc_data/Megatron-LM/data/gpt2-merges.txt - -#ZeRO Configs -stage=1 - -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -script_path=$(realpath $0) -script_dir=$(dirname $script_path) -host="${HOSTNAME}" - -if [ "${CONFIG_TEMPLATE}" = "true" ]; then -template_json="$script_dir/ds_zero_stage_${stage}_config_${CONFIG}.json" -config_json="$script_dir/ds_zero_stage_${stage}_config_${CONFIG}_min${CURRICULUM_MIN}_max${SEQ_LEN}_step${CURRICULUM_STEP}.json" -sed "s/CONFIG_CL_MIN/${CURRICULUM_MIN}/" ${template_json} \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CURRICULUM_STEP}/" \ - > ${config_json} -else -config_json="$script_dir/ds_zero_stage_${stage}_config_${CONFIG}.json" -fi - -JOB_NAME="gpt2_${MODEL_SIZE}M_bsz${TOTAL_BATCHSIZE}_seq${SEQ_LEN}_lr${LR}_warmup${LR_WARMUP_ITER}_decay${LR_DECAY_TOKEN}_seed${SEED}_${TAG}_stage${stage}_n${NUM_WORKERS}_g${NUM_GPUS_PER_WORKER}_mp${MP_SIZE}" -LOG_NAME="${JOB_NAME}_${host}_${current_time}" - -OUTPUT_BASEPATH="/vc_data_blob/users/conglli" -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/curriculum/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/curriculum/" -mkdir -p "${OUTPUT_BASEPATH}/log/curriculum/" -LOGDIR="${OUTPUT_BASEPATH}/tensorboard/curriculum/${LOG_NAME}" -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/curriculum/${JOB_NAME}" - -gpt_options=" \ - --tensor-model-parallel-size ${MP_SIZE} \ - --num-layers $NUM_LAYERS \ - --hidden-size $HIDDEN_SIZE \ - --num-attention-heads $NUM_ATTN_HEADS \ - --seq-length $SEQ_LEN \ - --max-position-embeddings $SEQ_LEN \ - --micro-batch-size $BATCHSIZE \ - --global-batch-size ${TOTAL_BATCHSIZE} \ - --train-iters $NUM_ITER \ - --train-tokens $NUM_TOKEN \ - --lr-decay-tokens $LR_DECAY_TOKEN \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH \ - --data-path $DATA_PATH \ - --vocab-file $VOCAB_PATH \ - --merge-file $MERGE_PATH \ - --data-impl mmap \ - --split 949,50,1 \ - --distributed-backend nccl \ - --override-opt_param-scheduler \ - --lr $LR \ - --lr-decay-style cosine \ - --min-lr 1.0e-5 \ - --weight-decay 1e-2 \ - --clip-grad 1.0 \ - --lr-warmup-iters $LR_WARMUP_ITER \ - --checkpoint-activations \ - --log-interval 100 \ - --save-interval $SAVE_INTERVAL \ - --eval-interval 100 \ - --eval-iters 10 \ - --fp16 \ - --seed $SEED \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --no-masked-softmax-fusion \ - --tensorboard-dir ${LOGDIR} -" - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${stage} \ - --pipeline-model-parallel-size ${PP_SIZE} \ - --deepspeed-activation-checkpointing -" - -full_options="${gpt_options} ${deepspeed_options}" - -run_cmd="deepspeed --num_nodes ${NUM_WORKERS} --num_gpus ${NUM_GPUS_PER_WORKER} ../../pretrain_gpt.py ${full_options} &>> ${OUTPUT_BASEPATH}/log/curriculum/${JOB_NAME}.log" -echo ${run_cmd} -eval ${run_cmd} - -set +x diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt_1.3B_rope_slw.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt_1.3B_rope_slw.sh deleted file mode 100644 index 209021a39..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_pretrain_gpt_1.3B_rope_slw.sh +++ /dev/null @@ -1,347 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=512 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=4 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=8 -no_pp="false" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### curriculum learning (sequence length warmup) configs -# The "divided by 3" means we use 1/3 of baseline's total steps for sequence length warmup. -# This is not always the best config, but usually a reasonable choice to start with. -cl_step=$(( ${lr_warmup_tokens} / 3 / ${global_batch_size} / ${seq_len} )) -# Starting sequence length during sequence length warmup. If the train/validation loss is -# unstable at the beginning of training, need to increase this but also need to keep as multiples -# of 8 in order to enable Tensor Core acceleration. -cl_min=64 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ or -## https://the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase_rope0.25" -jobname="${jobname}_cl_step${cl_step}_cl_min${cl_min}" - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-rotary-position-embeddings \ - --rotary-percent 0.25 \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}_cl_step${cl_step}_cl_min${cl_min}.json" -template_json="ds_config_gpt_slw_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - | sed "s/CONFIG_CL_MIN/${cl_min}/" \ - | sed "s/CONFIG_CL_MAX/${seq_len}/" \ - | sed "s/CONFIG_CL_DURATION/${cl_step}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_train.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_train.sh deleted file mode 100644 index aac11ab03..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_train.sh +++ /dev/null @@ -1,37 +0,0 @@ -# # baseline -# CONFIG=baseline -# TAG=baseline -# MODEL_SIZE=1558 -# LR=1.5e-4 -# BSZ=512 -# SEQ_LEN=1024 -# MP_SIZE=1 -# SEED=1234 -# SAVE_INTERVAL=5000 -# NUM_ITER=600000 -# NUM_TOKEN=157286400000 -# LR_DECAY_TOKEN=157286400000 -# LR_WARMUP_ITER=3000 -# CONFIG_TEMPLATE=false -# CURRICULUM_STEP=0 -# CURRICULUM_MIN=0 - -# curriculum learning -CONFIG=curriculum_fixed_linear -MODEL_SIZE=1558 -LR=6e-4 -BSZ=4096 -SEQ_LEN=1024 -MP_SIZE=1 -SEED=1234 -SAVE_INTERVAL=1000 -NUM_ITER=75000 -NUM_TOKEN=157286400000 -LR_DECAY_TOKEN=157286400000 -LR_WARMUP_ITER=3000 -CONFIG_TEMPLATE=true -CURRICULUM_STEP=45000 -CURRICULUM_MIN=64 -TAG="${CONFIG}_s${CURRICULUM_MIN}to${SEQ_LEN}_step${CURRICULUM_STEP}" - -bash ds_pretrain_gpt2.sh $CONFIG $TAG $MODEL_SIZE $LR $BSZ $SEQ_LEN $MP_SIZE $SEED $SAVE_INTERVAL $NUM_ITER $NUM_TOKEN $LR_DECAY_TOKEN $LR_WARMUP_ITER $CONFIG_TEMPLATE $CURRICULUM_STEP $CURRICULUM_MIN diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_baseline.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_baseline.json deleted file mode 100644 index 71494f374..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_baseline.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "train_batch_size": 512, - "gradient_accumulation_steps": 1, - "steps_per_print": 1, - "zero_optimization": { - "stage": 1 - }, - "optimizer": { - "type": "Adam", - "params": { - "lr": 0.00015, - "max_grad_norm": 1.0, - "betas": [0.9, 0.95] - } - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 1000, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "wall_clock_breakdown": false, - "zero_allow_untested_optimizer": false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_curriculum_fixed_linear.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_curriculum_fixed_linear.json deleted file mode 100644 index e2f947830..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/curriculum_learning/ds_zero_stage_1_config_curriculum_fixed_linear.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "train_batch_size": 512, - "gradient_accumulation_steps": 1, - "steps_per_print": 1, - "zero_optimization": { - "stage": 1 - }, - "optimizer": { - "type": "Adam", - "params": { - "lr": 0.00015, - "max_grad_norm": 1.0, - "betas": [0.9, 0.95] - } - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 1000, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "wall_clock_breakdown": false, - "zero_allow_untested_optimizer": false, - "curriculum_learning": { - "enabled": true, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/README.md deleted file mode 100644 index 7ed96ae72..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/README.md +++ /dev/null @@ -1,23 +0,0 @@ -This directory includes GPT-3/BERT pretraining example scripts for DeepSpeed Data Efficiency Library technologies (curriculum learning, random-LTD, and the two composed together). - -You need to install updated DeepSpeed version (>=0.8.0), which contains the DeepSpeed Data Efficiency Library. - -Additional tutorial can be found at [DeepSpeed website](https://www.deepspeed.ai/tutorials/data-efficiency/). - -Additional technical details can be found in our [random-LTD paper](https://arxiv.org/abs/2211.11586) and [data efficiency paper](https://arxiv.org/abs/2212.03597). - -## GPT-3 pretraining and evaluation -Inside ``gpt`` folder, first the ``ds_analyze_gpt_data_map.sh`` and ``ds_analyze_gpt_data_reduce.sh`` are used for curriculum learning's offline data analysis and indexing. - -``gpt/pretrain`` includes the pretraining example scripts. You can choose a setup to run by uncommenting one block in ``ds_pretrain_gpt_1.3B_dense_run.sh``. One thing to note is that in our [random-LTD paper](https://arxiv.org/abs/2211.11586) we did not scale peak learning rate when using less than 100% data, while in our later [data efficiency paper](https://arxiv.org/abs/2212.03597) we find that scaling LR based on used percentage of data helps improve model quality. - -``gpt/eval`` includes the zero-/few-shot evaluation example scripts. ``ds_evalharness_parallel_run.sh`` is for zero-shot, and ``ds_evalharness_parallel_run_10shot.sh`` is for 10-shot. - -## BERT pretraining and finetuning -Inside ``bert`` folder, first the ``pile_data_download_preprocess.py`` can be used to download and preprocess the public Pile dataset. - -The ``ds_analyze_bert_data_map.sh`` and ``ds_analyze_bert_data_reduce.sh`` are used for curriculum learning's offline data analysis and indexing. - -``bert/pretrain`` includes the pretraining example scripts. You can choose a setup to run by uncommenting one block in ``ds_pretrain_bert_336M_run.sh``. One thing to note is that in our [random-LTD paper](https://arxiv.org/abs/2211.11586) we did not scale peak learning rate when using less than 100% data, while in our later [data efficiency paper](https://arxiv.org/abs/2212.03597) we find that scaling LR based on used percentage of data helps improve model quality. - -``bert/finetune`` includes the MNLI/QQP/RACE finetuning example scripts following the [Megatron-LM paper](https://arxiv.org/abs/1909.08053). However, we found that the RACE task's accuracy is not very stable and the Megatron-LM paper used a very long number of epochs for MNLI/QQP which is not necessary. Thus we added capability of finetuning other GLUE tasks, and switched to follow the hyperparameters of the [original BERT paper](https://arxiv.org/abs/1810.04805). The corresponding scripts are at ``bert/finetune_glue``, which we recommend to use instead of ``bert/finetune``. Our [data efficiency paper](https://arxiv.org/abs/2212.03597) also uses the scripts under ``bert/finetune_glue`` for GLUE finetuning. \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/analyze_data.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/analyze_data.py deleted file mode 100644 index fec6aa5c7..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/analyze_data.py +++ /dev/null @@ -1,239 +0,0 @@ -# coding=utf-8 -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -''' -Copyright 2022 The Microsoft DeepSpeed Team -''' - -import os -import time -import sys -import math -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir,os.path.pardir))) -from datetime import datetime -import numpy as np -import torch - -from deepspeed.runtime.data_pipeline.data_sampling.data_analyzer \ - import DataAnalyzer -from deepspeed.runtime.data_pipeline.data_sampling.indexed_dataset \ - import MMapIndexedDataset - -from megatron_ds import get_args -from megatron_ds import print_rank_0 -from megatron_ds.initialize import initialize_megatron - -def get_tasks_args(parser): - """Provide extra arguments required for data analyzing.""" - group = parser.add_argument_group(title='data_analyzing') - - group.add_argument('--analyzing-task', type=str, required=True, - default=None, - choices=['map', - 'reduce'], - help='What type of analyzing task to perform.') - group.add_argument('--analyzing-data-type', type=str, required=True, - default=None, - choices=['BERT', - 'GPT'], - help='What type of data.') - group.add_argument('--analyzing-metric', type=str, nargs='+', default=[], - help='What kinds of metrics to analyze.') - group.add_argument('--analyzing-num-workers', type=int, default=1, - help='Number of workers. Each worker could be a single CPU node.') - group.add_argument('--analyzing-worker-id', type=int, default=0, - help='Worker id of current node.') - group.add_argument('--analyzing-num-threads', type=int, default=1, - help='Number of threads for each worker.') - group.add_argument('--analyzing-num-threads-reduce', type=int, default=1, - help='Number of threads for each worker.') - group.add_argument('--analyzing-specific-threads', type=int, nargs='+', default=[], - help='Which specific threads to run. Helpful when there are specific thread failed in previous run.') - return parser - -def train_valid_test_datasets_provider_gpt(): - """Build train, valid, and test datasets.""" - args = get_args() - - print_rank_0('> building train, validation, and test datasets ' - 'for GPT ...') - from megatron_ds.data.gpt_dataset import build_train_valid_test_datasets - train_ds, valid_ds, test_ds = build_train_valid_test_datasets( - data_prefix=args.data_path, - data_impl=args.data_impl, - splits_string=args.split, - train_valid_test_num_samples=[1,1,1], # Just dummy numbers since we assume args.train_data_exact_num_epochs will override them - seq_length=args.seq_length, - seed=args.seed, - skip_warmup=(not args.mmap_warmup)) - print_rank_0("> finished creating GPT datasets ...") - - return train_ds, valid_ds, test_ds - -def train_valid_test_datasets_provider_bert(): - """Build train, valid, and test datasets.""" - args = get_args() - - print_rank_0('> building train, validation, and test datasets ' - 'for BERT ...') - from megatron_ds.data.dataset_utils import build_train_valid_test_datasets - train_ds, valid_ds, test_ds = build_train_valid_test_datasets( - data_prefix=args.data_path, - data_impl=args.data_impl, - splits_string=args.split, - train_valid_test_num_samples=[1,1,1], # Just dummy numbers since we assume args.train_data_exact_num_epochs will override them - max_seq_length=args.seq_length, - masked_lm_prob=args.mask_prob, - short_seq_prob=args.short_seq_prob, - seed=args.seed, - skip_warmup=(not args.mmap_warmup), - binary_head=args.bert_binary_head) - print_rank_0("> finished creating BERT datasets ...") - - return train_ds, valid_ds, test_ds - -def metric_seqlen(data): - metric = torch.count_nonzero(data['padding_mask'], dim=1) - return metric - -def metric_total_vocab_freq(data): - args = get_args() - if args.analyzing_data_type == 'BERT': - frequency = torch.bincount(data['text'].view(-1), - minlength=args.padded_vocab_size+1, - weights=data['padding_mask'].view(-1)) - elif args.analyzing_data_type == 'GPT': - frequency = torch.bincount(data['text'].view(-1), - minlength=args.padded_vocab_size+1) - return frequency - -def metric_vocab_rarity(data): - args = get_args() - if args.analyzing_data_type == 'BERT': - rarity = torch.sum(data['padding_mask'] * \ - args.total_vocab_freq[data['text']], dim=1).to(torch.long) - elif args.analyzing_data_type == 'GPT': - rarity = [] - # Do one by one to avoid too high memory consumption - for row in range(data['text'].size()[0]): - rarity.append(int(torch.sum(args.total_vocab_freq[data['text'][row]]).item())) - rarity = torch.tensor(rarity, dtype=torch.long) - print(f"rarity min {min(rarity)}, max {max(rarity)}, len {len(rarity)}, avg {sum(rarity)/len(rarity)}") - return rarity - -def metric_seqlen_vocab_rarity(data): - args = get_args() - metric = torch.count_nonzero(data['padding_mask'], dim=1).to(torch.long) * args.seqlen_coeff - metric += torch.sum(data['padding_mask'] * \ - args.total_vocab_freq[data['text']], dim=1).to(torch.long) - print(f"metric min {min(metric)}, max {max(metric)}, len {len(metric)}, avg {sum(metric)/len(metric)}") - return metric - -def get_metric_function(metric_name): - if metric_name == 'seqlen': - return metric_seqlen - if metric_name == 'total_vocab_freq': - return metric_total_vocab_freq - if metric_name == 'vocab_rarity': - return metric_vocab_rarity - if metric_name == 'seqlen_vocab_rarity': - return metric_seqlen_vocab_rarity - -def get_metric_type(metric_name): - if metric_name == 'seqlen': - return 'single_value_per_sample' - if metric_name == 'total_vocab_freq': - return 'accumulate_value_over_samples' - if metric_name == 'vocab_rarity': - return 'single_value_per_sample' - if metric_name == 'seqlen_vocab_rarity': - return 'single_value_per_sample' - -def run_map(): - args = get_args() - if args.analyzing_data_type == 'BERT': - args.mask_prob = 0 # When analyzing data, we don't want any mask. - train_ds, _, _ = train_valid_test_datasets_provider_bert() - elif args.analyzing_data_type == 'GPT': - train_ds, _, _ = train_valid_test_datasets_provider_gpt() - assert 'seqlen' not in args.analyzing_metric, 'GPT data has fixed seqlen, thus unnecessary to analyze seqlen metric.' - assert 'seqlen_vocab_rarity' not in args.analyzing_metric, 'GPT data has fixed seqlen, thus unnecessary to analyze seqlen metric.' - if 'vocab_rarity' in args.analyzing_metric or 'seqlen_vocab_rarity' in args.analyzing_metric: - total_vocab_freq_fname = f"{args.save}/total_vocab_freq/total_vocab_freq_metric_value" - assert os.path.isfile(f"{total_vocab_freq_fname}.bin") and os.path.isfile(f"{total_vocab_freq_fname}.idx"), "To analyze vocab rarity, first need to analyze the total vocab freq." - total_vocab_freq = MMapIndexedDataset(total_vocab_freq_fname, skip_warmup=True) - total_vocab_freq = np.copy(total_vocab_freq[0]) - total_vocab_freq[total_vocab_freq == 0] = 1 # Avoid log(0) error - total_vocab_freq = np.log(total_vocab_freq/sum(total_vocab_freq)) * -1 - args.total_vocab_freq = torch.tensor(total_vocab_freq, dtype=torch.double) - if 'seqlen_vocab_rarity' in args.analyzing_metric: - # Use large coeff to make seqlen dominates vocab_rarity - max_possible_rarity = args.seq_length * torch.max(args.total_vocab_freq).item() - args.seqlen_coeff = 10 ** (math.ceil(math.log(max_possible_rarity, 10)) + 1) - print(f"Metric seqlen_vocab_rarity: using {args.seqlen_coeff} as coefficient for seqlen.") - metric_functions = [get_metric_function(x) for x in args.analyzing_metric] - metric_types = [get_metric_type(x) for x in args.analyzing_metric] - # For metric_dtypes we int64 by default since it could be hard to estimate - # the appropriate dtype before the mapping analysis. During reduce where - # we merge the analysis results, the DataAnalyzer will automatically choose - # the dtype of merged result file as the smallest one that meet the range - # requirement. - metric_dtypes = [np.int64 for x in args.analyzing_metric] - start = time.time() - data_analyzer = DataAnalyzer(train_ds, - num_workers=args.analyzing_num_workers, - worker_id=args.analyzing_worker_id, - num_threads=args.analyzing_num_threads, - specific_threads=args.analyzing_specific_threads, - batch_size=args.global_batch_size, metric_names=args.analyzing_metric, - metric_functions=metric_functions, metric_types=metric_types, - metric_dtypes=metric_dtypes, save_path=args.save) - data_analyzer.run_map() - duration = (time.time() - start) / 3600.0 - print(f"map job finished in {duration} hr.") - -def run_reduce(): - args = get_args() - if args.analyzing_data_type == 'BERT': - args.mask_prob = 0 # When analyzing data, we don't want any mask. - train_ds, _, _ = train_valid_test_datasets_provider_bert() - elif args.analyzing_data_type == 'GPT': - train_ds, _, _ = train_valid_test_datasets_provider_gpt() - metric_functions = [get_metric_function(x) for x in args.analyzing_metric] - metric_types = [get_metric_type(x) for x in args.analyzing_metric] - metric_dtypes = [np.int64 for x in args.analyzing_metric] - start = time.time() - data_analyzer = DataAnalyzer(train_ds, - num_workers=args.analyzing_num_workers, - num_threads=args.analyzing_num_threads, - num_threads_reduce=args.analyzing_num_threads_reduce, - batch_size=args.global_batch_size, metric_names=args.analyzing_metric, - metric_functions=metric_functions, metric_types=metric_types, - metric_dtypes=metric_dtypes, save_path=args.save) - data_analyzer.run_reduce() - duration = (time.time() - start) / 3600.0 - print(f"reduce job finished in {duration} hr.") - -if __name__ == "__main__": - initialize_megatron(extra_args_provider=get_tasks_args, allow_no_cuda=True) - args = get_args() - if args.analyzing_task == 'map': - run_map() - elif args.analyzing_task == 'reduce': - run_reduce() - else: - raise NotImplementedError('Task {} is not implemented.'.format( - args.analyzing_task)) diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_map.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_map.sh deleted file mode 100644 index 7f23e3615..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_map.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -num_workers=1 # Num nodes to run the map job -num_threads=40 # Num threads on each node. Set this based on #CPU cores - -# If different data epochs have slightly different data samples (e.g., due -# to randomness), then you need to specify large enough num_epochs that cover -# whole pretraining. If different data epochs are the same, set num_epochs to -# 1 to only index 1 epoch, and during pretraining DeepSpeed data efficiency -# library will automatically handle reshuffling when reaching another epoch. -num_epochs=5 - -# Which node is this node (start with 0 and end with num_workers-1). This -# script only launch the map job on 1 worker node, since we don't expect -# running on many nodes and workers don't need any communication. But you -# can modify this script to add a MPI/torch distributed launcher. -worker_id=$1 -save_path="/blob/users/conglli/data/analysis_pile_bert_${num_epochs}epoch/" - -metric='total_vocab_freq' -# metric='vocab_rarity' # this requires the result of total_vocab_freq -# metric='seqlen_vocab_rarity' # this requires the result of total_vocab_freq -# metric='seqlen' - -seq_len=512 -batch_size=10000 - -jobname="bert-pile-analyzing-${metric}-${num_epochs}epoch-map-worker${worker_id}" -## Public the Pile dataset, see prepare_pile_data.py in the same directory -## about how to download and preprocess the data. -## Change data_home to your own training data path. -# data_home="/vc_data_blob/users/conglli/the_pile_bert" -data_home="/blob/data/the_pile_bert" -data_path="${data_home}/pile_bert_train_text_sentence" - -vocab_path="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -# Make sure the "--split" is the same as what you will use for pre-training. -options=" \ - --analyzing-task map \ - --analyzing-data-type BERT \ - --analyzing-metric ${metric} \ - --analyzing-num-workers ${num_workers} \ - --analyzing-worker-id ${worker_id} \ - --analyzing-num-threads ${num_threads} \ - --vocab-file ${vocab_path} \ - --data-path ${data_path} \ - --data-impl mmap \ - --tokenizer-type BertWordPieceLowerCase \ - --micro-batch-size ${batch_size} \ - --global-batch-size ${batch_size} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --num-layers 1 \ - --hidden-size 1 \ - --num-attention-heads 1 \ - --split 949,50,1 \ - --distributed-backend gloo \ - --train-data-exact-num-epochs ${num_epochs} \ - --return-data-index \ - --save-interval 1 \ - --save ${save_path}" - -python ../analyze_data.py ${options} &> ${jobname}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_reduce.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_reduce.sh deleted file mode 100644 index f0d14df96..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/ds_analyze_bert_data_reduce.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Set these 2 to the same as what you used during map job. We need these 2 -# configs to know how many map job result files do we have. -num_workers=1 -num_threads=40 -# Reduce job only has 1 worker but can accelerate by multithreading. -num_threads_reduce=40 - -# If different data epochs have slightly different data samples (e.g., due -# to randomness), then you need to specify large enough num_epochs that cover -# whole pretraining. If different data epochs are the same, set num_epochs to -# 1 to only index 1 epoch, and during pretraining DeepSpeed data efficiency -# library will automatically handle reshuffling when reaching another epoch. -num_epochs=5 - -save_path="/blob/users/conglli/data/analysis_pile_bert_${num_epochs}epoch/" - -metric='total_vocab_freq' -# metric='vocab_rarity' # this requires the result of total_vocab_freq -# metric='seqlen_vocab_rarity' # this requires the result of total_vocab_freq -# metric='seqlen' - -seq_len=512 -batch_size=10000 - -jobname="bert-pile-analyzing-${metric}-${num_epochs}epoch-reduce" -## Public the Pile dataset, see prepare_pile_data.py in the same directory -## about how to download and preprocess the data. -## Change data_home to your own training data path. -# data_home="/vc_data_blob/users/conglli/the_pile_bert" -data_home="/blob/data/the_pile_bert" -data_path="${data_home}/pile_bert_train_text_sentence" - -vocab_path="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -# Make sure the "--split" is the same as what you will use for pre-training. -options=" \ - --analyzing-task reduce \ - --analyzing-data-type BERT \ - --analyzing-metric ${metric} \ - --analyzing-num-workers ${num_workers} \ - --analyzing-num-threads ${num_threads} \ - --analyzing-num-threads-reduce ${num_threads_reduce} \ - --vocab-file ${vocab_path} \ - --data-path ${data_path} \ - --data-impl mmap \ - --tokenizer-type BertWordPieceLowerCase \ - --micro-batch-size ${batch_size} \ - --global-batch-size ${batch_size} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --num-layers 1 \ - --hidden-size 1 \ - --num-attention-heads 1 \ - --split 949,50,1 \ - --distributed-backend gloo \ - --train-data-exact-num-epochs ${num_epochs} \ - --return-data-index \ - --save-interval 1 \ - --save ${save_path}" - -python ../analyze_data.py ${options} &> ${jobname}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_config_bert_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_config_bert_TEMPLATE.json deleted file mode 100644 index 1ee35d7ae..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_config_bert_TEMPLATE.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_mnli.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_mnli.sh deleted file mode 100644 index e88f7beb0..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_mnli.sh +++ /dev/null @@ -1,150 +0,0 @@ -seed=1234 -pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 - -## From Table 6 in https://arxiv.org/abs/1909.08053. -task="MNLI" -global_batch_size=128 -lr=1e-5 -epochs=10 - -train_data="/blob/data/GlueData/MNLI/train.tsv" -valid_data="/blob/data/GlueData/MNLI/dev_matched.tsv \ - /blob/data/GlueData/MNLI/dev_mismatched.tsv" - -## Adjust based on number of GPUs. -batch_size=16 - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 -save_interval=500000 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -checkpoint_path="${pretrained_checkpoint}-finetune/${jobname}" -mkdir -p ${checkpoint_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.065 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --save-interval ${save_interval} \ - --save ${checkpoint_path} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev set accuracy numbers by -# "grep -e "overall:" -e "metrics for" ${checkpoint_path}/output.log" -deepspeed ../../../../tasks/main.py ${options} &> ${checkpoint_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_qqp.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_qqp.sh deleted file mode 100644 index 8083e1024..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_qqp.sh +++ /dev/null @@ -1,158 +0,0 @@ -seed=1234 -pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 - -## From Table 6 in https://arxiv.org/abs/1909.08053. -task="QQP" - -train_data="/blob/data/GlueData/QQP/train.tsv" -valid_data="/blob/data/GlueData/QQP/dev.tsv" - -## Adjust based on number of GPUs. -batch_size=16 - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=128 -# lr=5e-5 -# epochs=12 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 -global_batch_size=128 -lr=5e-5 -epochs=12 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 -# global_batch_size=128 -# lr=3e-5 -# epochs=12 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -# global_batch_size=256 -# lr=4e-5 -# epochs=12 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 -save_interval=500000 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -checkpoint_path="${pretrained_checkpoint}-finetune/${jobname}" -mkdir -p ${checkpoint_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.065 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --save-interval ${save_interval} \ - --save ${checkpoint_path} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev set accuracy numbers by -# "grep -e "overall:" -e "metrics for" ${checkpoint_path}/output.log" -deepspeed ../../../../tasks/main.py ${options} &> ${checkpoint_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_race.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_race.sh deleted file mode 100644 index 15658e3d2..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_bert_race.sh +++ /dev/null @@ -1,172 +0,0 @@ -seed=1234 -## RACE have two sub-tasks that need to be finetuned separately -difficulty="middle" -# difficulty="high" -pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 - -## From Table 6 in https://arxiv.org/abs/1909.08053. -task="RACE" - -## Race dataset can be downloaded by: -## wget http://www.cs.cmu.edu/~glai1/data/race/RACE.tar.gz -train_data="/blob/data/RACE/train/${difficulty}" - -## The Megatron paper https://arxiv.org/abs/1909.08053 says: "For the test set -## results of RACE, we first use the development set to find the checkpoint -## that gives us the median score on the 5 random seeds and we report the -## results from that checkpoint on the test set", which is a quite confusing -## description. For simplicity, instead we directly get the median dev and test -## set score on 5 random seeds from a single pretrained_checkpoint. -valid_data="/blob/data/RACE/dev/${difficulty} \ - /blob/data/RACE/test/${difficulty}" - -## Adjust based on number of GPUs. -batch_size=4 - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=32 -# lr=2e-5 -# epochs=3 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 -global_batch_size=32 -lr=2e-5 -epochs=3 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 -# global_batch_size=16 -# lr=1e-5 -# epochs=3 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -# global_batch_size=32 -# lr=2e-5 -# epochs=3 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 -save_interval=100000 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-${difficulty}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -checkpoint_path="${pretrained_checkpoint}-finetune/${jobname}" -mkdir -p ${checkpoint_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.06 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --save-interval ${save_interval} \ - --save ${checkpoint_path} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --clip-grad 1.0 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev/test set accuracy numbers -# by "grep -e "overall:" -e "metrics for" ${checkpoint_path}/output.log" -deepspeed ../../../../tasks/main.py ${options} &> ${checkpoint_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_gather_result.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_gather_result.py deleted file mode 100644 index 6fffe829d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune/ds_finetune_gather_result.py +++ /dev/null @@ -1,111 +0,0 @@ -import os -import statistics - -def gather_numbers(fname, match_keywords, index_keywords, index_offsets): - results = {} - for k in index_keywords: - results[k] = [] - file1 = open(fname, 'r') - while True: - line = file1.readline() - if not line: - break - splits = line.split(' ') - for i in range(len(match_keywords)): - if match_keywords[i] in line: - ref_idx = splits.index(index_keywords[i]) - results[index_keywords[i]].append(float(splits[ref_idx+index_offsets[i]])) - file1.close() - return results - -def gather_MNLI_results(result_path): - overall = [] - matched = [] - mismatched = [] - for file in os.listdir(result_path): - if file.startswith('MNLI'): - fname = f'{result_path}/{file}/output.log' - if os.path.exists(fname): - results = gather_numbers(fname, - ['overall:', 'metrics for dev-matched:', 'metrics for dev-mismatched:'], - ['overall:', 'dev-matched:', 'dev-mismatched:'], - [9, 9, 9]) - overall_candidate = results['overall:'] - matched_candidate = results['dev-matched:'] - mismatched_candidate = results['dev-mismatched:'] - if len(overall_candidate) > 0: - assert len(overall_candidate) == len(matched_candidate) and len(overall_candidate) == len(mismatched_candidate) - best_index = overall_candidate.index(max(overall_candidate)) - overall.append(overall_candidate[best_index]) - matched.append(matched_candidate[best_index]) - mismatched.append(mismatched_candidate[best_index]) - if len(overall) > 0: - if len(overall) % 2 == 1: - median_idx = overall.index(statistics.median(overall)) - else: - median_idx = overall.index(statistics.median_high(overall)) - print(f'MNLI how Megatron paper reported: overall results median {statistics.median(overall)}, corresponding matched/mismatched: {matched[median_idx]}/{mismatched[median_idx]}') - print(f'MNLI other results:') - print(f'MNLI overall results {overall}, median {statistics.median(overall)} (corresponding matched/mismatched {matched[median_idx]}/{mismatched[median_idx]}), mean {statistics.mean(overall)}, std {statistics.stdev(overall)}') - print(f'MNLI matched results {matched}, median {statistics.median(matched)}, mean {statistics.mean(matched)}, std {statistics.stdev(matched)}') - print(f'MNLI mismatched results {mismatched}, median {statistics.median(mismatched)}, mean {statistics.mean(mismatched)}, std {statistics.stdev(mismatched)}') - else: - print("Didn't find any MNLI result") - -def gather_QQP_results(result_path): - overall = [] - for file in os.listdir(result_path): - if file.startswith('QQP'): - fname = f'{result_path}/{file}/output.log' - if os.path.exists(fname): - results = gather_numbers(fname, ['overall:'], ['overall:'], [9]) - overall_candidate = results['overall:'] - if len(overall_candidate) > 0: - best_index = overall_candidate.index(max(overall_candidate)) - overall.append(overall_candidate[best_index]) - if len(overall) > 0: - print(f'QQP how Megatron paper reported: overall results median {statistics.median(overall)}') - print(f'QQP other results:') - print(f'QQP overall results {overall}, median {statistics.median(overall)}, mean {statistics.mean(overall)}, std {statistics.stdev(overall)}') - else: - print("Didn't find any QQP result") - -def gather_RACE_results(result_path, task): - dev = [] - test = [] - for file in os.listdir(result_path): - if file.startswith(f'RACE-{task}'): - fname = f'{result_path}/{file}/output.log' - if os.path.exists(fname): - results = gather_numbers(fname, - [f'metrics for dev-{task}:', f'metrics for test-{task}:'], - [f'dev-{task}:', f'test-{task}:'], - [9, 9]) - dev_candidate = results[f'dev-{task}:'] - test_candidate = results[f'test-{task}:'] - if len(dev_candidate) > 0: - assert len(dev_candidate) == len(test_candidate) - dev.append(max(dev_candidate)) - test.append(max(test_candidate)) - if len(dev) > 0: - if len(dev) % 2 == 1: - median_idx = dev.index(statistics.median(dev)) - else: - median_idx = dev.index(statistics.median_high(dev)) - print(f'RACE-{task} how Megatron paper reported: test result from the median of dev results {test[median_idx]}') - print(f'RACE-{task} other results:') - print(f'RACE-{task} dev results {dev}, median {statistics.median(dev)}, mean {statistics.mean(dev)}, std {statistics.stdev(dev)}') - print(f'RACE-{task} test results {test}, median {statistics.median(test)}, mean {statistics.mean(test)}, std {statistics.stdev(test)}') - else: - print(f"Didn't find any RACE-{task} result") - -def gather_finetune_results(result_path): - print(f'Gather finetune results for {result_path}') - gather_MNLI_results(result_path) - gather_QQP_results(result_path) - gather_RACE_results(result_path, 'middle') - gather_RACE_results(result_path, 'high') - -if __name__ == '__main__': - result_path='/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp-finetune/' - gather_finetune_results(result_path) \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_config_bert_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_config_bert_TEMPLATE.json deleted file mode 100644 index 1ee35d7ae..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_config_bert_TEMPLATE.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue.sh deleted file mode 100644 index 0e0c571a4..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue.sh +++ /dev/null @@ -1,156 +0,0 @@ -hostname_and_rank=$1 -master_port=$2 -seed=$3 -task=$4 -lr=$5 -pretrained_checkpoint=$6 - -# hostname_and_rank="worker-0:0,1,2,3" -# master_port=12345 -# seed=1234 -# task="MNLI" -# lr=2e-5 -# pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -############################################################################### -### Main configs -seq_len=512 - -global_batch_size=32 -epochs=3 - -train_data="/blob/data/GlueData/${task}/train.tsv" -valid_data="/blob/data/GlueData/${task}/dev.tsv" -if [[ "${task}" = "MNLI" ]]; then -valid_data="/blob/data/GlueData/MNLI/dev_matched.tsv \ - /blob/data/GlueData/MNLI/dev_mismatched.tsv" -fi - -## Adjust based on number of GPUs. -batch_size=8 - -## BERT 110M (BERT-Base) -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 - -## BERT 336M (BERT-Large) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO stage -zero_stage=0 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=50 -eval_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" -############################################################################### -vocab_file="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -jobname="${task}-bsz${global_batch_size}-lr${lr}-epochs${epochs}-seed${seed}" -# output_path="${pretrained_checkpoint}-finetune-glue-4v100/${jobname}" -output_path=$(basename "$pretrained_checkpoint") -output_path="glue-results/${output_path}-finetune-glue-4v100/${jobname}" -mkdir -p ${output_path} - -template_json="ds_config_bert_TEMPLATE.json" -config_json="ds_config_bert_bsz${global_batch_size}_mbsz${batch_size}_log${log_interval}_zero${zero_stage}.json" -if [[ $zero_stage -gt 0 ]]; then -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/false/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -else -sed "s/CONFIG_BATCH_SIZE/${global_batch_size}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/true/" \ - | sed "s/CONFIG_BF16_ENABLED/false/" \ - > ${config_json} -fi - -options=" \ - --finetune \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --task ${task} \ - --seed ${seed} \ - --train-data ${train_data} \ - --valid-data ${valid_data} \ - --tokenizer-type BertWordPieceLowerCase \ - --vocab-file ${vocab_file} \ - --epochs ${epochs} \ - --pretrained-checkpoint ${pretrained_checkpoint} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --global-batch-size ${global_batch_size} \ - --micro-batch-size ${batch_size} \ - --lr ${lr} \ - --lr-decay-style linear \ - --lr-warmup-fraction 0.1 \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --weight-decay 1.0e-1 \ - --fp16" - -if [ "${activation_checkpoint}" = "true" ]; then -options="${options} \ - --checkpoint-activations \ - --deepspeed-activation-checkpointing" -fi - -if [[ "${no_pp}" = "true" ]]; then -options="${options} \ - --no-pipeline-parallel" -fi - -# After the fine-tuning finishes, you can find the dev set accuracy numbers by -# "grep -e "overall:" -e "metrics for" ${output_path}/output.log" -deepspeed --include=${hostname_and_rank} --master_port=${master_port} ../../../../tasks/main.py ${options} &> ${output_path}/output.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue_run.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue_run.sh deleted file mode 100644 index 10e04f2c7..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_bert_glue_run.sh +++ /dev/null @@ -1,44 +0,0 @@ -hostname_and_rank=$1 -master_port=$2 -pretrained_checkpoint=$3 - -# hostname_and_rank="worker-0:0,1,2,3" -# master_port=12345 -# pretrained_checkpoint="/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp" - -tasks=( - RTE - MRPC - STS-B - CoLA - SST-2 - QNLI - QQP - MNLI -) - -seeds=( - 1234 - 1235 - 1236 - 1237 - 1238 -) - -lrs=( - 2e-5 - 3e-5 - 4e-5 - 5e-5 -) - -for ((i=0;i<${#tasks[@]};++i)); do - task=${tasks[i]} - for ((j=0;j<${#seeds[@]};++j)); do - seed=${seeds[j]} - for ((k=0;k<${#lrs[@]};++k)); do - lr=${lrs[k]} - bash ds_finetune_bert_glue.sh ${hostname_and_rank} ${master_port} ${seed} ${task} ${lr} ${pretrained_checkpoint} - done - done -done \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_gather_result.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_gather_result.py deleted file mode 100644 index b359ecb6f..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/finetune_glue/ds_finetune_gather_result.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import statistics - -def gather_numbers(fname, match_keywords, index_keywords, index_offsets): - results = {} - for k in index_keywords: - results[k] = [] - file1 = open(fname, 'r') - while True: - line = file1.readline() - if not line: - break - splits = line.split(' ') - for i in range(len(match_keywords)): - if match_keywords[i] in line: - ref_idx = splits.index(index_keywords[i]) - results[index_keywords[i]].append(float(splits[ref_idx+index_offsets[i]])) - file1.close() - return results - -def gather_GLUE_results(result_path, key, lr): - result = [] - mnli_matched_result = [] - mnli_mismatched_result = [] - for file in os.listdir(result_path): - if file.startswith(key) and lr in file: - fname = f'{result_path}/{file}/output.log' - if os.path.exists(fname): - if key == "STS-B": - results = gather_numbers(fname, ['metrics for'], ['spearmanr'], [2]) - overall_candidate = results['spearmanr'] - overall_candidate = [x * 100.0 for x in overall_candidate] - elif key == "CoLA": - results = gather_numbers(fname, ['metrics for'], ['mcc'], [2]) - overall_candidate = results['mcc'] - overall_candidate = [x * 100.0 for x in overall_candidate] - elif key == "MNLI": - results = gather_numbers(fname, - ['overall:', 'metrics for dev-matched:', 'metrics for dev-mismatched:'], - ['overall:', 'dev-matched:', 'dev-mismatched:'], - [9, 9, 9]) - overall_candidate = results['overall:'] - matched_candidate = results['dev-matched:'] - mismatched_candidate = results['dev-mismatched:'] - else: - results = gather_numbers(fname, ['overall:'], ['overall:'], [9]) - overall_candidate = results['overall:'] - if len(overall_candidate) > 0: - if len(overall_candidate) != 3: - print(f"{result_path} task {key} lr {lr} only has {len(overall_candidate)} epoch") - best_index = overall_candidate.index(max(overall_candidate)) - result.append(overall_candidate[best_index]) - if key == "MNLI": - mnli_matched_result.append(matched_candidate[best_index]) - mnli_mismatched_result.append(mismatched_candidate[best_index]) - if len(result) > 0: - if len(result) != 5: - print(f"{result_path} task {key} lr {lr} only has {len(result)} seed") - if key == "MNLI": - best_index = result.index(statistics.median_high(result)) - return round(mnli_matched_result[best_index],2), round(statistics.stdev(mnli_matched_result),2), round(mnli_mismatched_result[best_index],2), round(statistics.stdev(mnli_mismatched_result),2) - else: - return round(statistics.median_high(result),2), round(statistics.stdev(result),2) - else: - if key == "MNLI": - return None, None, None, None - else: - return None, None - -def gather_finetune_results(result_path, extra_col=[], lr="2e-5"): - output = "" - for field in extra_col: - output += f"{field} &" - task_output = "" - median_list, std_list = [], [] - m_median, m_std, mm_median, mm_std = gather_GLUE_results(result_path, "MNLI", lr) - if m_median is not None: - median_list += [m_median, mm_median] - std_list += [m_std, mm_std] - task_output += f"{m_median}±{m_std} & {mm_median}±{mm_std} &" - tasks = ["QQP", "QNLI", "SST-2", "CoLA", "STS-B", "MRPC", "RTE"] - for task in tasks: - t_median, t_std = gather_GLUE_results(result_path, task, lr) - if t_median is not None: - median_list += [t_median] - std_list += [t_std] - if task == "RTE": - task_output += f"{t_median}±{t_std} " - else: - task_output += f"{t_median}±{t_std} &" - overall_median = round(sum(median_list) / len(median_list), 2) - overall_std = round(sum(std_list) / len(std_list), 2) - output += f"{overall_median}±{overall_std} &" - output += task_output - output += " \\\\" - print(output) - -if __name__ == '__main__': - print("\\begin{table}") - print("\centering") - print("\\tiny") - text = "\\begin{tabular}{@{}l|" - for _ in range(11): - text += "c" - text += "@{}}" - print(text) - print("\\toprule") - print("Case & Train tokens & Average & MNLI-m & MNLI-mm & QQP & QNLI & SST-2 & CoLA & STS-B & MRPC & RTE \\\\") - print("\midrule") - - result_path='/blob/users/conglli/project/bert_with_pile/checkpoint/bert-pile-0.336B-iters-2M-lr-1e-4-min-1e-5-wmup-10000-dcy-2M-sty-linear-gbs-1024-mbs-16-gpu-64-zero-0-mp-1-pp-1-nopp-finetune/' - gather_finetune_results(result_path) - - print("\\bottomrule") - print("\end{tabular}") - print("\end{table}") - print("") - print("") \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pile_data_download_preprocess.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pile_data_download_preprocess.py deleted file mode 100644 index 5a020359d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pile_data_download_preprocess.py +++ /dev/null @@ -1,129 +0,0 @@ -import zstandard -import sys -import time -import os - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir,os.path.pardir,os.path.pardir))) -from megatron_ds.data import indexed_dataset - -def pile_download(download_url, file_path, i): - start = time.time() - zstd_file_path = f"{file_path}{i:02}.jsonl.zst" - download_path = f"{download_url}{i:02}.jsonl.zst" - if not os.path.exists(zstd_file_path): - os.system(f"wget -P {file_path} {download_path}") - print(f"Finished downloading chunk {i} in {time.time() - start} sec") - -def pile_decompress(download_url, file_path, i): - zstd_file_path = f"{file_path}{i:02}.jsonl.zst" - output_path = f"{file_path}{i:02}.jsonl" - if not os.path.exists(output_path): - if not os.path.exists(zstd_file_path): - pile_download(download_url, file_path, i) - start = time.time() - with open(zstd_file_path, 'rb') as compressed: - decomp = zstandard.ZstdDecompressor() - with open(output_path, 'wb') as destination: - decomp.copy_stream(compressed, destination) - os.remove(zstd_file_path) - print(f"Finished decompressing chunk {i} in {time.time() - start} sec") - -def pile_preprocess(download_url, file_path, vocab_file, num_workers, i): - json_file_path = f"{file_path}{i:02}.jsonl" - output_prefix = f"{file_path}pile_bert_train_{i:02}" - if not os.path.exists(f"{output_prefix}_text_sentence.idx"): - if not os.path.exists(json_file_path): - pile_decompress(download_url, file_path, i) - start = time.time() - cmd = f"python ../../tools/preprocess_data.py \ - --input {json_file_path} \ - --output-prefix {output_prefix} \ - --vocab {vocab_file} \ - --dataset-impl mmap \ - --tokenizer-type BertWordPieceLowerCase \ - --split-sentences \ - --workers {num_workers} " - # It's possible to hit MemoryError during above cmd since the memory - # usage is proportional to num_workers. In this case we delete the - # incomplete output and user shall retry with smaller num_workers. - # Our experience show that chunk 6, 7, 9, 17, 18, 20, 21, 24, 27 - # particularly have large memory usage. - if os.system(cmd) == 0: # Success - os.remove(json_file_path) - else: - print(f"Error: chunk {i} preprocessing got error, delete \ - incomplete output. If MemoryError appeared, please retry \ - with num_workers smaller than {num_workers}.") - if os.path.exists(f"{output_prefix}_text_sentence.idx"): - os.remove(f"{output_prefix}_text_sentence.idx") - if os.path.exists(f"{output_prefix}_text_sentence.bin"): - os.remove(f"{output_prefix}_text_sentence.bin") - print(f"Finished preprocessing chunk {i} in {time.time() - start} sec") - -def pile_merge(file_path): - start = time.time() - num_chunks = 30 - vocab_size = 30524 - for i in range(num_chunks): - output_prefix = f"{file_path}pile_bert_train_{i:02}" - assert os.path.exists(f"{output_prefix}_text_sentence.idx") - assert os.path.exists(f"{output_prefix}_text_sentence.bin") - builder = indexed_dataset.make_builder( - f"{file_path}pile_bert_train_text_sentence.bin", impl="mmap", - vocab_size=vocab_size) - for i in range(num_chunks): - chunk_file = f"{file_path}pile_bert_train_{i:02}_text_sentence" - print(f"Merging file {chunk_file}") - builder.merge_file_(chunk_file) - print("Finalizing merged file ...") - builder.finalize(f"{file_path}pile_bert_train_text_sentence.idx") - print(f"Finished merging in {time.time() - start} sec") - # After verifying the merged data with real training, you may want to - # delete the data chunks. - # for i in range(num_chunks): - # output_prefix = f"{file_path}pile_bert_train_{i:02}" - # os.remove(f"{output_prefix}_text_sentence.idx") - # os.remove(f"{output_prefix}_text_sentence.bin") - -if __name__ == '__main__': - # Path to download and store all the output files during the whole process. - # Estimated max storage usage would be around 1.6 TB (or 780GB if skip the - # final merge). Memory usage is proportional to the num_workers below (can - # be as high as O(300GB) if num_workers is around 20). - file_path = "/blob/data/the_pile_bert/" - # The raw Pile data has 30 compressed .zst chunks. To run on single - # machine for all chunks, run "python prepare_pile_data.py range 0 30". - # You can also split and run on multiple machines to speed up, since - # processing one chunk can take hours. The whole process only uses CPU. - if sys.argv[1] == "merge": - # "python prepare_pile_data.py merge" means merge all 30 processed data - # chunks. Run it only after all 30 chunks are preprocessed. The memory - # usage during merge is about 600GB. If you don't have enough memory, - # one solution is to directly use the 30 data chunks as multiple - # datasets. See '--data-path' in - # github.com/microsoft/Megatron-DeepSpeed/blob/main/megatron/arguments.py - pile_merge(file_path) - else: - if sys.argv[1] == "range": - # "python prepare_pile_data.py range 0 30" means process chunk 0-29 - selected_chunk = range(int(sys.argv[2]), int(sys.argv[3])) - else: - # "python prepare_pile_data.py 2 5 8" means process chunk 2, 5, 8 - selected_chunk = [int(x) for x in sys.argv[1:]] - print("selected_chunk: ", selected_chunk) - # Number of process. Adjust based on your CPU/Memory. - num_workers = 20 - # Where the raw Pile data can be downloaded. The url may change in - # future. Contact EleutherAI (https://github.com/EleutherAI/the-pile) - # if this url does not work. - download_url = "https://the-eye.eu/public/AI/pile/train/" - vocab_file = "bert-large-uncased-vocab.txt" - vocab_url = "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt" - if not os.path.exists(vocab_file): - os.system(f"wget {vocab_url}") - os.makedirs(file_path, exist_ok=True) - - for i in selected_chunk: - pile_preprocess(download_url, file_path, vocab_file, num_workers, i) diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_1clmetric_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_1clmetric_TEMPLATE.json deleted file mode 100644 index cca845096..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_1clmetric_TEMPLATE.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false, - "dataloader_drop_last": true, - "data_efficiency": { - "enabled": true, - "seed": DATA_EFFICIENCY_SEED, - "data_routing": { - "enabled": LTD_ENABLED, - "random_ltd":{ - "enabled": LTD_ENABLED, - "total_layer_num": 24, - "random_ltd_layer_num": 22, - "random_ltd_layer_id": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22], - "model_mask_name": "attention_mask", - "model_type": "encoder", - "hidden_state_order": "seq_batch_dim", - "random_ltd_schedule": { - "min_value": LTD_MIN, - "max_value": LTD_MAX, - "schedule_type":"fixed_linear", - "schedule_config": { - "require_steps": LTD_STEP, - "seq_per_step": 16 - } - } - } - }, - "data_sampling": { - "enabled": CL_ENABLED, - "num_workers": DATA_SAMPLING_NUM_WORKERS, - "curriculum_learning": { - "enabled": CL_ENABLED, - "data_cluster_path": "CL_CLUSTER_PATH", - "curriculum_metrics": { - "CL_1st_METRIC_NAME": { - "index_to_sample_path": "CL_1st_SAMPLE_PATH", - "index_to_metric_path": "CL_1st_METRIC_PATH", - "difficulty_type": "CL_1st_DIFF_TYPE", - "clustering_type": "CL_1st_CLUSTER_TYPE", - "min_difficulty": CL_1st_MIN, - "max_difficulty": CL_1st_MAX, - "schedule_type": "fixed_root", - "schedule_config": { - "total_curriculum_step": CL_1st_TOTAL_STEP, - "difficulty_step": CL_1st_DIFF_STEP, - "root_degree": CL_1st_ROOT - } - } - } - } - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_2clmetrics_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_2clmetrics_TEMPLATE.json deleted file mode 100644 index 9461d6d5d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_config_bert_2clmetrics_TEMPLATE.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false, - "dataloader_drop_last": true, - "data_efficiency": { - "enabled": true, - "seed": DATA_EFFICIENCY_SEED, - "data_routing": { - "enabled": LTD_ENABLED, - "random_ltd":{ - "enabled": LTD_ENABLED, - "total_layer_num": 24, - "random_ltd_layer_num": 22, - "random_ltd_layer_id": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22], - "model_mask_name": "attention_mask", - "model_type": "encoder", - "hidden_state_order": "seq_batch_dim", - "random_ltd_schedule": { - "min_value": LTD_MIN, - "max_value": LTD_MAX, - "schedule_type":"fixed_linear", - "schedule_config": { - "require_steps": LTD_STEP, - "seq_per_step": 16 - } - } - } - }, - "data_sampling": { - "enabled": CL_ENABLED, - "num_workers": DATA_SAMPLING_NUM_WORKERS, - "curriculum_learning": { - "enabled": CL_ENABLED, - "data_cluster_path": "CL_CLUSTER_PATH", - "curriculum_metrics": { - "CL_1st_METRIC_NAME": { - "index_to_sample_path": "CL_1st_SAMPLE_PATH", - "index_to_metric_path": "CL_1st_METRIC_PATH", - "difficulty_type": "CL_1st_DIFF_TYPE", - "clustering_type": "CL_1st_CLUSTER_TYPE", - "min_difficulty": CL_1st_MIN, - "max_difficulty": CL_1st_MAX, - "schedule_type": "fixed_root", - "schedule_config": { - "total_curriculum_step": CL_1st_TOTAL_STEP, - "difficulty_step": CL_1st_DIFF_STEP, - "root_degree": CL_1st_ROOT - } - }, - "CL_2nd_METRIC_NAME": { - "index_to_sample_path": "CL_2nd_SAMPLE_PATH", - "index_to_metric_path": "CL_2nd_METRIC_PATH", - "difficulty_type": "CL_2nd_DIFF_TYPE", - "clustering_type": "CL_2nd_CLUSTER_TYPE", - "min_difficulty": CL_2nd_MIN, - "max_difficulty": CL_2nd_MAX, - "schedule_type": "fixed_root", - "schedule_config": { - "total_curriculum_step": CL_2nd_TOTAL_STEP, - "difficulty_step": CL_2nd_DIFF_STEP, - "root_degree": CL_2nd_ROOT - } - } - } - } - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_base_script.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_base_script.sh deleted file mode 100644 index cded15843..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_base_script.sh +++ /dev/null @@ -1,472 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -### The main configs are from Megatron-LM paper -### https://arxiv.org/abs/1909.08053. Choose based on your desired model size -### or build your own configs. -seq_len=512 -global_batch_size=1024 -# lr=1e-4 -lr=$1 -min_lr=1e-5 - -## init_std is the standard deviation for weight initialization. Usually larger -## model needs lower std. Here we roughly follow a heuristic equation of -## sqrt(1/3/hidden_size) from https://arxiv.org/pdf/2201.11990.pdf - -## In addition, we find that the 3.9B model (even after tuning init_std) has -## NaN loss issue from the beginning thus unable to train. This is probably -## because in this example we use the public Pile data, which is a more diverse -## (and potentially more noisy) data than what used in Megatron paper. One -## potential solution is only use the sub datasets in Pile that are also -## used by Megatron paper. - -## BERT 110M (same config as original BERT-Base model) -## This config is not included in Megatron-LM paper -# model_size=0.11 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# init_std=0.02 - -## BERT 336M (same config as original BERT-Large model) -model_size=0.336 -num_layers=24 -hidden_size=1024 -num_attn_heads=16 -init_std=0.02 - -## BERT 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=32 -# init_std=0.013 - -## BERT 3.9B -# model_size=3.9 -# num_layers=48 -# hidden_size=2560 -# num_attn_heads=40 -# init_std=0.011 -############################################################################### -### Training duration configs -## The main termination condition, original Megatron paper trains for 2M iters. -## We changed to token-based termination since data efficiency techniques could -## change token per step. -calc() { awk "BEGIN{ printf \"%.0f\n\", $* }"; } -# train_iters_in_million=2 -train_iters_in_million=$2 -train_tokens=$(calc $train_iters_in_million*1000000*$seq_len*$global_batch_size) -train_tokens_in_billion=$(calc $train_tokens/1000000000) - -## A large enough number of iters, just to make sure we index enough data. The -## only effective termination condition is the train_tokens above. -train_iters=4000000 - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. Original Megatron paper uses 10000 warmup -## iters. We changed lr decay to token based since data efficiency techniques -## could change token per step. -lr_warmup_iters=10000 -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=${train_tokens} -lr_decay_style="linear" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Currently pipeline parallelism is not supported for BERT model: DeepSpeed's -## pipeline parallelism is only integrated with the GPT case, and currently -## DeepSpeed is not integrated with Megatron's own pipeline parallelism. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=0 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -batch_size=$(( ${global_batch_size} / ${dp_size} )) -############################################################################### -### Random layerwise token dropping (random-LTD) configs -## random-LTD's main switch. "false" means disabled. "true" means enabled. -ltd_enabled=${3:-'false'} -## How much dropping ratio to start with. The value denotes the seqlen after -## dropping. -ltd_start=${4:-512} -## How many steps for random-LTD to gradually reduce dropping ratio to zero. -ltd_step_in_million=${5:-1} - -# ltd_enabled="true" -# ltd_start=200 -# ltd_step_in_million=1.8 -ltd_step=$(calc $ltd_step_in_million*1000000) - -## For BERT pretraining, we observe that random-LTD when combined with zero -## dropout can achieve better finetune accuracy on certain tasks. However, this -## is not guaranteed for all models/tasks. It is still recommend to try both -## with and without dropout for random-LTD. -dropout=${6:-0.1} -############################################################################### -### Curriculum learning (CL) configs -## CL's main switch. "false" means disabled. "true" means enabled. -cl_enabled=${7:-'false'} -## Number of CL metrics to use. -cl_num_metric=${8:-1} - -## Name of difficulty metric -cl_1st_metric=${9:-'dummy'} -## Path to the data indexes for this difficulty metric. Samples on ith row of -## index_to_sample have the difficulty value equals to ith row of -## index_to_metric. -cl_1st_index_to_sample_path=${10:-'dummy'} -cl_1st_index_to_metric_path=${11:-'dummy'} -## During training, whether increase difficulty by value- or percentile-based. -cl_1st_difficulty_type=${12:-'value'} -## "single_cluster" means no clustering required and probably CL is achieved by -## data postprocessing. "schedule_based" means will cluster data based on the -## difficulty schedule (pacing function) below. -cl_1st_clustering_type=${13:-'single_cluster'} -## Start difficulty -cl_1st_min=${14:-512} -## End difficulty -cl_1st_max=${15:-512} -## Total step to reach end difficulty -cl_1st_total_step_in_million=${16:-1} -## When changing difficulty, always make sure it's a multiple of the -## difficulty_step below. -cl_1st_difficulty_step=${17:-1} -## Root degree of the schedule (pacing function). -cl_1st_root=${18:-1} - -cl_2nd_metric=${19:-'dummy'} -cl_2nd_index_to_sample_path=${20:-'dummy'} -cl_2nd_index_to_metric_path=${21:-'dummy'} -cl_2nd_difficulty_type=${22:-'value'} -cl_2nd_clustering_type=${23:-'single_cluster'} -cl_2nd_min=${24:-2048} -cl_2nd_max=${25:-2048} -cl_2nd_total_step_in_million=${26:-1} -cl_2nd_difficulty_step=${27:-1} -cl_2nd_root=${28:-1} - -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# ## The *_index_to_sample_percentile_merged is a concatenated index for perf -# ## improvement, but it only works when you set difficulty_type="percentile" in -# ## ds_config. If you use difficulty_type="value", you need to change this to -# ## *_index_to_sample -# # cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="value" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=600 -# cl_1st_max=9069 -# cl_1st_total_step_in_million=0.96 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 - -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=128 -# cl_2nd_max=512 -# cl_2nd_total_step_in_million=0.96 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 - -cl_1st_total_step=$(calc $cl_1st_total_step_in_million*1000000) -cl_2nd_total_step=$(calc $cl_2nd_total_step_in_million*1000000) -############################################################################### -### Misc configs -log_interval=100 -eval_iters=10 -eval_interval=1000 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -save_interval=$((${estimated_train_iter} / ${num_save})) - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -## Number of workers for dataloader. We found that for BERT pre-training, -## num_workers will greatly affect data loading time and overall training -## time. In our experiment with 64 GPUs, the performance reaches peak at -## num_workers = 4 but it may differ depending on hardware. Also note that -## larger num_workers add more CPU computation/memory overhead. -num_workers=4 - -## Public the Pile dataset, see ../pile_data_download_preprocess.py about how -## to download and preprocess the data. Change data_home to where you store the -## pile_bert_train_text_sentence.bin and pile_bert_train_text_sentence.idx. -data_home="/vc_data_blob/users/conglli/the_pile_bert" -if [[ "$host" == *"webxt"* ]]; then - data_home="/blob/data/the_pile_bert" -fi -data_path="${data_home}/pile_bert_train_text_sentence" -## train_idx_path forces Megatron to use a specific data index file generated -## when we analyze data. This is needed because our index for curriculum -## learning difficulty metric is based on this data index. -train_idx_path="${data_home}/pile_bert_train_text_sentence_train_indexmap_exact5ep_509msl_0.10ssp_1234s.npy" - -vocab_path="bert-large-uncased-vocab.txt" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt -fi - -prescale_grad="true" -jobname="bert_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_iters}_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}" -if [ "${ltd_enabled}" = "true" ]; then - jobname="${jobname}_ltd_${ltd_start}_${ltd_step_in_million}M_drop${dropout}" -fi -if [ "${cl_enabled}" = "true" ]; then - jobname="${jobname}_cl_${cl_1st_metric}_${cl_1st_min}_${cl_1st_max}_${cl_1st_total_step_in_million}M_${cl_1st_root}" - if [[ $cl_num_metric -gt 1 ]]; then - jobname="${jobname}_${cl_2nd_metric}_${cl_2nd_min}_${cl_2nd_max}_${cl_2nd_total_step_in_million}M_${cl_2nd_root}" - fi -fi - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_bert" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_bert/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -if [ "${cl_enabled}" = "true" ]; then - data_cluster_path="${output_home}/data_cluster/${jobname}" - mkdir -p ${data_cluster_path} -fi -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.999 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-iters ${lr_warmup_iters} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-iters ${train_iters} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 1e-2 \ - --clip-grad 1.0 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -if [ "${ltd_enabled}" = "true" ]; then -megatron_options="${megatron_options} \ - --attention-dropout ${dropout} \ - --hidden-dropout ${dropout} \ - --random-ltd" -fi - -if [ "${cl_enabled}" = "true" ]; then -megatron_options="${megatron_options} \ - --train-idx-path ${train_idx_path} \ - --data-efficiency-curriculum-learning" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}_seed${seed}" -if [ "${ltd_enabled}" = "true" ]; then - config_json="${config_json}_ltd_${ltd_start}_${ltd_step}" -fi -if [ "${cl_enabled}" = "true" ]; then - config_json="${config_json}_cl_${cl_1st_metric}_${cl_1st_min}_${cl_1st_max}_${cl_1st_total_step}_${cl_1st_root}" - if [[ $cl_num_metric -gt 1 ]]; then - config_json="${config_json}_${cl_2nd_metric}_${cl_2nd_min}_${cl_2nd_max}_${cl_2nd_total_step}_${cl_2nd_root}" - fi -fi -config_json="${config_json}.json" -if [[ $cl_num_metric -gt 1 ]]; then -template_json="ds_config_bert_2clmetrics_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - | sed "s/DATA_EFFICIENCY_SEED/${seed}/" \ - | sed "s/LTD_ENABLED/${ltd_enabled}/" \ - | sed "s/LTD_MIN/${ltd_start}/" \ - | sed "s/LTD_MAX/${seq_len}/" \ - | sed "s/LTD_STEP/${ltd_step}/" \ - | sed "s/CL_ENABLED/${cl_enabled}/" \ - | sed "s/DATA_SAMPLING_NUM_WORKERS/${num_workers}/" \ - | sed "s#CL_CLUSTER_PATH#${data_cluster_path}#" \ - | sed "s#CL_1st_METRIC_NAME#${cl_1st_metric}#" \ - | sed "s#CL_1st_SAMPLE_PATH#${cl_1st_index_to_sample_path}#" \ - | sed "s#CL_1st_METRIC_PATH#${cl_1st_index_to_metric_path}#" \ - | sed "s#CL_1st_DIFF_TYPE#${cl_1st_difficulty_type}#" \ - | sed "s#CL_1st_CLUSTER_TYPE#${cl_1st_clustering_type}#" \ - | sed "s/CL_1st_MIN/${cl_1st_min}/" \ - | sed "s/CL_1st_MAX/${cl_1st_max}/" \ - | sed "s/CL_1st_TOTAL_STEP/${cl_1st_total_step}/" \ - | sed "s/CL_1st_DIFF_STEP/${cl_1st_difficulty_step}/" \ - | sed "s/CL_1st_ROOT/${cl_1st_root}/" \ - | sed "s#CL_2nd_METRIC_NAME#${cl_2nd_metric}#" \ - | sed "s#CL_2nd_SAMPLE_PATH#${cl_2nd_index_to_sample_path}#" \ - | sed "s#CL_2nd_METRIC_PATH#${cl_2nd_index_to_metric_path}#" \ - | sed "s#CL_2nd_DIFF_TYPE#${cl_2nd_difficulty_type}#" \ - | sed "s#CL_2nd_CLUSTER_TYPE#${cl_2nd_clustering_type}#" \ - | sed "s/CL_2nd_MIN/${cl_2nd_min}/" \ - | sed "s/CL_2nd_MAX/${cl_2nd_max}/" \ - | sed "s/CL_2nd_TOTAL_STEP/${cl_2nd_total_step}/" \ - | sed "s/CL_2nd_DIFF_STEP/${cl_2nd_difficulty_step}/" \ - | sed "s/CL_2nd_ROOT/${cl_2nd_root}/" \ - > ${config_json} -else -template_json="ds_config_bert_1clmetric_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - | sed "s/DATA_EFFICIENCY_SEED/${seed}/" \ - | sed "s/LTD_ENABLED/${ltd_enabled}/" \ - | sed "s/LTD_MIN/${ltd_start}/" \ - | sed "s/LTD_MAX/${seq_len}/" \ - | sed "s/LTD_STEP/${ltd_step}/" \ - | sed "s/CL_ENABLED/${cl_enabled}/" \ - | sed "s/DATA_SAMPLING_NUM_WORKERS/${num_workers}/" \ - | sed "s#CL_CLUSTER_PATH#${data_cluster_path}#" \ - | sed "s#CL_1st_METRIC_NAME#${cl_1st_metric}#" \ - | sed "s#CL_1st_SAMPLE_PATH#${cl_1st_index_to_sample_path}#" \ - | sed "s#CL_1st_METRIC_PATH#${cl_1st_index_to_metric_path}#" \ - | sed "s#CL_1st_DIFF_TYPE#${cl_1st_difficulty_type}#" \ - | sed "s#CL_1st_CLUSTER_TYPE#${cl_1st_clustering_type}#" \ - | sed "s/CL_1st_MIN/${cl_1st_min}/" \ - | sed "s/CL_1st_MAX/${cl_1st_max}/" \ - | sed "s/CL_1st_TOTAL_STEP/${cl_1st_total_step}/" \ - | sed "s/CL_1st_DIFF_STEP/${cl_1st_difficulty_step}/" \ - | sed "s/CL_1st_ROOT/${cl_1st_root}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../../../pretrain_bert.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_run.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_run.sh deleted file mode 100644 index c771a0e27..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/bert/pretrain/ds_pretrain_bert_336M_run.sh +++ /dev/null @@ -1,363 +0,0 @@ -############################################################################### -### Each block below is one pretraining setup. Uncomment one block to try. -############################################################################### -### Baseline cases, mostly based on Megatron-LM's BERT-Large hyperparameters, -### but with some changes (different LR schedule). -## Baseline 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} -############################################################################### -## Baseline 703B tokens (67%): -# lr=1.5e-4 -# train_iters_in_million=134e-2 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} -############################################################################### -## Baseline 524B tokens (50%): -# lr=2e-4 -# train_iters_in_million=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} -############################################################################### -### Curriculum learning (CL) + Random layerwise token dropping (random-LTD). -### DeepSpeed Data Efficiency's composed solution. -### BERT pretraining. -## CL+random-LTD 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step_in_million=2 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=96e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=128 -# cl_2nd_max=512 -# cl_2nd_total_step_in_million=96e-2 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step_in_million} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL+random-LTD 524B tokens (50%): -# lr=2e-4 -# train_iters_in_million=1 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=48e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=128 -# cl_2nd_max=512 -# cl_2nd_total_step_in_million=48e-2 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step_in_million} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -### Random layerwise token dropping (random-LTD). -## random-LTD 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step_in_million=2 -# dropout=1e-1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} -############################################################################### -## random-LTD 703B tokens (67%): -# lr=1.5e-4 -# train_iters_in_million=134e-2 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step_in_million=134e-2 -# dropout=1e-1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} -############################################################################### -## random-LTD 524B tokens (50%): -# lr=2e-4 -# train_iters_in_million=1 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step_in_million=1 -# dropout=1e-1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} -############################################################################### -### Curriculum learning (CL). -## CL vocab rarity + seqlen truncation 524B tokens (50%): -# lr=2e-4 -# train_iters_in_million=1 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=48e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=128 -# cl_2nd_max=512 -# cl_2nd_total_step_in_million=48e-2 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step_in_million} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity + seqlen truncation 703B tokens (67%): -# lr=1.5e-4 -# train_iters_in_million=134e-2 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/vc_data/users/conglli/code/data_efficiency/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=64e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=128 -# cl_2nd_max=512 -# cl_2nd_total_step_in_million=64e-2 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step_in_million} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity + seqlen truncation 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=96e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=128 -# cl_2nd_max=512 -# cl_2nd_total_step_in_million=96e-2 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step_in_million} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity + seqlen reorder 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="seqlenvocabrarity" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/seqlen_vocab_rarity/seqlen_vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/seqlen_vocab_rarity/seqlen_vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=96e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### -## CL vocab rarity 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_sample" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=96e-2 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### -## CL seqlen truncation 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="seqlen_truncate" -# cl_1st_index_to_sample_path="dummy" -# cl_1st_index_to_metric_path="dummy" -# cl_1st_difficulty_type="value" -# cl_1st_clustering_type="single_cluster" -# cl_1st_min=128 -# cl_1st_max=512 -# cl_1st_total_step_in_million=96e-2 -# cl_1st_difficulty_step=8 -# cl_1st_root=1 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### -## CL seqlen reorder 1049B tokens (100%): -# lr=1e-4 -# train_iters_in_million=2 -# ltd_enabled="false" -# ltd_start=512 -# ltd_step_in_million=1 -# dropout=1e-1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="seqlen" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/seqlen/seqlen_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_bert_5epoch/seqlen/seqlen_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="single_cluster" -# cl_1st_min=5 -# cl_1st_max=100 -# cl_1st_total_step_in_million=96e-2 -# cl_1st_difficulty_step=8 -# cl_1st_root=2 -# bash ds_pretrain_bert_336M_base_script.sh ${lr} ${train_iters_in_million} \ -# ${ltd_enabled} ${ltd_start} ${ltd_step_in_million} ${dropout} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step_in_million} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_map.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_map.sh deleted file mode 100644 index 3b1caf06f..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_map.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -num_workers=1 # Num nodes to run the map job -num_threads=40 # Num threads on each node. Set this based on #CPU cores - -# If different data epochs have slightly different data samples (e.g., due -# to randomness), then you need to specify large enough num_epochs that cover -# whole pretraining. If different data epochs are the same, set num_epochs to -# 1 to only index 1 epoch, and during pretraining DeepSpeed data efficiency -# library will automatically handle reshuffling when reaching another epoch. -num_epochs=1 - -# Which node is this node (start with 0 and end with num_workers-1). This -# script only launch the map job on 1 worker node, since we don't expect -# running on many nodes and workers don't need any communication. But you -# can modify this script to add a MPI/torch distributed launcher. -worker_id=$1 -save_path="/blob/users/conglli/data/analysis_pile_gpt_${num_epochs}epoch/" - -metric='total_vocab_freq' -# metric='vocab_rarity' # this requires the result of total_vocab_freq - -seq_len=2048 -batch_size=10000 - -jobname="gpt-pile-analyzing-${metric}-${num_epochs}epoch-map-worker${worker_id}" -# Public the Pile dataset, can be downloaded at -# https://mystic.the-eye.eu/public/AI/pile_neox/ -## Change data_home to your own training data path. -# data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_home="/blob/data/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -# Make sure the "--split" is the same as what you will use for pre-training. -options=" \ - --analyzing-task map \ - --analyzing-data-type GPT \ - --analyzing-metric ${metric} \ - --analyzing-num-workers ${num_workers} \ - --analyzing-worker-id ${worker_id} \ - --analyzing-num-threads ${num_threads} \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap \ - --tokenizer-type GPT2BPETokenizer \ - --micro-batch-size ${batch_size} \ - --global-batch-size ${batch_size} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --num-layers 1 \ - --hidden-size 1 \ - --num-attention-heads 1 \ - --split 949,50,1 \ - --distributed-backend gloo \ - --train-data-exact-num-epochs ${num_epochs} \ - --return-data-index \ - --save-interval 1 \ - --save ${save_path}" - -python ../analyze_data.py ${options} &> ${jobname}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_reduce.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_reduce.sh deleted file mode 100644 index a1242ea94..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/ds_analyze_gpt_data_reduce.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Set these 2 to the same as what you used during map job. We need these 2 -# configs to know how many map job result files do we have. -num_workers=1 -num_threads=40 -# Reduce job only has 1 worker but can accelerate by multithreading. -num_threads_reduce=40 - -# If different data epochs have slightly different data samples (e.g., due -# to randomness), then you need to specify large enough num_epochs that cover -# whole pretraining. If different data epochs are the same, set num_epochs to -# 1 to only index 1 epoch, and during pretraining DeepSpeed data efficiency -# library will automatically handle reshuffling when reaching another epoch. -num_epochs=1 - -save_path="/blob/users/conglli/data/analysis_pile_gpt_${num_epochs}epoch/" - -metric='total_vocab_freq' -# metric='vocab_rarity' # this requires the result of total_vocab_freq - -seq_len=2048 -batch_size=10000 - -jobname="gpt-pile-analyzing-${metric}-${num_epochs}epoch-reduce" -# Public the Pile dataset, can be downloaded at -# https://mystic.the-eye.eu/public/AI/pile_neox/ -## Change data_home to your own training data path. -# data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_home="/blob/data/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -# Make sure the "--split" is the same as what you will use for pre-training. -options=" \ - --analyzing-task reduce \ - --analyzing-data-type GPT \ - --analyzing-metric ${metric} \ - --analyzing-num-workers ${num_workers} \ - --analyzing-num-threads ${num_threads} \ - --analyzing-num-threads-reduce ${num_threads_reduce} \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap \ - --tokenizer-type GPT2BPETokenizer \ - --micro-batch-size ${batch_size} \ - --global-batch-size ${batch_size} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --num-layers 1 \ - --hidden-size 1 \ - --num-attention-heads 1 \ - --split 949,50,1 \ - --distributed-backend gloo \ - --train-data-exact-num-epochs ${num_epochs} \ - --return-data-index \ - --save-interval 1 \ - --save ${save_path}" - -python ../analyze_data.py ${options} &> ${jobname}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_config_eval_dummy.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_config_eval_dummy.json deleted file mode 100644 index 72ffd2a7a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_config_eval_dummy.json +++ /dev/null @@ -1,27 +0,0 @@ -{ -"train_batch_size" : 2048, -"train_micro_batch_size_per_gpu": 16, -"steps_per_print": 10, - -"zero_optimization": { - "stage": 0 -}, - -"gradient_clipping": 1.0, -"prescale_gradients": true, - -"fp16": { - "enabled": false, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 -}, - -"bf16": { - "enabled": false -}, - -"wall_clock_breakdown" : false -} \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_1gpu.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_1gpu.sh deleted file mode 100644 index 32ade4917..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_1gpu.sh +++ /dev/null @@ -1,78 +0,0 @@ -## CAUTION: first read Megatron-DeepSpeed/blob/main/examples_deepspeed/MoE/readme_evalharness.md -## and follow the steps of installation/data downloading. - -## Code below only works when you run each evalharness task on a single GPU. -## For multi-GPU evalharness, check Megatron-DeepSpeed/blob/main/examples_deepspeed/MoE/ds_evalharness.sh -checkpoint_path=$1 -config_path=$2 -result_path=$3 -rank=$4 -tasks=$5 -hostname=$6 -master_port=$(( 12345 + ${rank} )) -batch_size=$7 -num_fewshot=$8 - -mp_size=1 -pp_size=1 -no_pp="true" -ep_size=1 - -vocab_file="gpt2-vocab.json" -if [ ! -f "$vocab_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_file="gpt2-merges.txt" -if [ ! -f "$merge_file" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -# export HF_DATASETS_OFFLINE=1 - -dir2=$(dirname "$checkpoint_path") -dirname=$(basename "$dir2")/$(basename "$checkpoint_path") -result_path="${result_path}/${dirname}" -mkdir -p $result_path -result_file="${result_path}/${tasks}_${num_fewshot}shot.json" - -# Dummy arguments to make megatron happy. No need to configure them. -# The reason we don't need to configure them and many other arguments is -# because the eval framework will read the arguments from checkpoint file. -megatron_required_args="\ - --num-layers -1 \ - --hidden-size -1 \ - --num-attention-heads -1 \ - --seq-length -1 \ - --max-position-embeddings -1 -" - -command="../../../../tasks/eval_harness/evaluate.py \ - --load ${checkpoint_path} \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --moe-expert-parallel-size ${ep_size} \ - --vocab-file ${vocab_file} \ - --merge-file ${merge_file} \ - --micro-batch-size ${batch_size} \ - --no-load-optim \ - --no-load-rng \ - --inference \ - --disable-moe-token-dropping \ - --tokenizer-type GPT2BPETokenizer \ - --adaptive_seq_len \ - --eval_fp32 \ - --num_fewshot ${num_fewshot} \ - --task_list ${tasks} \ - --results_path ${result_file} \ - --deepspeed \ - --deepspeed_config ${config_path} \ - ${megatron_required_args} \ - " - -if [[ "${no_pp}" = "true" ]]; then -command="${command} \ - --no-pipeline-parallel" -fi - -launcher="deepspeed --include=$hostname:$rank --master_port=${master_port}" -$launcher $command &> "${result_path}/${tasks}_${num_fewshot}shot.log" \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_gather_result.py b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_gather_result.py deleted file mode 100644 index e0c0c332c..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_gather_result.py +++ /dev/null @@ -1,358 +0,0 @@ -import json -import os -import math -from math import log10, floor -import copy - -def mean(arr): - return sum(arr) / len(arr) - - -def pop_stddev(arr): - mu = mean(arr) - return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr)) - - -def sample_stddev(arr): - mu = mean(arr) - return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1)) - - -def mean_stderr(arr): - return sample_stddev(arr) / math.sqrt(len(arr)) - - -def median(arr): - return arr[len(arr) // 2] - -metric_dict = { - "hellaswag":"acc_norm", - "lambada":"acc", - "triviaqa":"acc", - "webqs":"acc", - "winogrande":"acc", - "piqa":"acc_norm", - "arc_challenge":"acc_norm", - "arc_easy":"acc_norm", - "openbookqa":"acc_norm", - "race":"acc", - "boolq":"acc", - "cb":"acc", - "copa":"acc", - "rte":"acc", - "wic":"acc", - "wsc":"acc", - "multirc":"acc", - "record":"f1", - "anli_r1":"acc", - "anli_r2":"acc", - "anli_r3":"acc", - "wikitext":"word_perplexity", - "logiqa":"acc_norm", - "mathqa":"acc_norm", - "mc_taco":"f1", - "mrpc":"acc", - "prost":"acc_norm", - "pubmedqa":"acc", - "qnli":"acc", - "qqp":"acc", - "sciq":"acc_norm", - "sst":"acc", - "wnli":"acc" -} - -official_dict = { - "hellaswag":["HellaSwag","acc"], - "lambada":["LAMBADA","acc"], - "triviaqa":["TriviaQA","acc"], - "webqs":["WebQs","acc"], - "winogrande":["Winogrande","acc"], - "piqa":["PIQA","acc"], - "arc_challenge":["ARC Challenge","acc"], - "arc_easy":["ARC Easy","acc"], - "openbookqa":["OpenBookQA","acc"], - "race":["RACE-h","acc"], - "boolq":["BoolQ","acc"], - "cb":["CB","acc"], - "copa":["Copa","acc"], - "rte":["RTE","acc"], - "wic":["WiC","acc"], - "wsc":["WSC","acc"], - "multirc":["MultiRC","acc"], - "record":["ReCoRD","f1"], - "anli_r1":["ANLI R1","acc"], - "anli_r2":["ANLI R2","acc"], - "anli_r3":["ANLI R3","acc"], - "wikitext":["WikiText-2","ppl"], - "logiqa":["LogiQA","acc"], - "mathqa":["MathQA","acc"], - "mc_taco":["MC-TACO","f1"], - "mrpc":["MRPC","acc"], - "prost":["PROST","acc"], - "pubmedqa":["PubMedQA","acc"], - "qnli":["QNLI","acc"], - "qqp":["QQP","acc"], - "sciq":["SciQ","acc"], - "sst":["SST-2","acc"], - "wnli":["WNLI","acc"] -} - -# When comparing with gpt3 paper, the most trustful tasks are the hellaswag to -# anli_r3, who have >= 1000 samples (less variation), and have <= 43% data -# contamination in the paper. -gpt3paper_zeroshoteval = { - "hellaswag":[33.7,43.6,51.0,54.7,62.8,67.4,70.9,78.9], - "lambada":[42.7,54.3,60.4,63.6,67.1,70.3,72.5,76.2], - "triviaqa":[4.15,7.61,14.0,19.7,31.3,38.7,41.8,64.3], - "webqs":[1.77,3.20,4.33,4.63,7.92,7.73,8.22,14.4], - "winogrande":[52.0,52.1,57.4,58.7,62.3,64.5,67.9,70.2], - "piqa":[64.6,70.2,72.9,75.1,75.6,78.0,78.5,81.0], - "arc_challenge":[26.6,29.5,31.8,35.5,38.0,41.4,43.7,51.4], - "arc_easy":[43.6,46.5,53.0,53.8,58.2,60.2,63.8,68.8], - "anli_r1":[33.4,34.2,33.4,33.4,34.2,32.3,33.2,34.6], - "anli_r2":[33.2,31.9,33.3,33.3,33.8,33.5,33.5,35.4], - "anli_r3":[33.6,34.0,33.8,33.4,35.3,34.8,34.4,34.5], - "openbookqa":[35.6,43.2,45.2,46.8,53.0,50.4,55.6,57.6], - "race":[35.2,37.9,40.1,40.9,42.4,44.1,44.6,45.5], - "boolq":[49.7,60.3,58.9,62.4,67.1,65.4,66.2,60.5], - "cb":[0.00,32.1,8.93,19.6,19.6,28.6,19.6,46.4], - "copa":[66.0,68.0,73.0,77.0,76.0,80.0,84.0,91.0], - "rte":[47.7,49.8,48.4,56.0,46.6,55.2,62.8,63.5], - "wic":[0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00], - "wsc":[59.6,56.7,65.4,61.5,66.3,60.6,64.4,65.4], - "multirc":[4.72,9.65,12.3,13.6,14.3,18.4,24.2,27.6], - "record":[71.9,79.2,82.8,85.2,87.3,89.5,90.4,91.0] -} - -gpt3paper_fewshoteval = { - "hellaswag":[33.5,43.1,51.3,54.9,62.9,67.3,71.3,79.3], - "lambada":[22.0,40.4,63.2,57.0,78.1,79.1,81.3,86.4], - "triviaqa":[6.96,16.3,26.5,32.1,42.3,51.6,57.5,71.2], - "webqs":[5.46,12.6,15.9,19.6,24.8,27.7,33.5,41.5], - "winogrande":[51.3,52.6,57.5,59.1,62.6,67.4,70.0,77.7], - "piqa":[64.3,69.4,72.0,74.3,75.4,77.8,79.9,82.3], - "arc_challenge":[25.5,28.4,32.3,36.7,39.5,43.7,44.8,51.5], - "arc_easy":[42.7,51.0,58.1,59.1,62.1,65.8,69.1,70.1], - "anli_r1":[32.1,32.5,30.9,32.5,33.5,33.1,33.3,36.8], - "anli_r2":[35.7,33.8,32.1,31.4,32.6,33.3,32.6,34.0], - "anli_r3":[35.0,34.4,35.1,36.0,32.7,33.9,34.5,40.2], - "openbookqa":[37.0,43.6,48.0,50.6,55.6,55.2,60.8,65.4], - "race":[34.3,37.0,40.4,41.4,42.3,44.7,45.1,46.8], - "boolq":[43.1,60.6,62.0,64.1,70.3,70.0,70.2,77.5], - "cb":[42.9,58.9,53.6,69.6,67.9,60.7,66.1,82.1], - "copa":[67.0,64.0,72.0,77.0,83.0,83.0,86.0,92.0], - "rte":[52.3,48.4,46.9,50.9,56.3,49.5,60.6,72.9], - "wic":[49.8,55.0,53.0,53.0,51.6,53.1,51.1,55.3], - "wsc":[58.7,60.6,54.8,49.0,62.5,67.3,75.0,75.0], - "multirc":[6.09,11.8,16.8,20.8,24.7,23.8,25.0,32.5], - "record":[70.7,77.9,82.1,84.0,87.5,88.8,89.8,90.1] -} - -gpt3paper_zeroshoteval_index = { - "125M":0, # Small - "350M":1, # Medium - "760M":2, # Large - "1.3B":3, # XL - "2.7B":4, - "6.7B":5, - "13B":6, - "175B":7 -} - -def round_sig(x, sig=3): - if x == 0: - return 0 - return round(x, sig-int(floor(log10(abs(x))))-1) - -def generate_result_table(tab_header, configs, task_order, caption, avg_range, - avg_tag, avg_only=False, fontsize="\\footnotesize", find_best=False, - candidate_range=None, candidate_task=None, split_name_by_space=False, - print_stderr=False, few_shot=False): - # Gather results - result_list = [] - for i in range(len(configs)): - result_dict = {} - eval_path = configs[i][-1] - if "paper" in configs[i][0]: - assert eval_path is None - if eval_path is None: - assert "paper" in configs[i][0] - assert configs[i][1] in gpt3paper_zeroshoteval_index, "the second element has to be the model size" - paper_result_idx = gpt3paper_zeroshoteval_index[configs[i][1]] - if few_shot: - for task in gpt3paper_fewshoteval: - result_dict[task] = [gpt3paper_fewshoteval[task][paper_result_idx]] - else: - for task in gpt3paper_zeroshoteval: - result_dict[task] = [gpt3paper_zeroshoteval[task][paper_result_idx]] - else: - for file in os.listdir(eval_path): - if file.endswith(".json"): - result = json.load(open(eval_path+"/"+file, "r")) - for task in result['results']: - if task != "wikitext": - result_dict[task] = [100.0*result['results'][task][metric_dict[task]]] - else: - result_dict[task] = [result['results'][task][metric_dict[task]]] - result_list.append(result_dict) - avg_list = [] - for i in range(len(configs)): - average_results = [] - for j in range(len(avg_range)): - results = [] - for k in range(avg_range[j]+1): - if task_order[k] in result_list[i]: - results.append(result_list[i][task_order[k]][0]) - if len(results) > 0: - average_results.append(float(sum(results))/len(results)) - else: - average_results.append(0) - avg_list.append(average_results) - - if find_best: - best_avg_value = [0 for _ in range(len(avg_range))] - best_avg_idx = [0 for _ in range(len(avg_range))] - best_task_value = [0 for _ in range(len(candidate_task))] - best_task_idx = [0 for _ in range(len(candidate_task))] - for i in range(candidate_range, len(configs)): - for j in range(len(avg_range)): - if avg_list[i][j] > best_avg_value[j]: - best_avg_value[j] = avg_list[i][j] - best_avg_idx[j] = i - for j in range(len(candidate_task)): - if result_list[i][candidate_task[j]] > best_task_value[j]: - best_task_value[j] = result_list[i][candidate_task[j]] - best_task_idx[j] = i - # reorder configs, result_list, avg_list to only keep the best cases - new_configs = configs[:candidate_range] - new_result_list = result_list[:candidate_range] - new_avg_list = avg_list[:candidate_range] - for i in range(len(avg_range)): - selected_config = copy.deepcopy(configs[best_avg_idx[i]]) - selected_config[0] = "({})Best Avg{}".format(len(new_configs), - avg_tag[i]) - new_configs.append(selected_config) - new_result_list.append(result_list[best_avg_idx[i]]) - new_avg_list.append(avg_list[best_avg_idx[i]]) - - for i in range(len(candidate_task)): - selected_config = copy.deepcopy(configs[best_task_idx[i]]) - selected_config[0] = "({})Best {}".format(len(new_configs), - official_dict[candidate_task[i]][0]) - new_configs.append(selected_config) - new_result_list.append(result_list[best_task_idx[i]]) - new_avg_list.append(avg_list[best_task_idx[i]]) - configs = new_configs - result_list = new_result_list - avg_list = new_avg_list - - # split the case names by space - if split_name_by_space: - max_num_row = 1 - splitted_names = [] - for i in range(len(configs)): - new_name = configs[i][0].split() - max_num_row = max(max_num_row, len(new_name)) - splitted_names.append(new_name) - tab_header = ["" for _ in range(max_num_row-1)] + tab_header - for i in range(len(configs)): - padding = ["" for _ in range(max_num_row-len(splitted_names[i]))] - configs[i] = padding + splitted_names[i] + configs[i][1:] - - # generate the table - print("\\begin{table}") - print("\centering") - print(fontsize) - print("\caption{"+caption+"}") - text = "\\begin{tabular}{@{}l|" - for _ in range(len(configs)): - text += "c" - text += "@{}}" - print(text) - print("\\toprule") - for i in range(len(tab_header)): - text = "{} &".format(tab_header[i]) - for j in range(len(configs)): - if j != len(configs) - 1: - text += (configs[j][i] + "& ") - else: - text += (configs[j][i] + "\\\\") - print(text) - print("\midrule") - for i in range(len(avg_range)): - text = ("Avg. " + avg_tag[i]) - arr = [] - for j in range(len(configs)): - arr.append(avg_list[j][i]) - text += " & {}".format(round_sig(avg_list[j][i])) - text += "\\\\" - if print_stderr: - arr_mean = mean(arr) - arr_std = sample_stddev(arr) - text += " % mean {:.3f}, std {:.3f}, mean+1std {:.3f}, mean+2std {:.3f}, mean+3std {:.3f}".format( - arr_mean, arr_std, arr_mean+arr_std, arr_mean+arr_std*2, arr_mean+arr_std*3) - print(text) - if not avg_only: - print("\midrule") - for i in range(len(task_order)): - task = task_order[i] - text = "({}) {}".format(i, official_dict[task][0]) - arr = [] - for j in range(len(configs)): - result_dict = result_list[j] - if task in result_dict: - text += " & {}".format(round_sig(result_dict[task][0])) - arr.append(result_dict[task][0]) - else: - text += " & N/A" - text += "\\\\" - if print_stderr: - arr_mean = mean(arr) - arr_std = sample_stddev(arr) - if task != "wikitext": - text += " % mean {:.3f}, std {:.3f}, mean+1std {:.3f}, mean+2std {:.3f}, mean+3std {:.3f}".format( - arr_mean, arr_std, arr_mean+arr_std, arr_mean+arr_std*2, arr_mean+arr_std*3) - else: - text += " % mean {:.3f}, std {:.3f}, mean-1std {:.3f}, mean-2std {:.3f}, mean-3std {:.3f}".format( - arr_mean, arr_std, arr_mean-arr_std, arr_mean-arr_std*2, arr_mean-arr_std*3) - print(text) - print("\\bottomrule") - print("\end{tabular}") - print("\end{table}") - print("") - print("") - -if __name__ == '__main__': - task_order = ["hellaswag","lambada","triviaqa","webqs","winogrande","piqa", - "arc_challenge","arc_easy","anli_r1","anli_r2","anli_r3","openbookqa", - "race","boolq","copa","rte","wsc","multirc","record","wikitext"] - avg_range = [18] - avg_tag = ["0-18"] - tab_header = ["Case","Model size","Train tokens","Batch size","Bsz warmup","LR","min LR","LR warmup","LR decay","decay style"] - - configs = [ - ["(0)paper","125M","300B","256","4B","6e-4","6e-5","375M","260B","cosine", None], # gpt3 paper orig results, thus result path is None - ["(1)repro","125M","300B","256","4B","6e-4","6e-5","375M","260B","cosine", - '/blob/users/conglli/project/data_efficiency_gpt/eval_results/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup375M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234-bwup4B/global_step591581/'], - ["(2)fixedBsz","125M","300B","256","N/A","6e-4","6e-5","3000M","260B","cosine", - '/blob/users/conglli/project/data_efficiency_gpt/eval_results/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup3000M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234/global_step572205/'], - ["(3)fixedBsz 300B+minLR","125M","300B","256","N/A","6e-4","1e-6","3000M","300B","cosine", - '/blob/users/conglli/project/data_efficiency_gpt/eval_results/gpt-pile-0.125B-tok300B-lr6.0e-4-min1.0e-6-wup3000M-dcy300B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234/global_step572205/'] - ] - caption = 'Conglong: GPT-3 125M results zero-shot' - generate_result_table(tab_header, configs, task_order, caption, avg_range, - avg_tag, split_name_by_space=True, fontsize="\\tiny") - - configs = [ - ["(0)paper","125M","300B","256","4B","6e-4","6e-5","375M","260B","cosine", None], # gpt3 paper orig results, thus result path is None - ["(1)repro","125M","300B","256","4B","6e-4","6e-5","375M","260B","cosine", - '/blob/users/conglli/project/data_efficiency_gpt/eval_results_fewshot/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup375M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234-bwup4B/global_step591581/'], - ["(2)fixedBsz","125M","300B","256","N/A","6e-4","6e-5","3000M","260B","cosine", - '/blob/users/conglli/project/data_efficiency_gpt/eval_results_fewshot/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup3000M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234/global_step572205/'], - ["(3)fixedBsz 300B+minLR","125M","300B","256","N/A","6e-4","1e-6","3000M","300B","cosine", - '/blob/users/conglli/project/data_efficiency_gpt/eval_results_fewshot/gpt-pile-0.125B-tok300B-lr6.0e-4-min1.0e-6-wup3000M-dcy300B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234/global_step572205/'], - ] - caption = 'Conglong: GPT-3 125M results few-shot' - generate_result_table(tab_header, configs, task_order, caption, avg_range, - avg_tag, split_name_by_space=True, fontsize="\\tiny", few_shot=True) - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run.sh deleted file mode 100644 index 2bfbec3a1..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run.sh +++ /dev/null @@ -1,67 +0,0 @@ -## CAUTION: first read Megatron-DeepSpeed/blob/main/examples_deepspeed/MoE/readme_evalharness.md -## and follow the steps of installation/data downloading. -checkpoint_paths=( - /vc_data_blob/users/conglli/project/data_efficient_gpt/checkpoint/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup375M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234-bwup4B/global_step591581/ - /vc_data_blob/users/conglli/project/data_efficient_gpt/checkpoint/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup3000M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234/global_step572205/ -) - -## No need to use the exact training config json, just use this dummy is fine -config_path=ds_config_eval_dummy.json -username=$(whoami) -result_path="/blob/users/${username}/project/data_efficient_gpt/eval_results" - -## Task(s) on the same row will be performed together in the same process. -## There exist other tasks that can run but we skip because they didn't appear -## or have strange scores in GPT-3 paper: qqp, prost, cb, wic, mrpc, sst, wnli -## pubmedqa, logiqa, qnli, sciq, mc_taco, mathqa. For wikitext, it didn't -## appear in paper but we include it for a perplexity task. -tasks=( - record - triviaqa - hellaswag - arc_challenge - arc_easy - race - multirc - openbookqa - lambada - webqs - winogrande - piqa - anli_r1,anli_r2,anli_r3 - boolq,copa - rte,wsc - wikitext -) - -## Use localhost if you didn't setup hostfile as described in -## https://www.deepspeed.ai/getting-started/#resource-configuration-multi-node. -## If hostfile exist, use hostname (e.g., worker-0) in hostfile. -# hostname="localhost" -hostname="worker-0" - -batch_size=32 - -## This script is for zero-shot -num_fewshot=0 - -num_gpus=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -cuda_id=-1 -total_mem=$(nvidia-smi --query-gpu=memory.total --format=csv -i 0 | grep -Eo [0-9]+) -total_mem=$(( ${total_mem}*99/100 )) # somehow there could exist tiny (4MB or so) gpu memory leak - -## Code below only works when you run each evalharness task on a single GPU. -## For multi-GPU evalharness, check Megatron-DeepSpeed/blob/main/examples_deepspeed/MoE/ds_evalharness.sh -for l in "${!checkpoint_paths[@]}"; do - checkpoint_path=${checkpoint_paths[l]} - for ((i=0;i<${#tasks[@]};++i)); do - task=${tasks[i]} - free_mem=0 - while [ $free_mem -lt $total_mem ]; do - cuda_id=$(((cuda_id+1)%num_gpus)) - free_mem=$(nvidia-smi --query-gpu=memory.free --format=csv -i $cuda_id | grep -Eo [0-9]+) - sleep 60s - done - bash ds_evalharness_1gpu.sh $checkpoint_path $config_path $result_path $cuda_id $task $hostname $batch_size $num_fewshot & - done -done diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run_10shot.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run_10shot.sh deleted file mode 100644 index 8e6406477..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/eval/ds_evalharness_parallel_run_10shot.sh +++ /dev/null @@ -1,62 +0,0 @@ -## CAUTION: first read Megatron-DeepSpeed/blob/main/examples_deepspeed/MoE/readme_evalharness.md -## and follow the steps of installation/data downloading. -checkpoint_paths=( - /vc_data_blob/users/conglli/project/data_efficient_gpt/checkpoint/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup375M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234-bwup4B/global_step591581/ - /vc_data_blob/users/conglli/project/data_efficient_gpt/checkpoint/gpt-pile-0.125B-tok300B-lr6.0e-4-min6.0e-5-wup3000M-dcy260B-sty-cosine-gbs256-mbs4-gpu64-zero0-mp1-pp1-nopp-seed1234/global_step572205/ -) - -## No need to use the exact training config json, just use this dummy is fine -config_path=ds_config_eval_dummy.json -username=$(whoami) -result_path="/blob/users/${username}/project/data_efficient_gpt/eval_results_10shot" - -## Task(s) on the same row will be performed together in the same process. -tasks=( - record - triviaqa - hellaswag - arc_challenge - arc_easy - race - multirc - openbookqa - lambada - webqs - winogrande - piqa - anli_r1,anli_r2 - anli_r3 - boolq,copa - rte,wsc -) - -num_fewshot=10 - -## Use localhost if you didn't setup hostfile as described in -## https://www.deepspeed.ai/getting-started/#resource-configuration-multi-node. -## If hostfile exist, use hostname (e.g., worker-0) in hostfile. -# hostname="localhost" -hostname="worker-0" - -batch_size=16 - -num_gpus=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -cuda_id=-1 -total_mem=$(nvidia-smi --query-gpu=memory.total --format=csv -i 0 | grep -Eo [0-9]+) -total_mem=$(( ${total_mem}*99/100 )) # somehow there could exist tiny (4MB or so) gpu memory leak - -## Code below only works when you run each evalharness task on a single GPU. -## For multi-GPU evalharness, check Megatron-DeepSpeed/blob/main/examples_deepspeed/MoE/ds_evalharness.sh -for l in "${!checkpoint_paths[@]}"; do - checkpoint_path=${checkpoint_paths[l]} - for ((i=0;i<${#tasks[@]};++i)); do - task=${tasks[i]} - free_mem=0 - while [ $free_mem -lt $total_mem ]; do - cuda_id=$(((cuda_id+1)%num_gpus)) - free_mem=$(nvidia-smi --query-gpu=memory.free --format=csv -i $cuda_id | grep -Eo [0-9]+) - sleep 60s - done - bash ds_evalharness_1gpu.sh $checkpoint_path $config_path $result_path $cuda_id $task $hostname $batch_size $num_fewshot & - done -done diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_1clmetric_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_1clmetric_TEMPLATE.json deleted file mode 100644 index c542c7cf3..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_1clmetric_TEMPLATE.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false, - "dataloader_drop_last": true, - "data_efficiency": { - "enabled": true, - "seed": DATA_EFFICIENCY_SEED, - "data_routing": { - "enabled": LTD_ENABLED, - "random_ltd":{ - "enabled": LTD_ENABLED, - "total_layer_num": 24, - "random_ltd_layer_num": 22, - "random_ltd_layer_id": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22], - "model_mask_name": "attention_mask", - "model_type": "decoder", - "hidden_state_order": "seq_batch_dim", - "random_ltd_schedule": { - "min_value": LTD_MIN, - "max_value": LTD_MAX, - "schedule_type":"fixed_linear", - "schedule_config": { - "require_steps": LTD_STEP, - "seq_per_step": 16 - } - } - } - }, - "data_sampling": { - "enabled": CL_ENABLED, - "num_workers": DATA_SAMPLING_NUM_WORKERS, - "curriculum_learning": { - "enabled": CL_ENABLED, - "data_cluster_path": "CL_CLUSTER_PATH", - "curriculum_metrics": { - "CL_1st_METRIC_NAME": { - "index_to_sample_path": "CL_1st_SAMPLE_PATH", - "index_to_metric_path": "CL_1st_METRIC_PATH", - "difficulty_type": "CL_1st_DIFF_TYPE", - "clustering_type": "CL_1st_CLUSTER_TYPE", - "min_difficulty": CL_1st_MIN, - "max_difficulty": CL_1st_MAX, - "schedule_type": "fixed_root", - "schedule_config": { - "total_curriculum_step": CL_1st_TOTAL_STEP, - "difficulty_step": CL_1st_DIFF_STEP, - "root_degree": CL_1st_ROOT - } - } - } - } - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_2clmetrics_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_2clmetrics_TEMPLATE.json deleted file mode 100644 index a556aa7af..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_config_gpt_2clmetrics_TEMPLATE.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false, - "dataloader_drop_last": true, - "data_efficiency": { - "enabled": true, - "seed": DATA_EFFICIENCY_SEED, - "data_routing": { - "enabled": LTD_ENABLED, - "random_ltd":{ - "enabled": LTD_ENABLED, - "total_layer_num": 24, - "random_ltd_layer_num": 22, - "random_ltd_layer_id": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22], - "model_mask_name": "attention_mask", - "model_type": "decoder", - "hidden_state_order": "seq_batch_dim", - "random_ltd_schedule": { - "min_value": LTD_MIN, - "max_value": LTD_MAX, - "schedule_type":"fixed_linear", - "schedule_config": { - "require_steps": LTD_STEP, - "seq_per_step": 16 - } - } - } - }, - "data_sampling": { - "enabled": CL_ENABLED, - "num_workers": DATA_SAMPLING_NUM_WORKERS, - "curriculum_learning": { - "enabled": CL_ENABLED, - "data_cluster_path": "CL_CLUSTER_PATH", - "curriculum_metrics": { - "CL_1st_METRIC_NAME": { - "index_to_sample_path": "CL_1st_SAMPLE_PATH", - "index_to_metric_path": "CL_1st_METRIC_PATH", - "difficulty_type": "CL_1st_DIFF_TYPE", - "clustering_type": "CL_1st_CLUSTER_TYPE", - "min_difficulty": CL_1st_MIN, - "max_difficulty": CL_1st_MAX, - "schedule_type": "fixed_root", - "schedule_config": { - "total_curriculum_step": CL_1st_TOTAL_STEP, - "difficulty_step": CL_1st_DIFF_STEP, - "root_degree": CL_1st_ROOT - } - }, - "CL_2nd_METRIC_NAME": { - "index_to_sample_path": "CL_2nd_SAMPLE_PATH", - "index_to_metric_path": "CL_2nd_METRIC_PATH", - "difficulty_type": "CL_2nd_DIFF_TYPE", - "clustering_type": "CL_2nd_CLUSTER_TYPE", - "min_difficulty": CL_2nd_MIN, - "max_difficulty": CL_2nd_MAX, - "schedule_type": "fixed_root", - "schedule_config": { - "total_curriculum_step": CL_2nd_TOTAL_STEP, - "difficulty_step": CL_2nd_DIFF_STEP, - "root_degree": CL_2nd_ROOT - } - } - } - } - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_base_script.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_base_script.sh deleted file mode 100644 index fe2144c6d..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_base_script.sh +++ /dev/null @@ -1,515 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=512 -# lr=2.0e-4 -lr=$1 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -# train_tokens_in_billion=300 -train_tokens_in_billion=$2 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -batch_size=$(( ${global_batch_size} / ${dp_size} )) -############################################################################### -### Random layerwise token dropping (random-LTD) configs -## random-LTD's main switch. "false" means disabled. "true" means enabled. -ltd_enabled=${3:-'false'} -## How much dropping ratio to start with. The value denotes the seqlen after -## dropping. -ltd_start=${4:-2048} -## How many steps for random-LTD to gradually reduce dropping ratio to zero. -ltd_step=${5:-1} - -# ltd_enabled="true" -# ltd_start=128 -# ltd_step=200000 -############################################################################### -### Curriculum learning (CL) configs -## CL's main switch. "false" means disabled. "true" means enabled. -cl_enabled=${6:-'false'} -## Number of CL metrics to use. -cl_num_metric=${7:-1} - -## Name of difficulty metric -cl_1st_metric=${8:-'dummy'} -## Path to the data indexes for this difficulty metric. Samples on ith row of -## index_to_sample have the difficulty value equals to ith row of -## index_to_metric. -cl_1st_index_to_sample_path=${9:-'dummy'} -cl_1st_index_to_metric_path=${10:-'dummy'} -## During training, whether increase difficulty by value- or percentile-based. -cl_1st_difficulty_type=${11:-'value'} -## "single_cluster" means no clustering required and probably CL is achieved by -## data postprocessing. "schedule_based" means will cluster data based on the -## difficulty schedule (pacing function) below. -cl_1st_clustering_type=${12:-'single_cluster'} -## Start difficulty -cl_1st_min=${13:-2048} -## End difficulty -cl_1st_max=${14:-2048} -## Total step to reach end difficulty -cl_1st_total_step=${15:-1} -## When changing difficulty, always make sure it's a multiple of the -## difficulty_step below. -cl_1st_difficulty_step=${16:-1} -## Root degree of the schedule (pacing function). -cl_1st_root=${17:-1} - -cl_2nd_metric=${18:-'dummy'} -cl_2nd_index_to_sample_path=${19:-'dummy'} -cl_2nd_index_to_metric_path=${20:-'dummy'} -cl_2nd_difficulty_type=${21:-'value'} -cl_2nd_clustering_type=${22:-'single_cluster'} -cl_2nd_min=${23:-2048} -cl_2nd_max=${24:-2048} -cl_2nd_total_step=${25:-1} -cl_2nd_difficulty_step=${26:-1} -cl_2nd_root=${27:-1} - -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# ## The *_index_to_sample_percentile_merged is a concatenated index for perf -# ## improvement, but it only works when you set difficulty_type="percentile" in -# ## ds_config. If you use difficulty_type="value", you need to change this to -# ## *_index_to_sample -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# # cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 - -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=110000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -############################################################################### -### Misc configs -log_interval=100 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -save_interval=$((${estimated_train_iter} / ${num_save})) - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -if [[ "$host" == *"webxt"* ]]; then - data_home="/blob/data/the_pile_public_merged_nopreprocessing" -fi -data_path="${data_home}/pile_text_document" -## *_idx_path force Megatron to use a specific data index file generated when -## we analyze data. This is needed because our index for curriculum learning -## difficulty metric is based on this data index. -doc_idx_path="${data_home}/pile_text_document_train_indexmap_exact1ep_2048sl_1234s_doc_idx.npy" -sample_idx_path="${data_home}/pile_text_document_train_indexmap_exact1ep_2048sl_1234s_sample_idx.npy" -shuffle_idx_path="${data_home}/pile_text_document_train_indexmap_exact1ep_2048sl_1234s_shuffle_idx.npy" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}" -if [ "${ltd_enabled}" = "true" ]; then - jobname="${jobname}_ltd_${ltd_start}_${ltd_step}" -fi -if [ "${cl_enabled}" = "true" ]; then - jobname="${jobname}_cl_${cl_1st_metric}_${cl_1st_min}_${cl_1st_max}_${cl_1st_total_step}_${cl_1st_root}" - if [[ $cl_num_metric -gt 1 ]]; then - jobname="${jobname}_${cl_2nd_metric}_${cl_2nd_min}_${cl_2nd_max}_${cl_2nd_total_step}_${cl_2nd_root}" - fi -fi - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -if [ "${cl_enabled}" = "true" ]; then - data_cluster_path="${output_home}/data_cluster/${jobname}" - mkdir -p ${data_cluster_path} -fi -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -if [ "${ltd_enabled}" = "true" ]; then -megatron_options="${megatron_options} \ - --random-ltd" -fi - -if [ "${cl_enabled}" = "true" ]; then -megatron_options="${megatron_options} \ - --train-doc-idx-path ${doc_idx_path} \ - --train-sample-idx-path ${sample_idx_path} \ - --train-shuffle-idx-path ${shuffle_idx_path} \ - --data-efficiency-curriculum-learning" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}_seed${seed}" -if [ "${ltd_enabled}" = "true" ]; then - config_json="${config_json}_ltd_${ltd_start}_${ltd_step}" -fi -if [ "${cl_enabled}" = "true" ]; then - config_json="${config_json}_cl_${cl_1st_metric}_${cl_1st_min}_${cl_1st_max}_${cl_1st_total_step}_${cl_1st_root}" - if [[ $cl_num_metric -gt 1 ]]; then - config_json="${config_json}_${cl_2nd_metric}_${cl_2nd_min}_${cl_2nd_max}_${cl_2nd_total_step}_${cl_2nd_root}" - fi -fi -config_json="${config_json}.json" -if [[ $cl_num_metric -gt 1 ]]; then -template_json="ds_config_gpt_2clmetrics_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - | sed "s/DATA_EFFICIENCY_SEED/${seed}/" \ - | sed "s/LTD_ENABLED/${ltd_enabled}/" \ - | sed "s/LTD_MIN/${ltd_start}/" \ - | sed "s/LTD_MAX/${seq_len}/" \ - | sed "s/LTD_STEP/${ltd_step}/" \ - | sed "s/CL_ENABLED/${cl_enabled}/" \ - | sed "s/DATA_SAMPLING_NUM_WORKERS/${num_workers}/" \ - | sed "s#CL_CLUSTER_PATH#${data_cluster_path}#" \ - | sed "s#CL_1st_METRIC_NAME#${cl_1st_metric}#" \ - | sed "s#CL_1st_SAMPLE_PATH#${cl_1st_index_to_sample_path}#" \ - | sed "s#CL_1st_METRIC_PATH#${cl_1st_index_to_metric_path}#" \ - | sed "s#CL_1st_DIFF_TYPE#${cl_1st_difficulty_type}#" \ - | sed "s#CL_1st_CLUSTER_TYPE#${cl_1st_clustering_type}#" \ - | sed "s/CL_1st_MIN/${cl_1st_min}/" \ - | sed "s/CL_1st_MAX/${cl_1st_max}/" \ - | sed "s/CL_1st_TOTAL_STEP/${cl_1st_total_step}/" \ - | sed "s/CL_1st_DIFF_STEP/${cl_1st_difficulty_step}/" \ - | sed "s/CL_1st_ROOT/${cl_1st_root}/" \ - | sed "s#CL_2nd_METRIC_NAME#${cl_2nd_metric}#" \ - | sed "s#CL_2nd_SAMPLE_PATH#${cl_2nd_index_to_sample_path}#" \ - | sed "s#CL_2nd_METRIC_PATH#${cl_2nd_index_to_metric_path}#" \ - | sed "s#CL_2nd_DIFF_TYPE#${cl_2nd_difficulty_type}#" \ - | sed "s#CL_2nd_CLUSTER_TYPE#${cl_2nd_clustering_type}#" \ - | sed "s/CL_2nd_MIN/${cl_2nd_min}/" \ - | sed "s/CL_2nd_MAX/${cl_2nd_max}/" \ - | sed "s/CL_2nd_TOTAL_STEP/${cl_2nd_total_step}/" \ - | sed "s/CL_2nd_DIFF_STEP/${cl_2nd_difficulty_step}/" \ - | sed "s/CL_2nd_ROOT/${cl_2nd_root}/" \ - > ${config_json} -else -template_json="ds_config_gpt_1clmetric_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - | sed "s/DATA_EFFICIENCY_SEED/${seed}/" \ - | sed "s/LTD_ENABLED/${ltd_enabled}/" \ - | sed "s/LTD_MIN/${ltd_start}/" \ - | sed "s/LTD_MAX/${seq_len}/" \ - | sed "s/LTD_STEP/${ltd_step}/" \ - | sed "s/CL_ENABLED/${cl_enabled}/" \ - | sed "s/DATA_SAMPLING_NUM_WORKERS/${num_workers}/" \ - | sed "s#CL_CLUSTER_PATH#${data_cluster_path}#" \ - | sed "s#CL_1st_METRIC_NAME#${cl_1st_metric}#" \ - | sed "s#CL_1st_SAMPLE_PATH#${cl_1st_index_to_sample_path}#" \ - | sed "s#CL_1st_METRIC_PATH#${cl_1st_index_to_metric_path}#" \ - | sed "s#CL_1st_DIFF_TYPE#${cl_1st_difficulty_type}#" \ - | sed "s#CL_1st_CLUSTER_TYPE#${cl_1st_clustering_type}#" \ - | sed "s/CL_1st_MIN/${cl_1st_min}/" \ - | sed "s/CL_1st_MAX/${cl_1st_max}/" \ - | sed "s/CL_1st_TOTAL_STEP/${cl_1st_total_step}/" \ - | sed "s/CL_1st_DIFF_STEP/${cl_1st_difficulty_step}/" \ - | sed "s/CL_1st_ROOT/${cl_1st_root}/" \ - > ${config_json} -fi - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_run.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_run.sh deleted file mode 100644 index 8878c1792..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/data_efficiency/gpt/pretrain/ds_pretrain_gpt_1.3B_dense_run.sh +++ /dev/null @@ -1,366 +0,0 @@ -############################################################################### -### Each block below is one pretraining setup. Uncomment one block to try. -############################################################################### -### Baseline cases, mostly based on OpenAI's GPT-3 hyperparameters, but with -### some changes (without batch size warmup, and different LR schedule). -## Baseline 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} -############################################################################### -## Baseline 200B tokens (67%): -# lr=3.0e-4 # scaled based on train token reduction ratio -# train_tokens_in_billion=200 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} -############################################################################### -## Baseline 150B tokens (50%): -# lr=4.0e-4 -# train_tokens_in_billion=150 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} -############################################################################### -### Curriculum learning (CL) + Random layerwise token dropping (random-LTD). -### DeepSpeed Data Efficiency's best composed solution. -## CL+random-LTD 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step=200000 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=110000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL+random-LTD 150B tokens (50%): -# lr=4.0e-4 -# train_tokens_in_billion=150 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step=100000 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=55000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=55000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -### Random layerwise token dropping (random-LTD). -## random-LTD 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step=200000 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} -############################################################################### -## random-LTD 200B tokens (67%): -# lr=3.0e-4 -# train_tokens_in_billion=200 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step=133333 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} -############################################################################### -## random-LTD 150B tokens (50%): -# lr=4.0e-4 -# train_tokens_in_billion=150 -# ltd_enabled="true" -# ltd_start=128 -# ltd_step=100000 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} -############################################################################### -### Curriculum learning (CL). -## CL vocab rarity + seqlen truncation 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=110000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity + seqlen truncation 200B tokens (67%): -# lr=3.0e-4 -# train_tokens_in_billion=200 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=73000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=73000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity + seqlen truncation 150B tokens (50%): -# lr=4.0e-4 -# train_tokens_in_billion=150 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=55000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_truncate" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=55000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity + seqlen reshape 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=2 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# cl_2nd_metric="seqlen_reshape" -# cl_2nd_index_to_sample_path="dummy" -# cl_2nd_index_to_metric_path="dummy" -# cl_2nd_difficulty_type="value" -# cl_2nd_clustering_type="single_cluster" -# cl_2nd_min=80 -# cl_2nd_max=2048 -# cl_2nd_total_step=110000 -# cl_2nd_difficulty_step=8 -# cl_2nd_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} ${cl_2nd_metric} ${cl_2nd_index_to_sample_path} \ -# ${cl_2nd_index_to_metric_path} ${cl_2nd_difficulty_type} \ -# ${cl_2nd_clustering_type} ${cl_2nd_min} ${cl_2nd_max} \ -# ${cl_2nd_total_step} ${cl_2nd_difficulty_step} ${cl_2nd_root} -############################################################################### -## CL vocab rarity 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="voc" -# cl_1st_index_to_sample_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_sample_percentile_merged" -# cl_1st_index_to_metric_path="/blob/users/conglli/data/analysis_pile_gpt_1epoch/vocab_rarity/vocab_rarity_index_to_metric" -# cl_1st_difficulty_type="percentile" -# cl_1st_clustering_type="schedule_based" -# cl_1st_min=1 -# cl_1st_max=100 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=1 -# cl_1st_root=2 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### -## CL seqlen truncation 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="seqlen_truncate" -# cl_1st_index_to_sample_path="dummy" -# cl_1st_index_to_metric_path="dummy" -# cl_1st_difficulty_type="value" -# cl_1st_clustering_type="single_cluster" -# cl_1st_min=80 -# cl_1st_max=2048 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=8 -# cl_1st_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### -## CL seqlen reshape 300B tokens (100%): -# lr=2.0e-4 -# train_tokens_in_billion=300 -# ltd_enabled="false" -# ltd_start=2048 -# ltd_step=1 -# cl_enabled="true" -# cl_num_metric=1 -# cl_1st_metric="seqlen_reshape" -# cl_1st_index_to_sample_path="dummy" -# cl_1st_index_to_metric_path="dummy" -# cl_1st_difficulty_type="value" -# cl_1st_clustering_type="single_cluster" -# cl_1st_min=80 -# cl_1st_max=2048 -# cl_1st_total_step=110000 -# cl_1st_difficulty_step=8 -# cl_1st_root=1 -# bash ds_pretrain_gpt_1.3B_dense_base_script.sh ${lr} \ -# ${train_tokens_in_billion} ${ltd_enabled} ${ltd_start} ${ltd_step} \ -# ${cl_enabled} ${cl_num_metric} ${cl_1st_metric} \ -# ${cl_1st_index_to_sample_path} ${cl_1st_index_to_metric_path} \ -# ${cl_1st_difficulty_type} ${cl_1st_clustering_type} ${cl_1st_min} \ -# ${cl_1st_max} ${cl_1st_total_step} ${cl_1st_difficulty_step} \ -# ${cl_1st_root} -############################################################################### \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/README.md deleted file mode 100644 index 540763fdd..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Megatron-DeepSpeed Rebase with Optimizations - -We rebased and enabled DeepSpeed with the latest Megatron repo. This folder contains examples that demonstrate how to use the new Megatron-DeepSpeed for training GPT like models with new features. - -## Rebasing Efforts/Achievements -New features: -- Enabled Megatron-LM's sequence parallel. -- Enabled rotary positional embedding. -- Enabled FlashAttention v1 and v2. -- Enabled new fused kernels from NVIDIA. - -New optimizations: -- Enabled attention map memory optimization, where we first generated attention mask on CPU memory and then moved it into GPU memory to avoid out-of-memory errors when training with very large sequence lengths. -- Position embedding partitioning, where we split weights of position encoding across all GPUs when enabling sequence parallel to further reduce the memory footprint. - -Resolved Issues: -- Fixed the conflicts related to activation checkpointing when DeepSpeed was used with the newest Megatron-LM. NVIDIA introduced new fine-grained partial checkpointing technique, which DeepSpeed was not compatible with. Support for fine-grained checkpointing will be left as future work. -- Major refactoring to DeepSpeed pipeline parallelism implementation for GPT model in order to work with the newest Megatron-LM. -- Fixed model checkpoint save/load when DeepSpeed was used with the newest Megatron-LM. -- Fully verified the performance and correctness of GPT pretraining after rebasing. - -## Setting Up the Virtual Environment - -```shell -# clone source code -git clone https://github.com/microsoft/DeepSpeed.git -git clone https://github.com/microsoft/Megatron-DeepSpeed.git -git clone https://github.com/NVIDIA/apex - -# creat a new virtual environment -cd Megatron-DeepSpeed -python3 -m venv ./venvs/megatron-deepspeed --system-site-packages -source ./venvs/megatron-deepspeed/bin/activate - -# install the newest DeepSpeed -cd ../DeepSpeed/ -pip install -e . - -# install apex -cd ../apex/ -pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --global-option="--cpp_ext" --global-option="--cuda_ext" -e ./ - -# install pybind11 -cd ../ -pip install pybind11 -``` - -Megatron-DeepSpeed's sequence parallelism can be combined with the following types of attention. - -- Classic attention -- FlashAttention version 1.x (enabled by `--use-flash-attn-v1`) -- FlashAttention version 2.x (enabled by `--use-flash-attn-v2`) -- FlashAttention + Triton (enabled by `--use-flash-attn-triton`) - -FlashAttention version 2.x may have numerical stability issues. For the best performance, we recommend using FlashAttention + Triton. -We show installation steps of thoes 3 types of FlashAttention - -```shell - -# install FlashAttention version 1.x -pip install flash-attn==1.0.4 - -# install FlashAttention version 2.x -cd ../ -git clone https://github.com/Dao-AILab/flash-attention.git -cd flash-attention -python setup.py install - -# install Triton-based FlashAttention -git clone -b legacy-backend https://github.com/openai/triton -cd triton/python/ -pip install cmake -pip install . - -cd ../ -git clone -b v1.0.4 https://github.com/HazyResearch/flash-attention -cd flash-attention -python setup.py install -``` - -## Example Showcase - -One of the optimizations enabled from this rebase is to enable Megatron-style long sequence parallelism. To enable sequence parallelism, add the `--sequence-parallel` flag in the training script. We provide two training scripts for ([GPT1.3B](pretrain_gpt_1.3B_seq_parallel.sh) and [GPT30B](pretrain_gpt_13B_seq_parallel.sh)) that enable sequence parallelism, which are available in this foloder. - -By default, the degree of sequence parallelism is equal to the degree of model tensor parallelism. The users may also want to ensure that the sequence length is divisible by the degree of sequence parallelism to avoid performance penalties. -Please also ensure that your model dimension is compliant with FlashAttention's requirements. For instance, to achieve the optimal performance, the head size should be divisible by 8. Refer to the document of [FlashAttention](https://github.com/Dao-AILab/flash-attention/tree/v1.0.4) for more details. - -## Performance Comparison between Old Megatron-DeepSpeed and New Megatron-DeepSpeed - -The following experiments are performed on 4 NVIDIA DGX A100-40GB nodes, connected through 8 HDR InfiniBand (200Gb/s per HDR). TP stands for tensor parallelism. - -| Sequence Length | Old Megatron-DeepSpeed (TFLOPS) | New Megatron-DeepSpeed (TFLOPS) | -|-----------------|----------------------------------|----------------------------------| -| 2k | 25 (TP=32) | 68 (TP size=32) | -| 4k | 28 (TP=32) | 80 (TP size=32) | -| 8k | OoM | 86 (TP size=32) | -| 16k | OoM | 92 (TP size=32) | -| 32k | OoM | 100 (TP size=32) | -| 64k | OoM | 106 (TP size=32) | -| 128k | OoM | 119 (TP size=32) | -| 256k | OoM | 94 (TP size=32) | - -The new Megatron-DeepSpeed is able to support longer sequence lengths without triggering out-of-memory errors because it enables sequence parallelism, which partitions the activation memory when sequence lengths are massive. The new Megatron-DeepSpeed supports FlashAttention, which reduces the memory consumption of the attention map calculation from quadratic to linear complexity with respect to the sequence length. It supports position embedding partitioning, which further reduces the memory consumption. The new Megatron-DeepSpeed can achieve higher TFLPOS because it includes new fused kernels from NVIDIA and supports larger batch sizes using the memory optimizations without triggering out-of-memory errors. - -## Acknowledgements - -We would like to acknowledge the use of the supercomputing resources of the Argonne Leadership Computing Facility (ALCF), which is a DOE Office of Science User Facility supported under Contract DE-AC02-06CH11357. The resources provided by ALCF(Argonne) have been invaluable in helping us to conduct this work and achieve our goals. diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/ds_config_gpt_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/ds_config_gpt_TEMPLATE.json deleted file mode 100644 index 14290ec03..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/ds_config_gpt_TEMPLATE.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "flops_profiler": { - "enabled": true, - "profile_step": 1, - "module_depth": -1, - "top_modules": 3, - "detailed": true, - "output_file": null - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/host_file b/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/host_file deleted file mode 100644 index 91fe1ab43..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/host_file +++ /dev/null @@ -1 +0,0 @@ -worker-1 slots=4 diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_1.3B_seq_parallel.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_1.3B_seq_parallel.sh deleted file mode 100644 index 410a047b1..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_1.3B_seq_parallel.sh +++ /dev/null @@ -1,349 +0,0 @@ -#!/bin/bash - -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -n_k=2 -seq_len=$(( 1024 * $n_k )) - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=2 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=8 - -## Sequence parallelism, 0 is no SP, 1 enable SP -enable_sequence_parallel=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=0 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=1 - -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -# estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -estimated_train_iter=6 -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -data_path="BookCorpusDataset_text_document" -if [ ! -f "BookCorpusDataset_text_document.bin" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -fi -if [ ! -f "BookCorpusDataset_text_document.idx" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx -fi - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi - -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="output" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -tensorboard_dir="${output_home}/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-flash-attn-triton \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [[ "$enable_sequence_parallel" == 1 ]]; then -megatron_options="\ - --sequence-parallel \ - ${megatron_options}" - -export CUDA_DEVICE_MAX_CONNECTIONS=1 -fi - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} 2>&1 | tee ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_30B_seq_parallel.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_30B_seq_parallel.sh deleted file mode 100644 index 12d49d570..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/deepspeed4science/megatron_long_seq_support/pretrain_gpt_30B_seq_parallel.sh +++ /dev/null @@ -1,360 +0,0 @@ -#!/bin/bash - -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -n_k=2 -seq_len=$(( 1024 * $n_k )) - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=16 -# global_batch_size=2 -# lr=2.0e-4 -# min_lr=1.0e-6 -# init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 30B -model_size=30 -num_layers=64 -hidden_size=6144 -num_attn_heads=64 -global_batch_size=2 -lr=1.0e-4 -min_lr=1.0e-6 -init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=32 - -## Sequence parallelism, 0 is no SP, 1 enable SP -enable_sequence_parallel=1 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=0 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=1 - -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -# estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -estimated_train_iter=6 -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -data_path="BookCorpusDataset_text_document" -if [ ! -f "BookCorpusDataset_text_document.bin" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -fi -if [ ! -f "BookCorpusDataset_text_document.idx" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx -fi - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi - -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="output" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -tensorboard_dir="${output_home}/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --pipeline-model-parallel-size ${pp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-flash-attn-triton \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [[ "$enable_sequence_parallel" == 1 ]]; then -megatron_options="\ - --sequence-parallel \ - ${megatron_options}" - -export CUDA_DEVICE_MAX_CONNECTIONS=1 -fi - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -# Since mp_size=32 involving multi-node compute resources. Users may need to specify hostfile via "--hostfile=myhostfile" command line option. -deepspeed ${dir}/../../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} 2>&1 | tee ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/README.md deleted file mode 100644 index e8641ced2..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/README.md +++ /dev/null @@ -1,24 +0,0 @@ -## Example of Finetuning LLAMA-7B from Hugging Face Weights - -### Dataset -You can access the dataset from [here](https://github.com/tatsu-lab/stanford_alpaca/blob/main/alpaca_data.json). - -### Pre-trained Weights -The pre-trained weights can be found at [Hugging Face - LLAMA-7B](https://huggingface.co/huggyllama/llama-7b). - -### Usage: - -#### 1. Converting Hugging Face Model Weights to Megatron-Deepspeed Model -```bash -bash examples_deepspeed/finetune_hf_llama/finetune_llama.sh convert -``` -This command writes the Hugging Face model weights into the Megatron-Deepspeed model and saves it. You can adjust the parallel configuration in the script. - -#### 2. Fine-tuning Process -```bash -bash examples_deepspeed/finetune_hf_llama/finetune_llama.sh -``` -Execute this command to initiate the finetuning process. The task originates from [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca.git). - - - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/ds_config.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/ds_config.json deleted file mode 100644 index 9c0b33247..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/ds_config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "train_batch_size" : 256, - "train_micro_batch_size_per_gpu": 16, - "steps_per_print": 100, - "zero_optimization": { - "stage": 0 - }, - "bf16": { - "enabled": true - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/finetune_llama.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/finetune_llama.sh deleted file mode 100644 index c48ea11b9..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/finetune_hf_llama/finetune_llama.sh +++ /dev/null @@ -1,110 +0,0 @@ -DS_CONFIG=./examples_deepspeed/finetune_hf_llama/ds_config.json -DATASET_PATH=./alpaca_data.json -# dataset link: https://github.com/tatsu-lab/stanford_alpaca/blob/main/alpaca_data.json - -HF_LLAMA_PATH=/data/llama-7b/ -# weights link: https://huggingface.co/huggyllama/llama-7b - -MICRO_BATCH_SIZE=16 -GLOBAL_BATCH_SIZE=256 -TP=2 -PP=2 -# require to align with weight dimensions -HIDDEN_SIZE=4096 -FFN_HIDDEN_SIZE=11008 -NUM_LAYERS=32 -NUM_HEADS=32 -SEQ_LENGTH=512 -###################################### - -MEGA_DS_LLAMA_PATH=./"llama-7b-mega-ds-T${TP}P${PP}" - -# Below configuration required for llama model as per llama paper -# --no-query-key-layer-scaling \ -# --attention-dropout 0 \ -# --hidden-dropout 0 \ -# --use-rotary-position-embeddings \ -# --untie-embeddings-and-output-weights \ -# --swiglu \ -# --normalization rmsnorm \ -# --disable-bias-linear \ -###################################### -cat < $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH_SIZE, - "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, - "steps_per_print": 100, - "zero_optimization": { - "stage": 0 - }, - "bf16": { - "enabled": true - } -} -EOT - - -covert_args="deepspeed tools/hf2megads_weight_converter.py \ ---hf-ckpt-num-shards 2 \ ---origin-hf-ckpt-dir $HF_LLAMA_PATH \ ---save $MEGA_DS_LLAMA_PATH" - -finetune_args="deepspeed finetune_llama.py \ ---load $MEGA_DS_LLAMA_PATH" - -comm_args="--tensor-model-parallel-size $TP \ ---pipeline-model-parallel-size $PP \ ---lr-warmup-iters 2000 \ ---weight-decay 0.1 \ ---clip-grad 1 \ ---num-layers $NUM_LAYERS \ ---hidden-size $HIDDEN_SIZE \ ---num-attention-heads $NUM_HEADS \ ---ffn-hidden-size $FFN_HIDDEN_SIZE \ ---attention-dropout 0 \ ---hidden-dropout 0 \ ---no-query-key-layer-scaling \ ---disable-bias-linear \ ---normalization rmsnorm \ ---use-rotary-position-embeddings \ ---untie-embeddings-and-output-weights \ ---swiglu \ ---seq-length $SEQ_LENGTH \ ---max-position-embeddings $SEQ_LENGTH \ ---micro-batch-size $MICRO_BATCH_SIZE \ ---global-batch-size $GLOBAL_BATCH_SIZE \ ---train-iters 3500 \ ---lr 2e-5 \ ---tensorboard-dir tensorboard_output \ ---lr-decay-iters 320000 \ ---lr-decay-style cosine \ ---log-interval 1 \ ---eval-iters 100 \ ---eval-interval 100 \ ---data-path $DATASET_PATH \ ---save-interval 1500 \ ---split 100,0,0 \ ---bf16 \ ---zero-stage 0 \ ---tokenizer-type HFTokenizer \ ---tokenizer-model $HF_LLAMA_PATH \ ---deepspeed_config ./examples_deepspeed/finetune_hf_llama/ds_config.json \ ---deepspeed \ ---distributed-backend nccl \ ---num-workers 0 \ ---no-masked-softmax-fusion \ ---no-bias-gelu-fusion \ ---no-bias-dropout-fusion \ ---no-gradient-accumulation-fusion \ ---repeated-dataloader" - -if [ "$1" = "convert" ]; then - task_args="$covert_args" -else - task_args="$finetune_args" -fi - -full_cmd="$task_args $comm_args" - -eval "$full_cmd" - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/generate_text.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/generate_text.sh deleted file mode 100644 index e29d521e1..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/generate_text.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -export TORCH_CUDA_ARCH_LIST=8.6+PTX -CHECKPOINT_PATH=dataset/checkpoints/gpt2_345m -VOCAB_FILE=dataset/gpt2-vocab.json -MERGE_FILE=dataset/gpt2-merges.txt -b=8 -mp=1 -experts=1 -nodes=1 -gpus=1 - - -use_tutel="" -#use_tutel="--use-tutel" - - -ds_inference="" -#ds_inference="--ds-inference" - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -launch_cmd="deepspeed --num_nodes $nodes --num_gpus $gpus" -L=24 -H=1024 -A=16 -#experts1=${experts[$k]} -program_cmd="tools/generate_samples_gpt.py \ - --tensor-model-parallel-size $mp \ - --num-layers $L \ - --hidden-size $H \ - --num-attention-heads $A \ - --max-position-embeddings 1024 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --num-experts ${experts} \ - --mlp-type standard \ - --micro-batch-size $b \ - --seq-length 1024 \ - --out-seq-length 1024 \ - --temperature 1.0 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --genfile unconditional_samples.json \ - --top_p 0.9 \ - --log-interval 1 \ - --num-samples 0 \ - --load $CHECKPOINT_PATH \ - $use_tutel $ds_inference" - -echo $launch_cmd $program_cmd -$launch_cmd $program_cmd diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/README.md deleted file mode 100644 index eb5fb415a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# ZeRO-Offload++ Tutorials - -This folder contains examples that demonstrate how to use the new ZeRO-Offload++ features. - -ZeRO-Offload++ now supports **Twin-Flow** feature. - -## Twin-Flow - -Instead of all-or-nothing offloading strategy, **Twin-Flow** allows a portion of data to run on CPU and the other part on GPU simultaneously. Thus, we not only mitigate the memory pressure on GPU side by offloading data to CPU, but also utilize both CPU and GPU computation resources more efficiently. - -![Twin-Flow-img](./twin-offload.png) - -As shown in above Figure, when ZeRO-Offload is triggered, **Twin-Flow** now allow user to set a new configuration arguement called `ratio` (default value == 1) to adjust the portion of parameter updates on CPU optimizer. For example, if this `ratio==0.4`, it means 0-40% of parameters are updated using CPUAdam on CPU side, while the rest 60% parameters are updatedusing FusedAdam on GPU side. - -## How to use - -Now **Twin-Flow** can be used at ZeRO stage 3 with Offload. Below we provide two tutorial examples on how to use **Twin-Flow**. - -### DeepSpeed Toy Example - -Here is a toy example for using **Twin-Flow** inside DeepSpeed repo. - -Under `/tests/small_model_debugging/` folder, Run - -``` -deepspeed partial_offload_test.py --zero 3 -``` - -### GPT Model Training in Megatron-DeepSpeed - -To enable **Twin-Flow** here, we need to add two flags for Megatron configs as follows: - -#### Megatron Configurations -``` ---no-pipeline-parallel \ ---cpu-optimizer \ -``` -which have been added to `ds_pretrain_gpt_350M.sh` - -#### DeepSpeed Configurations -On the DeepSpeed side, we need to add follow configurations: - -``` - "offload_optimizer": { - "device": "cpu", - "pin_memory": true, - "ratio": 0.3 - } -``` - -Basically, we need to first enable CPU Offload. Then user can adjust the portion of parameter updating on CPU by adjusting `ratio` here. Its default value is 1, which means all parameter updates happen on CPU side. The above config example with ` "ratio" : 0.3` meaning 0-30% parameters are updating on CPU side, while the other 70% parameter updates happens on GPU side. - -#### Tuning suggestion on ratio - -To get best performance, we recommend to set this `ratio` value as low as possible without causing GPU memory Out-Ouf-Memory issue. - -One additional config on DeepSpeed side is - -``` - "prescale_gradients": false, -``` -mainly because right now ZeRO-3 does not support prescale gradients. - -All above configs have been added to `ds_config_gpt_TEMPLATE.json` - -#### End-to-end Training - -To run a sample training of GPT-350M model using Megatron-Deepspeed, simply run as follows: - -``` -bash ds_pretrain_gpt_350M.sh -``` - -Now the training start running with **Twin-Flow**. Enjoy! - -## On-going optimizations - -We have some other features inside ZeRO-Offload++ which will come soon, stay tuned! - -* Removing uncessary D2H memcpy in ZeRO-offload -* On-the-fly fp16 to fp32 data casting inside CPUAdam diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_config_gpt_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_config_gpt_TEMPLATE.json deleted file mode 100644 index ebcefa09e..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_config_gpt_TEMPLATE.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "train_batch_size" : CONFIG_BATCH_SIZE, - "train_micro_batch_size_per_gpu": CONFIG_MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": 3, - "offload_optimizer": { - "device": "cpu", - "pin_memory": true, - "ratio": 0.3 - } - }, - - "gradient_clipping": 1.0, - "prescale_gradients":false, - - "fp16": { - "enabled": CONFIG_FP16_ENABLED, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "bf16": { - "enabled": CONFIG_BF16_ENABLED - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_pretrain_gpt_350M.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_pretrain_gpt_350M.sh deleted file mode 100644 index 0a8a5ce9b..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/ds_pretrain_gpt_350M.sh +++ /dev/null @@ -1,316 +0,0 @@ -#!/bin/bash -DIR=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -SEQ_LEN=2048 - -### The "GPT-3 XXX" below are configs from GPT-3 paper -### https://arxiv.org/abs/2005.14165, choose based on -### your desired model size or build your own configs - -## GPT-3 Small 125M -# MODEL_SIZE=0.125 -# NUM_LAYERS=12 -# HIDDEN_SIZE=768 -# NUM_ATTN_HEADS=12 -# GLOBAL_BATCH_SIZE=256 -# LR=6.0e-4 -# MIN_LR=6.0e-5 - -## GPT-3 Medium 350M -MODEL_SIZE=0.35 -NUM_LAYERS=24 -HIDDEN_SIZE=1024 -NUM_ATTN_HEADS=16 -GLOBAL_BATCH_SIZE=256 -LR=3.0e-4 -MIN_LR=3.0e-5 - -## GPT-3 Large 760M -# MODEL_SIZE=0.76 -# NUM_LAYERS=24 -# HIDDEN_SIZE=1536 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=256 -# LR=2.5e-4 -# MIN_LR=2.5e-5 - -## GPT-3 XL 1.3B -# MODEL_SIZE=1.3 -# NUM_LAYERS=24 -# HIDDEN_SIZE=2048 -# NUM_ATTN_HEADS=16 -# GLOBAL_BATCH_SIZE=512 -# LR=2.0e-4 -# MIN_LR=2.0e-5 - -## GPT-3 2.7B -# MODEL_SIZE=2.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=2560 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=512 -# LR=1.6e-4 -# MIN_LR=1.6e-5 - -## GPT-3 6.7B -# MODEL_SIZE=6.7 -# NUM_LAYERS=32 -# HIDDEN_SIZE=4096 -# NUM_ATTN_HEADS=32 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.2e-4 -# MIN_LR=1.2e-5 - -## GPT-3 13B -# MODEL_SIZE=13 -# NUM_LAYERS=40 -# HIDDEN_SIZE=5120 -# NUM_ATTN_HEADS=40 -# GLOBAL_BATCH_SIZE=1024 -# LR=1.0e-4 -# MIN_LR=1.0e-5 - -## GPT-3 175B -# MODEL_SIZE=175 -# NUM_LAYERS=96 -# HIDDEN_SIZE=12288 -# NUM_ATTN_HEADS=96 -# GLOBAL_BATCH_SIZE=1536 -# LR=0.6e-4 -# MIN_LR=0.6e-5 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens -## For MoE model, we found sometimes training a bit more to 330B tokens helps -TRAIN_TOKENS=300000000000 -# TRAIN_TOKENS=330000000000 - -## TRAIN_SAMPLES is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the TRAIN_TOKENS -## above, and techniques like curriculum learning has less token in some steps, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by TRAIN_SAMPLES. -TRAIN_SAMPLES=$(( ${TRAIN_TOKENS} * 3 / ${SEQ_LEN} )) - -## Another termination condition in minutes. Set it large enough to avoid -## undesired early termination. -EXIT_DURATION=30000000 -############################################################################### -### LR configs -## LR warmup and decay duration, this token-based config is preferable since -## no need to readjust when the batch size/seqlen is changed. -## Original GPT-3 paper uses 375M warmup tokens and 260B decay tokens. -## For MoE model, we found that setting the decay token to 300B helps. -WARMUP_TOKENS=375000000 -LR_DECAY_TOKENS=260000000000 -# LR_DECAY_TOKENS=300000000000 -############################################################################### -### Parallelism configs -## Micro batch size per GPU -## Make sure that BATCH_SIZE <= GLOBAL_BATCH_SIZE*PP_SIZE*MP_SIZE/NUM_GPUS -BATCH_SIZE=2 - -## Model parallelism, 1 is no MP -MP_SIZE=1 - -## Pipeline parallelism -## Currently we don't support PP for MoE. To disable PP, set PP_SIZE -## to 1 and use the "--no-pipeline-parallel" arg. -PP_SIZE=1 -NUM_GPUS=16 -############################################################################### -### MoE configs -## Number of experts. EP_SIZE 1 means dense model without MoE -EP_SIZE=1 -# EP_SIZE=128 - -if [[ $EP_SIZE -gt $NUM_GPUS ]]; then - EP_PARALLEL_SIZE=$NUM_GPUS -else - EP_PARALLEL_SIZE=$EP_SIZE -fi - -## Original GPT-3 model always set min LR at 10% of max LR. For MoE model, we -## found that lower LR and min LR (than the base dense model) helps. -## For 1.3B MoE-128 model we used LR=1.2e-4 and MIN_LR=1.0e-6. -## For 350M MoE-128 model we used LR=2.0e-4 and MIN_LR=2.0e-6, but they are not -## heavily tuned. -# LR=2.0e-4 -# MIN_LR=2e-06 - -## Coefficient for MoE loss. We find that 0.01 is a good value at least for -## 1.3B MoE-128 model -MLC=0.01 - -## Below configs adjust the MoE expert token capacity limit during training and -## eval. To completely disable capacity limit, set MOE_DROP_TOKEN to false. -## Larger capacity factor or disabling capacity limit could improve training -## convergence, but will also reduce training throughput. -MOE_TRAIN_CAP_FACTOR=1.0 -MOE_EVAL_CAP_FACTOR=1.0 -MOE_MIN_CAP=4 -MOE_DROP_TOKEN="true" -# MOE_DROP_TOKEN="false" -############################################################################### -### Curriculum learning (CL) configs -## Enable/disable CL -CL_ENABLED="false" -## Consult the tutorial https://www.deepspeed.ai/tutorials/curriculum-learning/ -## for tuning the following configs -CL_START_SEQLEN=80 -CL_AVG_SEQLEN=$(( (${CL_START_SEQLEN} + ${SEQ_LEN}) / 2 )) -CL_TOKENS=60 -CL_TOKENS=$((${CL_TOKENS} * 1000000000)) -CL_STEP=$(( ${CL_TOKENS} / (${GLOBAL_BATCH_SIZE} * ${CL_AVG_SEQLEN}) )) -############################################################################### -### Misc configs -LOG_INTERVAL=1 -EVAL_ITERS=10 -EVAL_INTERVAL=100 -SAVE_INTERVAL=1000 - -## Standard deviation for weight initialization -## We used 0.014 for 350M/1.3B dense/MoE models, and used 0.01 for 6.7B -## dense model. Usually larger model needs lower std. -INIT_STD=0.014 -# INIT_STD=0.01 - -## Activation checkpointing saves GPU memory, but reduces training speed -ACTIVATION_CHECKPOINT="true" -# ACTIVATION_CHECKPOINT="false" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d-%H.%M.%S") -host="${HOSTNAME}" -NAME="gpt-${MODEL_SIZE}B-lr-${LR}-minlr-${MIN_LR}-bs-${GLOBAL_BATCH_SIZE}-gpus-${NUM_GPUS}-mp-${MP_SIZE}-pp-${PP_SIZE}" -if [[ $EP_SIZE -gt 1 ]]; then - NAME="${NAME}-ep-${EP_SIZE}-mlc-${MLC}-cap-${MOE_TRAIN_CAP_FACTOR}-drop-${MOE_DROP_TOKEN}" -fi -if [ "${CL_ENABLED}" = "true" ]; then - NAME="${NAME}-cl-${CL_START_SEQLEN}-${CL_STEP}" -fi - -OUTPUT_BASEPATH=$DIR/output -mkdir -p "${OUTPUT_BASEPATH}/tensorboard/" -mkdir -p "${OUTPUT_BASEPATH}/checkpoint/" -mkdir -p "${OUTPUT_BASEPATH}/log/" -TENSORBOARD_DIR="${OUTPUT_BASEPATH}/tensorboard/${NAME}_${host}_${current_time}" -mkdir -p ${TENSORBOARD_DIR} -## Note that for MoE model with billion-scale base model, the checkpoint can be -## as large as TB-scale which normal NFS cannot handle efficiently. -CHECKPOINT_PATH="${OUTPUT_BASEPATH}/checkpoint/${NAME}" - - -VOCAB_PATH=/data/users/guanhua/Megatron-DeepSpeed/dataset/gpt2-vocab.json -MERGE_PATH=/data/users/guanhua/Megatron-DeepSpeed/dataset/gpt2-merges.txt -# Public the Pile dataset, can be downloaded at https://mystic.the-eye.eu/public/AI/pile_neox/ -DATA_BLEND=/data/users/guanhua/Megatron-DeepSpeed/dataset/BookCorpusDataset_text_document - -############################################################################### -data_options=" \ - --vocab-file ${VOCAB_PATH} \ - --merge-file ${MERGE_PATH} \ - --data-path ${DATA_BLEND} \ - --data-impl mmap" - -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${MP_SIZE} \ - --moe-expert-parallel-size ${EP_PARALLEL_SIZE} \ - --num-experts ${EP_SIZE} \ - --moe-loss-coeff ${MLC} \ - --moe-train-capacity-factor ${MOE_TRAIN_CAP_FACTOR} \ - --moe-eval-capacity-factor ${MOE_EVAL_CAP_FACTOR} \ - --moe-min-capacity ${MOE_MIN_CAP} \ - --init-method-std ${INIT_STD} \ - --lr-decay-tokens ${LR_DECAY_TOKENS} \ - --lr-warmup-tokens ${WARMUP_TOKENS} \ - --micro-batch-size ${BATCH_SIZE} \ - --exit-duration-in-mins ${EXIT_DURATION} \ - --rampup-batch-size 32 32 1953125 \ - --global-batch-size ${GLOBAL_BATCH_SIZE} \ - --num-layers ${NUM_LAYERS} \ - --hidden-size ${HIDDEN_SIZE} \ - --num-attention-heads ${NUM_ATTN_HEADS} \ - --seq-length ${SEQ_LEN} \ - --max-position-embeddings ${SEQ_LEN} \ - --train-tokens ${TRAIN_TOKENS} \ - --train-samples ${TRAIN_SAMPLES} \ - --lr ${LR} \ - --min-lr ${MIN_LR} \ - --lr-decay-style cosine \ - --split 98,2,0 \ - --log-interval ${LOG_INTERVAL} \ - --eval-interval ${EVAL_INTERVAL} \ - --eval-iters ${EVAL_ITERS} \ - --save-interval ${SAVE_INTERVAL} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers 0 \ - --fp16 \ - --load ${CHECKPOINT_PATH} \ - --save ${CHECKPOINT_PATH} \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --timing-log-level 1 \ - --no-pipeline-parallel \ - --cpu-optimizer \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${TENSORBOARD_DIR}" - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [[ $EP_SIZE -gt 1 ]]; then -megatron_options="${megatron_options} \ - --create-moe-param-group" -fi - -if [ "${MOE_DROP_TOKEN}" = "false" ]; then -megatron_options="${megatron_options} \ - --disable-moe-token-dropping" -fi - -template_json="ds_config_gpt_TEMPLATE.json" -config_json="ds_config_gpt_${NAME}.json" -sed "s/CONFIG_BATCH_SIZE/${GLOBAL_BATCH_SIZE}/" ${template_json} \ - | sed "s/CONFIG_MBSIZE/${BATCH_SIZE}/" \ - | sed "s/LOG_INTERVAL/${LOG_INTERVAL}/" \ - | sed "s/ZERO_STAGE/3/" \ - | sed "s/PRESCALE_GRAD/true/" \ - | sed "s/CONFIG_FP16_ENABLED/false/" \ - | sed "s/CONFIG_BF16_ENABLED/true/" \ - | sed "s/CONFIG_CL_ENABLED/${CL_ENABLED}/" \ - | sed "s/CONFIG_CL_MIN/${CL_START_SEQLEN}/" \ - | sed "s/CONFIG_CL_MAX/${SEQ_LEN}/" \ - | sed "s/CONFIG_CL_DURATION/${CL_STEP}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --pipeline-model-parallel-size ${PP_SIZE}" - -# Currently MoE is not compatible with pipeline parallel -if [[ $EP_SIZE -gt 1 ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${ACTIVATION_CHECKPOINT}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -run_cmd="deepspeed ${DIR}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &> ${OUTPUT_BASEPATH}/log/${NAME}_${host}_${current_time}.log" -echo ${run_cmd} -eval ${run_cmd} -set +x \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/twin-offload.png b/toolbox/Megatron-DeepSpeed/examples_deepspeed/offload_pp/twin-offload.png deleted file mode 100644 index 1c8c3ef92454bc6443f9fbbe9a8a1940f9669f2f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59949 zcmeFYbyQUE`z}0m!vG>fNFzvtLrD!V5(1Jc9U|SKbmK76NQWRD0@9&0N_R+?LxXhZ z+4%XK-|u_Q`Qxnhu6MnEy|Z+=_c;66Pu%x)-}iMr&j+>V@4S5+*=>Y8paD!E+7zb``ur( zPKSawAkapeB3xR_-EjL8mis-OlM@TK+l8I&I{qGrP~i7}KC%qj=!}dz>0sOdFfBqF z(K=^f4w;xso;v(=>iWu321AjJ@zc(sukY zQ26I+^#9S-irU5D*W2OU+w(Br%fSrWvoCph3CHF|Mgk)}#V`CMqYR#)>8O97xtpEE*Q15f~ zlns}(*TRN*dpTj&*49(T1xA&1>6sW=8ls{1kXpp7pkcyY)V)~ms^xQtFW^1a_-2p!QB+)0By3PA@jH%1AWC+l$n18$V(2Fj!>@A6q?uH1A$?)NnwK?xWcPbMFOgob3fhS&^aNLA7XR@KSHdE;nUS!<0CVrn z%6QDXN2OHLDEm6)V{w%6xX-cb(p6+Uiz_jcY(9-_cO6h!lem>}n{pzqmMbiMq?%H0BGa@liwo4U8URPYb%b992htvO1>hZSDjAZux^8NeA zbiZW=8}|gNclp!(7P|JzE%GycGg18dz;eU>8Vn}#r2v@K8C9P0sg~$|!wDJp;cc2^ z*Q=xh3NAV3{Ij8>r(djAKUR;`Y(YFxe@F^ts`EyMLQ2*;#$V601RuVkUWGZ(MeGNy zD5PRHkCqDe=^8SANVLLgSNqg42f6=jWKVHkzeW)HrU6ILUINxb^U-~!siEvo_K8)F z;E2LEE(P+QHH*xNmxwf(v5HaLgD^9O-z#!Y7A9kz4fDB>q+3I#=+~&*o2X5}KZ>!d zPlv{^%ApaA9b%hFAod#R^U<9d=M1ZfR!0e^#)j<3)L)DlnB-hK_Nnj0+8qZv2Z?hj zjXWbb#w|@aX47=RtEBf6JY)bX&E8uk!8TxoLfQ#G_`f867dY_P}#$)y0a&6Hi0y zQID!Z+edE8x-DflH{LN(qPt_>yHfVOhb*V>&mj@l%&Kuzjs%kx4Z<3S9q%fdt4@;( zY2}{u`sn0Uj;l526>9+)neZ=gv0!I`-%a{%*V&cBu(NW_88}`v@o<~A;Qp4w+X6{3 z&idDGLq8gNG4m2Z{>Wv{8k^Qz{NT49%8DX9Z;^cT@j@upmA3{e^!M5AIbIEiLalW{SiSt2txfEHXr9Pr zhW^WQe9cZn%^c+$OLx%-DdEy~w#%G6kKK0F0>QPYx0AKa^@OJ8*rVU+y}7JH6N=?z z*-Ek+cKuk^`a%M5t+)n-2`~L4QdU(5o6qpy{*emveod8rrhWB$U}OmIOMy0kP3Cf7 zqgc<6ZmFfFR4=1?%TFNUQBhx$xa7}u`K#RJXw3@v41{yI4KgMBofa6Pg-q-*Uwj!& zH|J6LZSyNJpufrh^$X*?!6)Y1CliscC@SOuAGms*FD>!Yir4Rojk@|-g&^lX-HJ%- zu`LCZMTlm;K@W}7<Yj%4 zp1*;rGmZ*yo8tcnK_3X=tu_NOyuvv(5b85l-we$g8P>u#_1~(X#$TcDIxfnHYAw(1 z(R=RaiW9Q5v&Yd;wWu9?K48D?%&lwHeGVXi6NX`|r1@D=rFS;e-?Cv#=+9I7WnQ^! zIi6g_L(>96VRH~Kv~Q`~%`#v@$ezV!#AUFVrQI1=vXOPFR{h8)4F)ENEcejaBSDpw zX3KRj>!y@-GVCLokY$;gp}&6owB7oKpOn&1rHLGAQwhuMVznZTFA;>75pT{wdmxbu zLOr;{3OB2O6~jE|@|yt`FMJo2>#LHNEU;G7-T+LF`Ext>=A3&R@HS2jGI) zJ543mCZ$#>88dzvbPPc!FbDjERy+<3il#n4<|YwkKP53<54w6<;9T+xYNw8VOe>IK z=qXBq;q0Lk;&W=w-cfHkVJH4{Uohi4(|buC4MYEO-^zLP`fE9EQof(Fc~6_ZKXRXY zp-bWOaqt#wc=PC(x+eNz{SS@Yl#+FydHNlimPRN28eQ-NudDc83PF62`rojN2QCqj zao_A@J1JIKZN6aQ#s>2jfRG7iySe^?iJB8QC5=51lhpQS_!`yU{+P`zI&nVQy?kFl zAlw%7$nBBmVeN>>>I$7y7O{HjjsX081aQwb2{+}B*P)}{o#(~5Bn0=fpG0`^Y0u@o zpZ{K<&~1HWo$kgWL9Mzh?CBW|CjlaKPsF_70ZPvI$2}>uIExIg7|BKPCnxF8T0czG zT~X#JD`A{!vf4Xlc-@!*89+RZ4B_Ey?nW5-%4E^RF3ZN+*GZk5 z(5?M*dKZjFjteOF!Hx0&Ru&g1zCdLmTKfwue-A7E%Scjae33Mri99>g zIj6n&ApdM%9wmfM&lS?`@rkUiLVSTQCkPMh6ts}Lz%|9&6Lc&Q@@;5T_3>nUp-OLH z6WATI&*W3snvnC#;r$<-WHATvwx`lJSw~jG&zv7%Oz3Co5`FKfPpNPe%xK<%h7bNx zB82X{ep=X>Dd}xyH)gwP`B1{ zBv2a-1~0$=`17JS$}|4!bR537ev0{bR?V?hhXVC4XwJ9FNi>T5dd+Kg4$mZ!90Nx6 zFLom4WhwFf->wTj-AVMIOD$Z?H#?h-&(cXFIUrVT;@&%7ojJD@$FO|UBMg0Kj%>NV z6VE59+GP`YDaUbSrBkH)y@ZxvpxVi{2C_-{gbp9hA^-8;JiF|Q535+RmuQwMwU75E z@TlgHf`c(8`KPZ}R|XBKsSm8pET!JH@u+3E{S;S9PO#48y2{;5cd^=lS`J4386-<* zY229gTq9`Jr*IK))18C3F=#p|TY>G0oK_v2;^4J5to_ESya$~Q9gmPCfKa%UkOgA+ zNWUSp1>YaXOlQ41R@yS9rL8&L7xB!|qjNCwb97E>)tDdsi{g11>fiCCohx-ikk^+* z)iZcdu9G=gjLy<%Id2h+`^JoFh%g=n*{f-q-4Sv7&+^Jdi*l#3)IY4y==`yy5b^0j zYf4Z|qHEgIdmo4y$=;yT@`$ai{H{01QbiWKKTVH&+ffhgv$RC>K!$?28nZyEl0^#ak~Yng}Orweg1G$ zi=&a2eOYcdllk^d#Z!ZE-oE1_OTA==B*@&<;(hn1vq(NNn9*6-TK98Hx%{NM=r6@X zD&(0sW9a+qvmfc%(Qkzdh-CM0utEhUu{Dnpb}@fA^7iT0NPoWPKT&tgP@n%$KtWz0 z?u2gB8a*pL*6ezACA9zqt1-lxr%X&fBwpE?^^2ov?43ELDo;YYAO;d?Ed+@LmrkN* zMoH)smGR9g)E}kpW1KkZPV=x#Uln~P?Yko8Z&sU*3C_%vZ9l;WI-z?K+fn5?7kRX6 zkuE%lbq*)Iy*)I#%{v%R=~lyAEizuml{Ju~hC|*ce6i)m{*sLj_ouVPaF<3!H6yi+ zwR2kJb6Gdh)Y^Q}_`YNfDja?G2w6HB(^OS-eeJPs@?rAx{!N|2Q{A6x0oeD5#HfF4 zz7^{l8W4}-h8kCWk0U3l=Z{*MJC(|#G#X%z?X;-3`@9r;y}oek8#(X36*LA#`l2BW zJy(cQHdA0I$PdToAdLCpd@OIr`LZaWx`?x4XalpZ4kV4F9>5IG=-V|RWIeqr2*#+W zu$Q0T&x_Hc!5v(DIn5*nSWm{wRzN+2Akp!+5ic0fKUZyw>-w|Uw%bYGChbKieB97h zfyJo~NP}QuiOd z?8Kka)g%Ea+;7BS6k}Tk&9GxivvVqRr<(d5`Ovy3ggh?->N_$tu+%|(`F4L+Is+Y9 z1YMCM03#S1lxE9Pnc$GfgXIQgn7=9JdRij3t4Eg!$E#|p#uT=%aFN--CMbqF*1@vt zJip};??^=phE_ztdVAncsIA60@5?}s48nVK&_GXdR!6AfV!#>0ob4^*U}~P? zk4p+s#TLt`J1pE{C8FPgKL#)9U$E)NI5TCulXy6Kc^=xe|J3dx@`BJvW>EYPUABk7 zc>x6NePJ(CnZT+f8c6P-_LZ^JnS182rmm^7ABLIqvnkpQyx?^$&z*mIcIHL+E;Bku zZ>($_)FgbUSsQS;^LA)N8&;-ll^BB5CE_ z4W_(9fiJ#qv}-X;D>cnpdOpoqLEy^5#5S^-D5iQ`n5&azel28D6Ma)4@tf~?aIgN{ z3;(qxzOGx?gwInn8;58q;Wi>&;^+>C1ijFB=tTW+W8Yqxfj=(MMQEAn?7AV#t>A=Z z$H<%H&jyW#Y&^ZJI>fTc#_Sn-xT{C5IkkuP-{8(DDp7fJ@PSP-!mdx1Jez~R5aMO1 zZ-f#RzIlJ)EH3--T!RifxfS(FUHiZQe^qW=t5odj)^5riDx3%5*?j!^(RX?+yyEq!3=V8mxDsm8CKV!9M}k z!SuYldN2G*f5dk@ZKzs_{?1S9Bhy{VN`K;*(uc7>(MSB;2 z^buY#sFQC)+8`r&9}LA1v7AUTfKS#UTW3w~(+hCe^R3Zg5A9 zA?SXOHEm)E;^Mt$-ha_TpsaHHknRw#7Eb2D$v0nKLy!#+B>ug8vLvksj?w634o;?T zV^_U)vVh{ETT>aZ?2#gEScjnfq)bk0iB_c`H^Gx1L_>DYB%*TtV&Qo_7^~{wp!Pb^ zq&L@v(BQCZ5``3sCALD9+7Dw*6R!5f_5$EP#<*WN4+E@)cN969hag6I9d?K=@eK7@ zb}zY4D{>B8<4ea)v+EbE`ia<8gSf(o4~ZV%4VQZ(_L!QlwW^=vT=?DY8fn!$(|nY; zyZL3Te%IEMbx>@~E$4gO-213%t@j8460^%Fn%?mHzDM8XbGCv@Y)U6SA7bcv%uNm49H;B&y%?q}Fg$E( zFtAE#aOB45A0InddJ>gFC-=4K1c<3pfqq5|QrMjIh({1#)NY+<`B8@+y~Y<6l;+{=T@*ij9`UR`y;-OZe;7XKz|m*CUrHP?Kj~w-wl5EOy$zfnllNP zsfRtU3RsLrPxM!nJ_?u0F2wY17&LwTcHncr114f6jqamehDl*oV7Oi>Kkz~Rht=k! z*TX~`bPURx>PFudwPt`UhBSXG*sv}$#vQX0L#!1~fCW^x%hwd&Bvt!IPgm(nGv#g@mt&NSvHMC}^} zUD}&cIJ9gS)p3g3jdJW}-?7(U__TT$pgJ7sF?98&nta160{>bBF+SEsfmHs!yK8nn(zrKfZC9!C+xkS^(}ErH9*& znEa~cvHAMrTpX?B8mcz%`}}4{?>f17o7(J;E1Sb?KZ}O8DP|zEJem~Pvf{6$>;j&(Pf#@mPsby)~{`+5kg4wVMF3Vp1N zG2oby87@e3{F$m(1Cs?!u%mqHb%Bv71;Bm(Ppj~pVoY8u3-tO~1EFow>*C+B`kKoa zdsTRw6hG!r6#}7#zS?QM{Mz`4=wmsNKU&0U+*8H0dCd`GNNwn<*Pn+D_K+Cth#iid zPpU@WVlJWt^Oq1*Ez6Y`p~b0<-ZVb?#RBmblTQ0LvvKy}I1#Bdso&^9_$TkHs}rR2nOas{YwH?y^TSSxrC{oEpJq8x=@SE>-{3hxUbwxMeEe(PYxpXUrY-}H8@4?(bR+n;6;nIcxNn0rc`?q$WD)snf8Qd`bId3{jKHy zlBU>TkEfrOI!>$lYnTGPT4&wQoJp>HzYiF@1dOIteCy}fnbONURgV1?w|a1GRmTW@ zD$6s><%Z3Koisv0(|vOdX^6(Y#dxN1+8ObYG6-oVDf#8Alc~GmFE2TxH+)k{W?zKz z(KF&!uG3kY1#2}%Jsh)>Qxc~@j}y)_(@1ibQ02;~{?0J$gC?lAc!%fID;DZ?&XGmE zyh?L+se$s1hB>{+1xe=}l}-yQk^IT1^1&D7gu&kSWvaBoBa5}__tR{)J1qp)EF_H( zjB|}HKDUA$qLs5zxh5GRB$LEe;WpOOCvjKU;h&Ws&(;iy?fNL2aR@V>4#j#K*lV z$4ZmSZ)A=5Z!QnG!DpvKvVsXm;Z1&9su_(<;f^qU?~~Y9voF5POvT=55dSS9fwD-I zh-2h`Q0dH%cmTwGTp8jj}RxB99Lr+7xxQ__A@t^$rv|#wwf35_gjxHycF~3`IFD@lT zSLlDIA*lOcRY?y{(BI7hGMe8=S=zJGOiE~kuexKto+$=`}zFLkZTA$dbHeaP=38&|{ z5ya;fM6fN_R=ZliGuLlc8&q4+;*gI)2L#q0_3N{_2R*k--;dgBYx|V|xplMqugbbo zU_alWcz(E+Pa|xflvtp}F&+!3n|F@@-mS&q75)G1&GpscpjMf&^uHwPhK4s1BS3e@V*$$OS*~Xm`nvbc`P$LR zjKiqQpak2=Y4h!;be|KQ^V^cz`H#ndMa6F_KD$$#C7eu>ip_cx?#h1&V{YB5=wpuf zXU8nLupEuT^tv11$>CCi@9Xa=4eLk{>3%mJ)AcU-#jgsrir5|je)TM#`QrL?q0nxo z=BT#$TE!UGZ>Gvhw)y%bRrEceZA*yWO=J-*X%-BLO{LBR4|%senK{gO_aH!Q6l#g) z-CnIpjeUD-ldLsY^n&vPH87AD&jH0cS7W6BOu?a55PkImXd>YvBgFL#?0P58VK7><=zqYncfz@kX;p7>tX!9yeIBsk2k znjID-KR`;1nhk4a8og>^qkyq(%s4a)?n(o4f5r2=rQlEkGC!b$N^1k3$lcL?*v{6e ze>T57`6D21jwb=i3Q3L`aMmU|XJKROC3p~XyD^$?=6CUd1eimLzjG+GH7IqPBj&b) zn+DPVancCcf*&(<3E;mW@};xDre)0FZ90=##68D|<~zl`l<+uUnQQXVdua6q@C5F^ zEl`oAV}AEI%!Aht+%`3ehte1Vgp(~-3@{`s=91g<6joXjGU0SBC1vG%3T+O>?fb?RwZQu1#sh%K zgT;^{<=5nm>Y1DSbaRFxIi$|gtF>2e4a6$+y}meT=GZ!#b6Xhx8AgdkwT7y#-9uthS9DPQm#y=k?FfZqy87 zwJ%xLq4~N7YG2fcr)s$*j&gbjLq~k#)1lfkZeSl7 zG&Duh=gCtzTzn4?7mDEF;rVFZ5l)2;8sLD>?7{CdfXa-OL$O&v_xFey(s-GtUlR#9 zS5o;0i^{o^5qCy1YNz28capmx#ecU~0`K(zQke=HM$drE@{wGnUU28{pJA*|f;&qY z#T7A!9vL>&d2=`3;ls}N-H#a=M%FH{r*d`Jj zmaTmtO_Vu8(l8Rq9=#$K$dwIq_v{<)CyV5wU&Em_xgi|^MV1;h^JLHfD<8=Xxa}N4 z)Qjeui>)fhB^fG7j9KGSf-28rYtOY$Ze0@-uY6^h}d~NgD{Op1QAo80qD~R zK*wT#@c0e~0aE1S%PQgG6)Za``2sm;WnM7DruC zM;$Vw7iN&E7dn3hCZb-(@`vQvB1VdUStuOA08(G+i7!;klo`uWB)6V#sE4eQ9j_0| zQuCls|Qb9>vp@VXIbFwe6{#S0i8~a2SIqlNMmDrFcU-$BVA{R z57q&qZ3jV<+&>9PjQnJT~|wiT7A62X30!@Y%vhG#pm=J{^7{O!=w9q9_T*1 z!}L|fNVrQt7vsaNnF>{mNXImh!47?{G(TFlQk(&jhqro795T5YQ+Fu=(09vuzgr(_ zDPQkR%vXIY?5|4GxO$O843Yr89Gv;uZw#Yo4BVwzsr6d09?CF33qU|31Xk-l#|&_R zsLbKd!Hv@Mn)%Oj0f5h^_RS_lRzWt<@R6UxX@mhf8%0#4ksI3bC|$qZo5%KWHPr$C z$LoT}4<90cz{Jp=3JGm<(n(g_B~1wb1OAy zcEeU3TI%M57FqW3VSIMkQL0vN@=Q7G2w1Svn?BHe0^7hhhgz}B*( zB(u!>Q`EJ!wXJgi{IbgMy(sM4LRZe*){e}d0x==`Om2v>^hB`z@gBBdJO|zX)${QM z!W2DeY3?NvH8eU?E3RL6Iy}XR$WfwpQ-hcEM zL`$tgOG&NkkDi_*iwc#yA5ntFX+^VA4227W;{8Ls&ktgno13BQd~w|D>`2M0J=p+t zf2>jxx(#`q{b>w;PG1_AV2~3?R~!v#)`b&OQU!ZMOJCFKOZd12Ys=(}-Z;z$Vg>A+ zF$_OG3-fB}A{Z9jW)|~jFUh18A--}DA0QEFdyCnXHG3vB8zpsP*>nH|;Cujmdnnpa z1CXK*;o;tk?>#^{>e+I@pNImJb_Cg{7IGK@{D>NA8C@K7ygfBqx=M1lFp-vZ`)`tl z-LqOd8tyS)_;-wh0=J>iWyl4!z#H7glW7})1?)YOZF(~_1$R%+o*~F8<`TbeACC1* z4Y$PUtP7c9+xPU@*;$qchVh`!C0d0aVWbI+&VhLQAST+!i$DFDOr6C`w}SD=S$aa= zVmlh&x}&kR5Lodkhgim=F?C^_o1~L41#}TuMUyZY)RBNpKvJl>%3d`?)P)HhMY=YI z2)c;6Bx>s_iT3MXhri1>#>wxmaxY0d^B0FAELr%+FmYu}KP?T-BaMj*3SO%4yE)7_n)ll6egan|FV1c!z>9oYH>vf85xt6njwt(Kqi=A zYtLD=&r0Ua7bjGQDj{+t#mSDpkY<8r;=vlt%F{T4Q!F6v4k8P*RA2t>IXGRUP5dDc zrs2MP^tch3`|b^9*&j^e(*$7XS=>p&!TG1)Y8edsWb9$eI2s4iPBq9PuB=IGYwOW3 zJqJ9@Pa;k-gf0khf2$1%-|`2PJd|)ok2o3aHF#fI3tu(cM33<860OMqC7=YUvJ)6q z@_C)u2GA3}N9zKYv}q#zIa(;Pl-)DgMyR8vm;;Y@{cgnsWrAH3I*(cU$)X*vBRbEW zj21;mOdB#nJt@O1%e~(7hTw~!6sCQe$YdRJ%)mb9z-%Xkgbm|qxa0Zi4|<(rCq3z zi`fRkm4o75Vk?+PxwnfPU{smV5x3y`p&7_k;kCbJCs++Nu}>z&YkM~-%%pqf9Yz{o zAzqpW1(4Nf@utOzfdg?EY!hf=C~j4tT3o0##f-(o23~z=yhF4aSYX!d;pL@-ij*Tf zd^3dVe4`8J8JeVWL^Cdv#}EgJ^VzTug~=e`Pl8bW z0zzJ#{7JpIp|=gHmy-)-7Z=yhQb^i;Zz7w<2001Y9dEqY{6cmttpk>5-J4wQj$QdA zhvY)b+sk?m597NEjsl?+MNp4;nRy8-87tn~1UhTeB{n7jk)2)@m04UD zmG@6N{hUGMRYN$KTzkA?0UX}~H-^4W5)VVr6Nys`U?4`&5fW5c56jET1wwFJOyflD zsNvds&l%w!AO;;Y_!-Xw#7sCN9HnpK{4QPG0~1+BE9;U3$ZoAy3@u>9bQ$5=1Je6_ zoakVxGINleJsJ~a;DImdrj?(u4LLycnIxkCNNaT1?k14>3)9okB)`J+V7}r2Lve&9 zpB6%n3KU?8wHjhvFE~_&gCYJoEy)4{hE@z&G=|rKuMhV2q6Z6&e?ucNVshd+Urhts z7vvjqNL}8Ce^{BMykSBQK8KxcD+FCr znMvQnll3{lg^NUqn5T)&vUT_bH3Ktc$fsNowe>|yhxe|3Y3%7Y>MP-UflvExVay?N zL@qKz5Ysb^c1sYZ5}(YsG-ws`6IUWqFGoHZnzw+IcOLt z3p6uH5Tc6M;3Z$^X98F-+Ykt@J)iZI;KTNhpFi=ITfjuE=@x_Onmo{wXqn*gbXrIY}OVPlUWIuBwO|%I|On4{tP+oKb)?6 znx4fsYA`sXKLSrh?Gdwzzl=c8@&%iKp$8yzW_16KVNgyL(R!=^+)GWeZ>^Qw&~Caa zhSKo&25;X~D<4%B{~0k=}53^MIeOFZ!Rd;ySQ z8aFpLG!_tnyiUq4T@>ror){5|1#ROA<-T>P5cQ2&Ah==?E8^mYF`mOB z2eyoNbavNN|-f(-y&qZdyX?Pf^?{l+dfb zQ`K88;ZI)HV4t z(27akr07Ws6l;i9+(V|p@Zt5U?m5(lZ-=6@_J4TOU_=f@GxfY~QGX}A&1Wi=V9xzu z+hnVl+fUoD;~=C&HMPSz*^Js$XWDhjq(jotz|jX(j*Yo=K?M$eg{|#ts|}=zA^$e4 zclMIzAu;L+I+9SN&4{qtA)iN16JLV~K3#!6zdUT|C{pvRCa z1avta0VVrgotVogXKK?zN^RVyB#p9}(q`7kOsF>Z@QpZ!iQHo6pd&$w?iG89M!x(; zqVdnnc6-Kw?E-v@aeSvT$BvJ{qIr8hoTv|igq@!mg%)wk-t>tH^y#8X6C^9eqdI7L zZNRQ8Jro`v)d228(u@X-rm?87$Np%(6u*6xUf5wCn%~v1H+^G~?je^>Tkdtek1;H) z=>3{(@)z0l8P@RajfG+I^XuJaxYPrN3wzFChr~VtdcfH`CvMLSW#-!6nDmOuCpf;D zD$^4>vHsP6pigxe;3&yWAJxg?)WEOH_VkNZz0njl-2GT;{&kq}-0$~ReBLy8yNsK`r8WjUY9_;p#*i| zE0W0{qjqOq)^grS&vGmLfN5A#sdP`ZRa1-hz^iNtz_`ml)Cbcg{_-fiJ`H<69ii)^ z`KrRa-A^yJUS+v3Q~5&zeVa&nSjcXPe#r}P8QA@`>R=HeJ7uOZhZ4B7(*;TctQtbwBGVIYTrpTV89cuqz} z(nn8CvUY1yDU*&D96mgMXTr|$^k|0Suq|XgKN}umbI|R~c#(W7rS%jjfvrpqcoCWE z!h~SHC!`%>E?kW0;CQ|M0}(E^r!+AFEsO&g>5HRm?OGgb!0eJX=@V&F30lXA&>ppQ zM9{D;P_$v%$Sy4{EpKk7T;xDHR>SUdKl?LNt7?Ey4D+XHa9+?j6K?^iINr-ZJ}&=> zLsTXD84G%_0+}!B_&VwP5g+4gGbp|lkwfkE&E@V-9bG+=_T^rj5VTu?{$sSGEANqp zzT)N^Syx|{;ijuo#7ObR*99a`zYy9my_Az=Nb&^-C%CXe%kOGG#8qSOM~agwjbm{4 z4}WH*G6e$K6H7TA6ZU5QN7G0oTHxLdgLq-SDgxjvZ84Z&Buq~{4p-xMaZQ{Bn*xa( z-P10PHkj3d6*IwE3P?hgRG~jj?1NY5hbuO}u#Huc1)es&`}B_9(Z7p?KC!l92Y{Bn3i8*n_|&;Z@))PPadi4d2ID^EkYfs$d3M z9AD;(aAM6!c=!vB_&JW5kiNa2R=7)+-k-k7N6`tmnm_7e3?!Gmhxe=6<`tF9WB+x4 zG(v};T~x1xkPtEGet%bTW&f)<+50DT46V|qCvTrIDW63zlV5v6vtSlDsp;&MtTIQP z!tgbBFXcR;qb;i}*>NP#Ae7e~fCjB(Ql^Zv1ggzEH2DrSazr{GrdbOvY(p`%+##+# ziR29^4NaO|>MC3&8X&oo(${mW`zSmCuKW=PM^8iWIzZl#k+#id0%;er5g?QV9B|dC z*c)NNU#(F5-beP_?@ud@3rYcRAi-klQ><9EHF57d0SMr@p>Ng4d=2+O?bamgBhb-h zt3Hg1dx*9HA~OaK%M{8Z|4F9>Vy5Ziz>NOcrLpZIa7s+TTGy8b5U0%?GfyhGt^m^NWj=#H1GN z*iSfw5C#{oJ%P%b>&srpgcZj-X1nlNJO?@xm`M_n&_>JDOZhn$i%I|4_~2aAZ$j8lW%5cBB>Nw4g!Cg4hXsG>Eq{-o5I`7-f6%sU>Ya{m z9B8SmDK)ha-&yFBNKix>ur6Xgf@f#?;IAcBaNI*Qo>30&c^%maiE5uyB~- z2u9ld=vAryW;7MkWB6p^;EHaV^!?^g+7N}|9U7V#>iTpsvB*gylWQ5F#W2)^(pc2 z@G3l`xz*|)k33QQ(KKQwJVZ!DN}9Iiv@&2p^jkB7=h3^D1MvLPcM$WPPz z+R9#y=Bcm}Jyd0eM#`U{!EiHZ+LWkCO#}{txOFO)o9g9N_5HJK2xYw_`6Zlc68|V0 zAsE8WiIQMGzED|ppRA0iR7%}pUN)6o-fuhhhI{USWaIp-;L<%F`w7244UDMC%Cd!Y zlLyg4h}-7C{D~EgR?0^?G;*hxy?Dtnl+k zdw16w61w>C=L~oH5MUN`0OQQ|SJeZWi25hJ`+M-RNdEtL4LX>DM}P84>F>GB7rJ-t z?~UIA${PTBcSjz_#%#*~-EfHB@5; zhJuDvO9sN)GqcptdyleU%ZtF5usdC~>h0Fc&G|;LD}xzEH@v!4?B{1^L%YquVTclE zOu$$Qh-pP&yA6lL*C_hi&%p7Q-M zt7J=fb5gfgqYYXsmOW$)DI!i;aQbHnT!rQ#=D}W40Nwj#Yux!4$ID`cVE{v?a{!RM zcvYm;y?LFPaEOrKU6LiWtF1BQwjjFVH)+GXSO99CQuUfgqp%|&kOQK{d&~XbKdbEL zo(PIn=H#$US6Rh2sBy8f{+8*j9GdHRGw+sSdBRk@Co0iJpfvt(9gEj%nQK9V%IKX05Y1M1~?%9!U5qT2Vh=O{0h_5 z>uvx5etZke1l4fzV~b!}fFr$4N}SB&Ld%MhyilN*)he_F#FW?qKxkey;;^_^dV98y z>Td?%Q!bd7*IyQZts;FM0HOf--t7Awh~eDrTmUri3WGa^IOt9e2M}=T?l(QscVGw@ zz3tsrg%t+i@=jx~x!cj;3i z>(>I;9IpSUz6_qup%D?e7sp%q7xI(-Xc+5v%62}(e%M|>LeqW8@zGJut71mz4+*c6 zoF6}a;CQn(A)~$lcedZWPD?!sIe?v1%WvA=%bERf`wCdO%FWHq>q_(fEFrsTT-;0< z9{sxLbl(droycQfUtd7uWAg}O^ZmQKV0j-9==B=y^=fQ0DGlAF=hJL-&R^V7m+|Q8 zCtCnSk7df@aHwZtasvxD9?~XXV?7>EJsQm@ ztAS?mXS(`myK*peL4^Q*SM_&D$FM8bUlB0Rs-mq;Yf{E6wrEuX(tJu@Hl+HI6S zTDida5FEGnvm02c>!)yyxiX+atrn;}#I$7k>Lc zDK)bXy1Tm%_HEtVPI3J2wjksK;)@z!gd4!BuI3hP0|Aa4uaQ(l`N%dL-$upn=GI|MnD#BZ7tnEjg*JeBy@I;Pmp$^h|b7lelXVT=hu_SOAjc)KnNZ9Yn(ljndCa( z-`Q~h%kypRK%5ZS*4Fl>NZQ@K?rOfyDH}cJ0Wg;zf}N~EG5k2|Cg@;p!-lf}#>dK^ zeg9NXSne&hqKhCQchyip*YkgyWaNkxja$F>I_ohey$Aq9MX$c5LlC2@1D})taqQT&q-HG&u`ovXh6dJ^=LT!yO)ocDiqFsiT8Sw9924p1&y6c$%IA8!S+W zVS@VIT#bG3K|=1x#ee1`-?TGQ9T0w(ui8$fEo&E56M2oE1660%Df?>%h#*)OknrC`Uus;3y10qbVBP=X@9?_}H zZ6;29A;Qpx%EE_F0qo43!EOLfL?*a9TmSg_3>CGSd)n(>Os9#@04>{#dLZ`ho)mt& zW>c-hLc%fP9xGHH`I@S>wO!-&a~%J#`@-9pvw@}0*2sM(F(g}l&%bLr z$J!<|EG)J_)7!hjqg!H0F*|aj-~(C!-_^BEgXn2cVX$>T@)f_y0hp1YZS|( z9ebZD6?=QpTP1#jUZPxAknn!B{DSGHZS!-f>rL^6Tj0=@C=EE%x6@hTwCp#=n1F?4 z;*jL0=KacZTd}5lNN2Sry;Ai?_3xHLS-gd}GS$TuXtq9-w4dcr??!*7HYW|%pn!v96Pu$jR>jMtfw-i6c3`A$X1A_)W}&9y z##dH1qo5t?J%X=WWgW{E_CKgP>wu`bsND}BJ#=?VH^IL{4(mzuL_N6?R(4UPZ94&f* z;XfXl3&A?2IlV8E99UE>6WJWQ6p%Xo3!xv7wRt3d7JLwP&~VOccZ=QS_~eFAc4T)Y zcr}uRAw2=lCebh$Knd^*9dKDVIabpyo;f-cq@fJ?xm|B`t;YP%$XCfVCc7ZK0!VBp zm@k!1V`Wqn2l%W!)^TiJ%OwvR3`_kW#4BidIq(^$l;ehXRLq3UOkLJJCX+|`!$GO4 z{!xdD=_;mu_d|nh0jY;vuXR&IcA>HAt?H`n%{tNb_w0A>_!!+?cr_Eb%Kw|)dDO|9 z&}{g&o+pRrr-1ZxBPPcxN_$|<&OsOgGk|if1-m;JOPvIx2F4+mxJ_dR9jzKdN&oqF1ab$yg^oc=90 zwt%jN4WHR)1|+9oC>+|P}zR_t+f|b_v=iUA`?6+&Sem3 zCig`T>h~5GT2vi`Cc%Q7b zg>mUSsg+b*PGr+9Yp>M+M8Z*TkiwkASI5P8E0)30`&oVII9}u+t z3atM)7kY6VO~6;7GqZ?TJ!^p{Je@c)jt?fbPk!vVeCaCiO}N7MSWQ@zI4q&>@!wq{ zkS)Jqd5Z40)*O`gqibc9nUXPyfBNw&IyIKNzAD!A2k6VTyB>J47P8I|En^I3SVhVi zf_bl4$2m$iT=u6}07{^43s0+G`NTkAm+D;K)JC)?2LR8gU1*^*1IOGt43!GQ@6+W= z%hbmZ03j-2XxJ4UGbB}h5Mxk2-f>rpr;pH2AkR;uH~^}s=h$3Kk7ldApCLv1?kY{U zskrR5wq`HYO<=YkUcMq;_Iariub+pF@sONXn7^385dWF@M)Fxe*hOzD+HmC-wOGWKh%wcIrE?6QZgOheWXsxTd>+6mq$17zQuni!tA- z^Jzp+2~>tubCo18?)8iV_stahkEo08N5{RD6BRa)7ki4ZyBa0b$~*Mk~Uk5QbYZIpI4^YYn$ln z+a<=`Mz~P8$g|1F4<~sl(wJjgB7s##3!#nOEJjB;;$m?znN^Q!J8lv@dD`416ITL> z-~SMPE^}`KpmU3<7$w`yTdCI;JynQ&8p^S*%~&swLD+u~9~t8ii=w-TCR^d2 zPNN<1I-WDZpZmC#SpUsJ#=HOmVy-#7-W2q}Ikc@xeSO>nTN$U+b4MZ{CrCT{*}eUb z3!+7UyHvQkm=pcoGdAPuXXzB#o_72d9Y=~kN4|xbc1QWrE|NT<)Zx9xK^Xelw3*IV z;42UA?BlYLnR3grH*PjgT8Er)+Rc z0+l3_ePuadqhT_c%e<1%viW#Ptu7+m-7jlo{hyhgD1J4fJk%G6iz}d0i-eN6NvgLbJcm#d?g`$6QXZJ>%h<8z1pVae&{B zw*!QIS$L-0PcRl)_o?q53qRo9g&m=)5#Rhg4DTntr`SAsCjwiiz%;?P^o|t*``o%(jnEZwkDL$aMJRDLa3%d22sv zQT$D+Gu7&4Sc7-_u3-3UG?H=PAmyr$W5Fu7L*IOU)X}^}K=0@{8T|p)Fe}@6Gwm

ERt<_ zjG6OHAZ@y}q5Sq|v8{0E`}Y<}(UuHRJ3(bX#w2Qei6`;Rg43y+_F1|``q^TR5n1JY zQRQN-Rua~^Qfx;qYKahNtDjT(sef>4{WiB@+%vI#wZnzZWTCg$2#R5D`!R{V!5nqP zw(zq@|9vQs;$IEtI;6`lAP{4oBwnsf(fAxPSOICZT;E+k_e2Y(^euWNEQ6!rW0orM zu+eadqObn5s5NiP?Df79{gARG>NuLu(d{zQcs6AVdA~0Z$EEK$GWxyPNOdO3c!fUe z?NUf6M?jw!wXks1_v-_a>?V$UVyY;U`(j^|BSz~|u{K70^2mK%Dq{3(hCqyM^oid^+LM z4P%=a!uXfv!zaN_h%P`_g!JMPg)g$+m2z)Tc_*ppRVwmI;;)adQ(v_hqGzw?v)j;P*BV66kUY)*By&yvz~v@pM9&#k-Ij3IiR-;FZXwUsNZZ$7F}*DWl980Wn3}cTnQt1vmZ)pL5pe=& z6>QBb9jIb8-xn$a(g#GIr?8{DKY*GS?#lXMcchCk@Zv72z9~)wGJXwP*G&TT_K%w_ zy(^xfPrPZzZS2(x$ZJ317h&=qEisL1WmPjAm+&Uhl@x%weeBJ*avOtk;eIY-45fn* z0+~8?n{4^R`g@f#6PLOl6r=%H&J3YBn4fCx2SM1P{LWA8Y14g%%?NALCA zOD=sEzRpJP5J#C_{^f$@n-dOj8B@kePYx)AH)WF zyRo@E0?U1Ql2@j~M}U*MUV3@d#%HjhpQoH`ut9{Q(b=g?hCkJEV%jW=~^6zMdV1I~ZoDP0$_+W09aEjNEY9h)?f)ptOW)7JxY(S=yAs%1vy z8kNDeN!MM4an0`+MX)|{)09N72>VzNIB+oheZK)BkzVDnAWz&6@1njpW)?8V-p@61 zzd%oGa7g+^mzvJbgd=hF5=<}0|huCdF z`ezSWbP`5dU^qRy(#FL|?v{Wpzx!GjLqy9%bd;=&yxGKhf(NWY(aX+D8hcCBw<>z-1U*pcdy!>OII*nF!2W_@c@YlxY!H9rr zdKXFPPoUFs`sYrqJ|$}L6}GBujHR;F6qG#4h2>i)zdAa+xFiGKd=B@jD@N9?c62g* zx|7W;)3ad#-6r=E-M+IB3WA#Uz{pmimbV-EUwpGgWV8K6dCQu0I#oy>8Atg`;TfN? zEzf>wWWG@SanD+2Y-7Y}e$Y2s#zOr0gP}~)#xtST%ilJXKOS$sxCYNT`8C2#x7GH| z*Fb}sZ!y3>AXT zY|#r2Wq}Y)0jFB&k1}5oj$TgLl^`s{q$LbXR@Byy(ThRA7-HwpP5&z#Kg~kO6S1Nk zn^wu3?C*)MNL;H@;r&%t_9*85z(T|4jX)-9SsQtt>!VTaapymFqX#B^4b`7dUv!{LD}i! zk@42ac}B?$QOBO*m!m5wRFR!l(u+#gewXV|Q94$Y#Xk%``lZvh^Zh5k z(yWb8cdnLKO}T#`LKA@hOg4BX(GoDLA-)~osFl;%GaDHh0mwDrO*8_VmbM<{ zA{L6yRr2eqTAk3|Mnvv~7*8Zas?F9WxSZjIKq`aqRJedZPX#l-nt@#7pv|T&)pwQ4 zl$M4oD(UNeT&QskDr_~Ef)RJK`)ucHZ)7h&^=8z{k%g=weiv?+b8B$g;6dv{mp32` zi~NX58xOg>@>(^%$HnB#(Bn-qx%|3U)JuL<`ZhcO0N_*~FW?thvc1t_`6XBlmF*=P z3Q9of-X`&fKj8v?mfAQDOh(YI6uF{HI76UkC0{7mG>Oi#?(O*yGt7TDw&yagB1*T) zc`p*p^dqJ8eYP?_<$RfQ!x^|d$QILBLx{c?ghs)osk@nz{KdEx2(DAdnm(6GR|<}v+eib(Xi@m6@vHxTKM2S7FGSUXBlomW#$G9g_Tq8t z`THf~Q!zu_K*gN>?exqxUu%1a_vo%K%(VW)cYQ@0(_wsVWMIx+pYaKm+mpZR%k+I_ z{uKWWyu-}tX{n67j57#Eu%mRm*Z;jG5OcY4+rF?xXPkBBZguZtwwlpV2;e~LF*5o^ zVWE`~;IW61gVgiS9iaqT z{Vqz%j>zfiGq&HAEa#WBZ^LTCmXD&^&Fhwo&U|Kc{^U7ytP(Idjkbnmi^}#bveZ7= z@Qn;)o61#x>M}p=E*-!*U9fe_e^|dzPkU)dk9oF9?wfaON-EoC>xI=ns4TRROStS* z>YrQmG=&I?UJL~p3=EVg$;ZpeL3P^~N)I9$mlml+URruj)s^y{Y<@s#yuXIJVDp5` zD{Z>=tn_KLj{WxhZWu(XtSY0c+rw_PH<+X06@P63dg zPo&b1a{(?@qN(S@s}}M3Eb;iJ8n+^dx6EuNrol%t6N#{r<}cY7=8q@{J927V-opGz z4ylqz>gh*?9VWHQc?A87;;bz#!Fg9Z?20c@yU?O{5`;&nm8HFDlZL zD?ZMYvGR~CZ&;XiGOscB_doc8e^$^&TrstE#~`&!=vu)Mqo6rWDxOKu*rD$S*%m@A zi2m|}vdQYWBeRp4uyMd~(>;%&%s`$1(JAvl*gl4l^Ap(|M9JC6(WXg$;j5>OzVY1G zbGxK;ncWM8X#V<$PM@z9zepopl|3+dal0O2re00;Z%C4o=X^3BJh0|lUk6^BopS+* zVvh%odrkH-kLQc$= zpJ|p*9zl~jhycOpZ`#_fhkI)VL_c4|1kE3jM-09db|P_0@HhDG>2s{?nQpN)P+osB zwuMNe{LQ_mXeBxovGKu^rHm)=eU(69r?-ef4GHvl<}BxavG88b*g}n0&4g6&&*<3_~_U*}& zUI6xy=_*2TQZr;uIuqzBn4aJ(3IyfmEnp`D%+LBs#|B+rdJ2uJ@tAb3KM*?D-_G|e z9y_T0veJX#G3+svQS{McnEpq_q7rK(6ee9=5c0$XEaWaUya$=!<<(s zUfeQ4mCA;t+-0}@;oOIi4=qKazRI*KLIo7Kv8KvCVu7%_I->owm5|w@QcfKuJjNaR zppiQ;F=Vijr2>t)X1Q|vmqo?j#YfTOC1nH#6Z1W;+tHH&T=a4+W6d<*o%cOz=Y6}=MTwBue+ecqTAWjAlbt?Kl2o7J1|s_AMZIOwwHk2{hX0bfmD zja-@=(r1BaVPXK+3x>ZwVqLoi&@Br_jPyrVhIeO7^xz@MK2?it5Zb}_%k$K0x9{nmfoN- z;49pqZ~G>EF)L20JoWUj;`BzX$qi;a*BqG|zenPGyY!J0CM@uD%@oRY^fF26d9#DV z4tmu-cWdrL55_wF+u&E3dH#iJRQGhZZ1x$A)fN@EL8+EHs~sBiR~N`+eUgqg2v?%b z)=wNm<*#=Pwf!Y?XX8=UD=)Ikp!jck&!@6ZAKNP~TEgRA{1OOKd4mFLU;5BOcJimZ zos>MC!?)YcS1DkaP&?YYmg-cq0KIF6+AplP#Ndlh&i9INO4&1C3N|M-HSz6BzUvje z6#PyqZ*N7#E1|l~i{Um-^w#^Tb9QDKUL4!Qr@XsEAJq3D=!tNn>drgve{TB_g!%qW z^oZS?)+4-xkqCMv%iKsvvc^Z}nvHnYJNxv0DcsBvhK{?jWB`b z>i+RCUv+NRP--BKvRFfvDk>Y4y8m^b!GEi1+3hujOPL1|El^o=3B%&HFm79T6j>!C zf{N7849Pkp6-uU6PS|Dc%1t{xmCg&%?qx;zu}~!v?i`dT~kL9H;KaNo-3ByXMF9AonY3Gwh8scg=uQ38`QCNI84h1IF}7nji%A%5Hc z(w@lO1wXzF8!{vi0#5%BC7EV-VeI#pZAYbd%M!yfFEbxJCk}4TMzIQ-pgSU1R`W^0 z)v>lFZN)INklh`TI}t^1xP9sJj*kNZElbA zR7f?}(9mf-8qf)r5iS==sZf?Mbdd~rZsCI-I|)GJCRvwn79+P6z$qU)6yEVup{gT!G{(s#nb!Gm zzpq<6iu;nd=2h&DrlrAE(6Llwm_K8YmfK=Z_Gh^hBB_WDldR8^y(_AuBk4b#Rf6E` zE;TK7v&p&JfAXxvJt|a(_2!13Z?v22?qB(7*SLC%&ZMfQ8qD(X-9D=MNV2mA|FJ>U zl+!F#uyrBiT=^;Xe>`IxSnNynztMj4njVVbOp9%O6|MPnf%$e?qC(F7<)eE3TuvF6 z^K6Tg^{r;$=X8<|-w525OzK=ZLGjtq&-pX&`wRN3Oln_ItnM~0_N)k(=26D~1TnNjE3>sc|y5f~N%pQ=l&p@{p0uti+QUx5sP4dc~Bn&g{g$txq zWJ;N0x?2gqi5>VSoDK6C9O2&Bbl(4Y0N8|}24b|Y6_YSv-qr)VeMCB203MTV5>1YC z!GqQQaMjOU-Y9T@8kixF!mY&igy=&e`JJalm!UKLTX(*%*O)q-0<)RE)U^Q+{|d(OVcxO4I?xd|4ezF>FDh~ z5eg|6S$cFCe$#fkyvWJ(uz+14H=KQHEdGIm&Z63(mj`O6B?IM1_fhmkFZ%xHt*k%} z=ylHxcvRa0zZ|JBn%T&pp}cOav&`jpE}d%J3TMKIv`qx2@K9b^*3GiE+=btVei`ql z`^4_JG76c~24(!yXT|;U>~lWB%AFR*J4)`HD$zM?3+7#_tqA4m_uHNw!_pf`7ncot zd)kyf;@jvZ6}WqjlT8k#=32d6k1sTitUJEvb3 zk8M`gVyIu__t>M+na#ZPeoNmsI^K{fKG;pJFe93=>G{QBIlB`oPiIeMfY|K0rrLNt zVR0E~I8`Ct8Ar`^tl*slYK`nt8&D^EL;kr6l!zGrA^Ffqok#IhlK6tUxmLkGj}S}k zAvF8P&dI^El2?42fiW@AHJh0`5523Y_>#-w4;$uu_{xR%XoPg}NQ2RX?JhJ`&8P|U zKi*8{d9y<|@i)IT7OY})8ALd(i`cb#NW>oV^vbxug~h19bsP_Rk6UeR-xoA9$ZIag z#jJz7X4Sqk`(fIXwJy(-d38AxgQ4u{8|8NePMkqOe3`)yhGZ~0@1EFLibb2|sXY5Q zK@My9(^dR=8fvq`-EU6Ld08PXI}QE+SE(bs$OECaC-q?!UYOTFYq+a`h3{ z%!)$}*)%WGQTt|G2`YtvIUSeNjtRBK86#-n*{j}+*u0Cjm|JoXr z<5rZhstC#C&sb5>PvQP!Zt9~n#UeyPx2ZZvoYT-JZtpF*eR->RB60pH=>~B{Z4xc! zZb~sM<=Fr|t+qW-;3v2=pdpb_>`}s3PVJ}RZq;h9X17_mm**1ckSJ2jcR)6!H zTV2$a&Z2v=wZ4?YVm_KlQ5`)KOzup zCO@g_A2$h4*$u3enxLv9VlkirNQsP4GH7~qu?kss=VYw@Q_f~o9vRicE$bZ57~=Kt(6 zfSjg4B!^}wM;|hk@T$YiaaNtW6F>yLrPDZR8V|7Wy#Bn@_Zu%I{4I~T=D@2@;S&~( z8868Mv2jl6<%(p` z9KKr0Xy7IqJ+>P|hAwbs3l(jtK;Y@3UuZ_d8E7cVukNI&b|j!YW- z6c3pBSKj#A6L`tex^HatK6tLa^^G0@bQTuRDG8agX>q+6xZqUT za?o|UQXHlH?eE(^1qi)45E}#avuAIjqFnth%Q7o1`*A$~T-`g?PL~Ef*3$V_m3)E9 zWVfm!zZlSaCR;tWKRN22{VM|v{4rw|Tn75x#oUGiX4wP~vpAT(siy!JqJLu+bQuBA zWvNfH0Hnx)TH3GCw|d0%Z>@K|#)AhBCb?_dQ+_Bqeyxm zK+&7wNX`Q{07B2f3X%1u9e(GR8n2Y1|I&$+e{2^7mt&LMy|0;dd-*wd#H1L=Z-Ted zUgrYImrCf+4JieMGSKxLp~*A?q@pK~^>}clO9Pq3=|dp$h(T(5ZZ8I9caV6OKR;c9 zQ*RHOfx5)=?5&KOE&t!D+quk%8oxr_Ts`Mm)(CizF z*TsRwkGTeRJW7EOa2+;9JJ$~#{^#WYDl`eC0zd(TJPy1zfxB(LJ5&9z89*I*@JNmr z|F>E*vzy;vDhMvGufHHcMe5Zf4#)}O?I!AN2bJKW{91FV|K^7G6B|-`dzm?c93z!x zd^$-ODCsZD5p&7TtlWR}w?OxrJcNUZsRyBI;{tBK%(%tqwQiMynDe5*|DLECkli>4 z=jy;|cc**8FslI1%5yX3IAEV&?B7@~1NBqreXnwj?fHtoM;)#YRylkr7xFBxmj$;+ z8nthnC*@P#-7Obb|4ALFP*j@L^Y1_6HEt>mygtkNe{P4q=lt|^%rorG2>svZLT5(e z7Qp0mf;nCOdyn9~`)_zaqzU{PoD6w^f6oIB4n&Gook8#e{2BR@f`gIg;93Gk!{xub z`S)kN5C5I?-@(Xpy!QO>r2h{7dyd`zPWtcQzvq~$2512tpw)1=z4^1n#^nzZC&-)p z*5U7iQv6rU1hdfqq08`tt5f7V4MbFSkz%y$GINvVk7XCoCRS5u+EYVJU)$m*7#IjB@k0HwHR}hLk|f z84grA)#Yv?vGMWg>zzPDa~ZRMeD+pgCz=mVeZJmv^qQD_K38wEvA)hSNW;M3axcDw zYW`sWrm)z$M=ocr>C9OLu^}K-~hU(;Gf4 z65(}FpKKs+=|L+|l;)u3%-e7B%EER>1pp)$30A8U3+w9 zJ!frabY>VaIQIRb6GRR&nJ?}C$zT!8c^Wp5HZ+0BbO*&?U-ySWfLPZVy%=b_9K;1h z-)Ih-ne#4s_?xJGQs zTxv8RiSK6P!K@YnA+?sL zA9Cvfb|%jI_xHz`__Qy~^W30RyuwF{od@ryrx(JC#^fgCLOH^;89yv*3Tnz9WW|9G zCd%o1dltZ@GNt3BV+FgF0A-f+Jv^vGQlu7NX&+CpF1;fe@inA<#bw@WplOI?Fvq*& zp5o>}+XJNe0t%LN$7gs=`%B2jeLRimn#TC#P-O1dyftdL>$N|t zHG+O?pM|mK^K0=^^EP+dwI)d!ypomjVHi^w7Bva)8zr1T3K|S(G77tqO$)~miP$8d z*X;nFY%u_UO#$nrTBefD3?we0Z2Q4LzjHG0SUV~8yPd`1;QWh*Y#BN(d8t0Fcq}kZ zn&ACp3vfuI*gL!h1Xm<$5UIhrX2e-=o^Ol>bU=mL_9GksTGMF?HOdgHwvhUSE9Z-W zN6lJr^7D^ z=cn!f`MzB#F%Z@z0nDRvbaXTUWD<`5S*gUXfkp5ld`JHkb%)jOT^fX--+gc z4Vx94(g(1>kew{M{N!=Irn)AxlIOGZovnb{r(32%_N2k~F&Ci`cY%$mlFY2wC!weF z;M$I{On{m|lV%)>mHxRh*T|xU@g4ZVBzr-C0?_7Z%bU0CYPABw>jp-jCONZge+*fw z=6*B@`v%%uEiEl~K5o6&tYWOpFaW$r=SZ^gtz$sQ0wJIuFp3Ut2DUv-zIrIv@)_qpz~OmanP8~;X`PEazBk~fst+`uAZ50$|}#s)R3T0h=nwv zivxEA-d-mS-ClNIcq_W`KQjf%@l$&KEMzaT$?9CRh|S>ZR;GguoQlaYo!xkDVn|0t1Mg zl@8tx$Y|LQG=0Vn~yB z2&alH3n{c~O{Gz|5Q#ZbC34{bkbB@sDgc{QWdpl|7FO_%eu3Jzka#>tOr!WAmO&=a zL)nh!W>GWs`lV5&6EKJc{O&ksrMtnW5mn=2nO^@5q?*?4bkc151iKUd3UjVJlnRKI zFao5kETdi1mp4yYpE^MYtgSg4adIa>XgMlIUiV3e;R5|4Qaw*{I6w&&zw-MXZM+C0 z`pumn+8fMwFF6tf_c3m<&3i=QU44;8xTG9OKQbX~uk=HH;&o9H5;86@;{}j0AF!!r zlmZ!UDJs!aFYpZS-Vk?^oAnOfiZj?_#;teX)Y11@&9Wc$yY*=>f3U`ez2eepwAacq zwMZjau#|=dy$8%)A!^8Nv?nD>cL1wrV@F3vl}W36z0G1%|Nl$+=URo!o0)SOZi^Q zv;?)JcWXFRnG5@<-yQK_Qz1fL7wh_E!vpllRDmY+eMn^zegfsHv4MdB3VC=4%KeA2 z^dws(c(mf0Batd`ClJBsiS3#W5R>5Rg=K5m%3O$`?lf>n^N~2wXv3nHR=+|HX%=iY z?+HN9i-Shh(sR;iLb1G{USm(_0$%l0bf23KLFK{|%TYxjQegLlc@aC#D4B6U2Cr43 zzF_bi@;m>6H;F#6j<9SP?}aZZvE97Q9DXFT=2;)4U% z!!AG@Nh#|6@$F5lhOF0*v`^^hh#oVv8-WiYJ2C;XXR(uLrJ=RK0I*To5VJ|?{IRv$ z9acBNlo%D2- zgc>CfP3_?gl~d%$#D^4yA>yJZANAG0gtV-BlVkJ&CgbMdqyQp1>j>K`0(_&7UJCkfZC}OrWhP45-92V^${$S|3 zGJ5dYoLzriPc-BziGWKAj-JdvMSVTYr5K?P+_Jly`iu%s8}u<5Y!3tvd?3u8^h8Af zu86UWehV>!SBeBMNIpN%JMcNM34y}=AY!v()PW&tfZ50v48b@#qUgR8jK*;1 z7X3C-ots${peIvm03ZEm1y|DjK<7Z!5HmR8=V~k-BkD(#kQIYu3Irvys@IRN$$x;M zCXmtySGVNJrxq^GM{=D}vaaA0Y7il`*VK#Avz>xzF!G$pEs)sX(^pldru)(w2;$ zen%Qp4&P5}XOfXl;q}fvJO&w)sUFSM!!)W~NC+0_LaRx>Pl1oSfOdD)d-Jk}0N9 zLmW^x%Z3obN-?R?GVdr~8~?>jRi_=AA^nt*$(kEiF}b^qalt zO?Q|VbCrCxI1hv8v4}olRa*|>rZb3%_%Txs!aaIAHR}Yki26(cd{$+QWVG(PE92}m z2d1CnP8}mboDR!T*a+4(a#j>)=fyD)1TDQE6-yOO#l1Js*~{guk#$`_{}(R`4;J`j zeIa{++sBXQDe(1#gHh-%2%f>?Oal%d96q*BoDhWhZA6dXWxhK!D$tD!KL*IJ+-!FE z1&Zzv$tLXPRqY){1yAg5&`k$E3X!A?>Z@an1sKpCEAU~{HXFRyrf%lBkWet zf(jqfL8psx!gtV!=4Tqhtjj|whmp>Zcv5oM-#O9N<8I|%tYx&Ct4S1pPS``VG8x>i z1W6|$VhwjA8bc*qHH>?-24RLpXo^0Mb@PA~L7s*PgT6r6Np>gCTt5v+jBxcGu86NR zdW0d}X0j+~aJp$&fxKb@6Sv3#Wzfst;awsDjEFmkVw-TRn>%|em?-!MbvxEQGux=ST>>oQX|h`Q4rlIfg+apWOUe69kei7+A{>3mMCS>@t zD>7C0YlU?7Ri&~+w)){YmDWv;g1mV zZrmpMMof~F$xC5)<2z{tEqpi5Hz|Z1G(|d)e_@GxgX`g$gh7bG5D{o0j(>^jx8gBL z6u}T?77-kQa$f*}G1T|Pj)TAW5r=F(W8c{T%8u1wS4u8897bK`as(C%YPOv zvZIUk8Uh)_~ zZ`*7{Z*tHmj1An>G04#un{c2d;H_`$Jg}gfvY-)1p=CJ=Sq+B0N}A?Cn<_b4L&*M` zb-K3!B-Vf9Db|jE(9OHeAog#ol{*D4*r5_V=iQN(bdv7yV3obTBz!Ffp2*({)M)e* z@_l}#!o}xG)TNPKLQ{;go@*;@D+>p*2>Jx7ps%xh)hN!cCJ)if5`VtKI|{du7{Oh} zE{GQ@3^<5}2S*YN64GU$;An~FvTww$i;m-+Wlm=nn}|1yW6pl=8@UwklS$@p`V4kU zBjNQVQBXx2f!->C2`OhjG{$%Iqj5CXZg3&v(a(m70j_xO7Tu*(75x^)XPd#E#f(K` zBW%;5S&`bxzb{VYf?k@IQO%gN_4=#N(dbLye2&0}Z7Q zjhpo+76htRY1n&*6bqC=_zv-Z%v(3T;Y@f(_m1GT%1;oYXR(2d3@jQ}# z{{;u= zgebLq*dikc`~kCF_ZS#~1rW=%)c`Aci94t#7qW^o7^{PlQG49^-|*Gn;2H>++0#ra zaS_^R;=rDr*&>5?pdHRVe<846M>^fYms(kZQfA0sKl$rxErcLBHQN7TbANvh?hY_x zvO7ROD6VD@=^e{g3msD0v}IJV8E9j4pLYd)0rQQ=Ds`u)rLp@u{|*>f9QXTuHXA<= zKO5a$mtTnb8#WnB#q z_I{#HPjMD2s=ch3Ccmk~*sJ>HKwtB|XpHH2s?a_GTr6Sj{ZFh%Yhs;D@a8(1-6QMB zo=0mhUWSEjI#mhKawp7*2RSa-ytlY*c3u)PbNd_oZAO^{M74f9qc{F@P8l+!@uJWh zT3g;gXJJ}~1ay}s(S*ZC*P12J7opb9AQ67|1lcnP#H{mQ1)hF0K(?Iu0&pdYNcIn+ zpXX*cm3>rf_<&A8oU*i`37iGE9(hfm14c@7b8~>j6%xM#dcS1+`rg64&D@qqcHP~P zMFZw;#7QpvLxiMl^)H)9DjCunw4jYhhM=U{-gSR#WKgMgq(rN+$qC%n9_U!K!BT&L zu?50Jr5)XMG(QJZ4~pJ_bjjSmaBQn4*wm-sEY{noUTRtZGWm{6@O4hWmh&Zo-AO8Z6+{~nPld-%ho!K*Eh3%NF<{? z1@4P;vNjm8&Tvl0-T5k!Uu#|UTU1n)+TXm0^b-iv1)eAC;R8R_t|R37bcB8-dvnp3 zJ1-cX=(;|N$QX<{Lo=&)`C6I$iB~%dq$gv;TY3Jr6B!VF3mO5O?A3N<>qsWJhLuOi zQsz>?Cxfnkig~CY$s(aUlL}Jnzkc0>$cA+5(Ul`-&+A z99XN`i#2Qu)6pDiMHQWLHNgs39+V?Mzw%x_>^vNx5h5ZvkNc4iz9k%3B88+@et!PR zS}gzfZ3BQ_Om^J*V66*)N~zwx;`&g61WG}LEt0BVKmbee3^cfL*RlFecw^auM1p-X3w|b*!ga{-S9^Ydu4C#X(I|MFC;82y#Igy5 z{xh+*F$FX$)aj}wCO|^L=4D}tl-&K((2=sPCQ9gzDAMm4}>+BX>xpkh`8v@KzNJ^3h73wPGk>LBYR~BMo3T z$Kq$vrL5-*fxz)V;$rWsCeUqGC@3aY&_fhKko+FMp0wJ<8P!1F;SxQxPNL-Rb{0K z6opv)mp?%=51C+T{|cHeS=$LQE8-|NVQYq0d_LHSYLNd6d>dkg!3Fr0EI8ItN$=TD z$kBiH5#<2+DVEj4BM?;{ahyKw4{O5J$YatDH#m_>6$X_{oP=4OOA+U362^Z%tDwXn zqvsudr|w+H-;toeCkj{DV*Std1^M;nUW2R&_yjWU>#gx5BU7{uXn-@ z|1kB=ae4p$A8)Q@+ge<<>q^VEy==SHRjpdKy@lmkTD5H3wypEt=leV7cK)uetM~hb zr|*xUBE>9TiuEzKl1-q+Ljc_v!N?sR`H=WW#Wa9nhK*u{iF_WO1eVXmk{k-zWK7xE zIL|whp^o48j>-LOCH=R@>1QB>_aLh#EkT?C&fbRx2l10WWcI-$go=IPm_5(t~97k%hbsnw`XGb<7Y( zG4f@C4?ZBVhK0*fN$^n#c16q+ZV>tb=Vh-rmg3+0g8+IT&I!6R4>goDA7t!+|K9@P z^kTE;zn>R;q#GWXczY!z7@~)a0`z0}-RS;9G8VT0*dYch2ukSi3+r|UE4SMtjbKW* zPb4-;62KQ8?vr(t9IExH^?NWM++2nNuOw?~un@k?JMlNC%LDq;-5K-s)XKlos2TKR zE;bpFq5{!u{z94YHM@r5VT7aO^nl50xlZC!z2msmUU7?F3n7^MtW)8-XKW(t`!pxZ z&9~L1mkU?UZg(tyU0*sbq8+)j4?Gnmb?}))-MhQ486*31xUVnAxxf$};;5vd%~d1O z_O|P`?S!8->n^==%#%Fg_g&!Qgwy5}k?$Y2%3R$}wt6~2|I+Ss=-F*ARLqgWR`0@` zC&UW(LNZ+ZVWTF4+~u4e%UGjYaKf=Mei}$O*SR(teLDDHX1wO@l4&Y^hdbC%`n% z)&>2koP)<x7C4#C0N3x6mqCn&bsRn2KcG=wc|R3Y@R46 zIhozVsEh92{Y)bAF@2}Bowuo@R!?HUhGRzrQP69ZPo}N91nRV&pY#?ve0ojjV_KF| zYg77_%m0NdTL^!M|J$q4_GRngRHe~~6~7r_#|wgcM$P!ya%1Grv8iU7Wt)kSSI?v!N9MSsp2vgaPc z1NR27GaenA#oPD%DehAno)ww5ak~Oo%vWI#@pC#&h|p2o1;p!KVv)3UmLqigQ0$jO zHWFvXXT_*MjSwmePe=fph`)06mYL@4O?cL{X{9zpxE(j;@1sM9n61ZJvGo{PXJxT( z8ii0}&)vYs!ey&upxR#wEaz0=M@Z0xLOfBma(8U&{o_O!i4h0#!dg%d4=@vN-)_$} zy3NG?$X>bU^4GD|%>6Zy73d`3+=rkHge!O!4aVK6kNw++7JPY(>-;DuYLqE|{ zBdi!ID#*3vvM)IN5>jNyNqnnuC4Uo3sLZ@YSN*3L{FiA9>V^~EZ$nx(AXt%408xdx zaN@5kb{EdDE@`-=eVZbZrbC`#=>EmJd6cDBU@6#T; z+s0~}KXh7)9a#IJh6xRBi}sU#pPUsrkU+u6ASdLxyZjXA=xw;4)ll(AY{0gGO$$c& zw`Is&hXQ@*MVX8L#OUK>=cfwa-4zW@>n0R-&WcI}hB<%Ibq0}+681bH)AYGM6SK@M42%sHUboIe{}Y2!G5?;E=) z<#Vv@W$9iwA$pkS5;)(*2tL4k?>*-_fxjdY9{UHUN+SVr;P$5bQ}|Wd2=si%i-mve zzZQCEO}kyd_Fp-uCCwzn~{G_GVWn4^x2!*>sf6O{4 za4me0&mbCF_NPg%RBu_jxnitV+cZrs)5#33AF$=9+(C>*m?_ZEYwiwI*tUeZI8Wv@ zP58J!8GgihB+uck$MbG5Q)WT-L#J8@g``uT!ID#eC~l)Y!Q<`fRyExD1pk-gP0A71 zp%jVFw?Y6kRSW$PD=5*KH*a|} z-;)cz{8(JaXAuzgAi5JQ^Wrv-51_tUA530Q!2)6iK1#Uj^NqtYi#+_1!9SHQg(a-J zD~##qF>_DVo1C>S3maOfz9Gzc!Igk-wTco~0{@C%R%@?UoQfiT=B0YRjpmc`4<9L9 z9G+9%)_g6(I^C)f^W!>M^Y)$qeEI1tDDd6`$v*Thv|`(Zat{h|mQzX;KubtQP#!Vq3==X!EG7gMPW>NfJZ*vv`*;%{1fT!GyocEaHh z(Y@Sk{6oEdj-5Yw5ox`?6#S(SEVTs9vlo7oLjok5vU_5Ja@2Q{@8vk7`9I z!uTD)A{s)JOnfXZx9zl9(fcizxc}AJ;zUkC_r8#4{qeh@-yv6Uy~ZXy!I_WpCT23V zzCmcVcqBU)HGe~juY^Q@>omu>uu?JOT=MoIn=$Pph=g0iJo?&n)al=S1`Vo&1bx`2 z5AoIg-<@Wr+`B#b6BAflv{Yu%87$E-#p2}WG1kKAxg3FS;G)ce1+Zr3Tu(&7_F~Dg z^=0OoinKMu^XC_%5>uf|?*<}|ly!qA#bg=fVN{93%C<+|PRyBtSK|zRk%X03kwVqo zV^(r^6S4=CeGJ)mpCy=d-sw=Mqu z;W!BbwO-*dh`HmJ>DCj;SFId!aaG0fX`S2yyK!6)-J7fsC<<4e{A#5NjZ(=HTVMIY1}?#XSmFqP)oIcY`tEgWCf!) z$>A4Im71y9W_S*80r%SXTPI2|=u}9q{_syOlu$9_Oukb%-^uw`JfmV%Ze7o0)tE>ZJ} zh```;F{`s_Qyc;>)JeB_-XX1Dyj=VCrrhf#VW;b=rE7@?r}v16PF~Las2C2SGd_2z zX;49Yq7Z~`FaU*a)cY?W7`UG=Rr!Qb!a%<62OAv;8BoBpx^pT$`F!$|O&?CgD4L?& z@)Z@key>7QZ)#s;n3aG(;vZ@*Ip1jD2F^|SImWG+c}4h)SFwJ;HxnWgi7eAxzRJy^Q#@aBN`>S_$UL4TTrg#e^xkkXv5CnI zpiHdw)yYBNtQdfaEvo~dy7GtH%ve2Rf?B zdnPvFgAmX|YwyWgz-wO5{u#enKff^K?D#W}FRNF5mN@)wDpQTvvp792C>PrQd-Cd! z5VACl&Bj6T-nN`b8qkGX$_I-+dMBjlmGQx>5sib7K!1_YnMMLuzbgo{^^wug(~!wy zSc^T9<=pqHxgWnt|NTtkGJaKAC_w-|*lQWMnac@7KlEk3ZYxa#AUt->*s?hK4A*F> z^k8c`=*i#XfyrRSOLi`s$aB;vs|^AX0g!lSS=Qx>-c>9=!>*X*xAi9!tvO#&?511B zMaO$c4)Zkol@U>AbSH?@X1jj?EdV9-j-!B-go{2iemQF} zXg}8%9Q$_>0+XuMz^GBvQ(M^df9E6o(h)cv+vZ5bH{F0Y|#O8KWOElVPt1d2>Yv zFoqpak{0s3V8<)>+;{QF*1!@A$lMQQ*eq%Wq=nKJXA8EN?WyN#gu;VrQxAQ_rBGz4 z8=t2Wf$JdOBvJFTlFbgbqi*N92zJxyrAYiJ7wYx1aGu|)9|lZuc{e*%43nK z&yr2x`wcq z+@q|K>X7VS*VZjw^J<1iM6p@=?N9v#NB%QoXrTQa8(!Pz0UIO>=_p&|erx-7A;LY6 z+w^yUka84XivqqIeziF5sN^+`?>2xON8RRS#9U{7e*WwJvIIxww|waL0`wH3ifEppQ3{4 zx{*O*m}1@fqd(krGlLO+IpqXfR{@&Wl%Pplt`j;@?Cxx7rk{pQ*(3DgRxZrX8g7aP z3!Zz7+DT79={kw!y}({tLD+3tEMbiO4mgxMr~uylm;KW9C2P<28{~)mS_p9LXZ4;X zVV5QG2cJR`7z8~%$2i=0x8UG|W+2JJlxr52>H}NJ_b$`(p-U3Pdvn_MSPohwgyOl) za8J#ztxY}}o;Evh2Nka*$@(Cq7c_-BG0^Vdd{T+09YfhB+GNz-NT=*ke?-Jd^dw-u ze*Y$=bf0Jln{)B`=|$_ue7yk9Q$1%(?KT8N%;qfw0VsdV(~mK^!Rzp-E$%nac*={G z_(M)*K46+YmIRTO(iD?CeD%mxhYe2HOhwS!g#{rIXuwxBOeyqZ<%Jt@JmTq227Gtu z^ce*5S`T4TOWsu8(sU?F515Y2eRI{q27h2}V|Ql8j&rC<20TURsD-g7 zli6{O{{b3n&vgcD#$~es7N{hdzpkbqariwXds^Xi%eXngP{@0zlJDj2{DPK8A#j|{ z2P(`Tm$t?sG{8SPGxqHXxmHst7kQdtPb`gAMGHVB<>>q6Nb?pWqaZhgX#xm!2=XMn zcz0l58HlQ{(i-eW>US9^TAWVP*P= zZ$@K6Kay($J~)a5F&qfdL3$fSeLeI?o#9cKcO1QFg|2u+?n|=yAj+YZ+@TfMYQ;iw6m@` zZ2@@g>W3;GQ1K?tdS_Fnxu;uk$Yh9QTs98P4_cgQL8;Dl;$>2+9-L0!h&(r*WV3r7S1C+bRwLE|KtLqLk9L z*}Eyj^$5*}naX$MD~rG*B3JR3y@7IJ?B*w?o@JlQa|jl}-Ej6Bk~4+?;cx!CL6dS8 zeBp~55|AK93AFD@H_r6O$YVqjmNL0_f`TJ1$l!%}@~aq=egL&t@}&F?EVJCZFREu+9pOJ|GN6|dnVk3%kfyb+y)aq>1W5elz??IHUYATxM#APW({X~yJ_ zhWs{k!Sg^inSG@nX!S`)!q(%J+6LVKFA(gvpnAQjNP0v`X!js7DCbA}%y(xI9>`tU zNpm~1kcyFwz7CN)bZnw+Rhczl6#I3$TSTuadig0P2@L{<>`Rna+7QEXQI88E*lC1! zzxT6N>6V3UI{)j5#80-Qlpo5{1s1$y>9_gTrRb79YK!0(08p3XgXWwM=$a}P1g6{lb2oW$XGkNq>$vP zui|$@CDtBMY=N0yDD%BKy#+iBu?Z9@01|=9C%m&u--ay~4XeL@*<|&IMCAMMkOCB_ zht_+jgLVX^P;7g}2KFAg2F)%M5SmBB9WplCiE!WaVhD$v(Iy+S=;pWM@Op7!c#yTp zsAUbGJAE`53SpP3x#pVcuiZ|59?G&>x=xs5T{Hlae!PoNpjDt>`QPPg$2B7jnut!W z5=-2Z{4CUvGKRtY{WZrC@W0jJDE$P+mW0nZ?#K?*fBQI=zL2HnnHK+u_ni>dt$Vh* z%)jvWmu4zpLk3+q#v>~>3qK$kVb@%S0}U!+iXK0d%*d<`bHAewV|ly{43DX%f4(i} zZ;Ho2F1;v!mHJfB!pNOg#WnOd(5!J~813-e%P&+52pFOO+&7gM>eTsJ-^?I`%qlbM z)wT7%`(^?UI#eQRIXr6Dej9B?Z1wiBXrMJ@(D6f1Okx6Tc&6=3mNFq~_fVz2*eUCWtYtT@2 za?iJDB8>pDTr_NLKM9s?rf5`@0P&B2GVbFEI=Z+wgz~{Hrr%AAApoD?Jq({1(3NO1@CO8^|#IW8l@wgw%QOb-P5Lm=;1Ia zeyoe-^$B_9J}B>)JWK%G&oYC!OZC+2&~p;5QKg|^UuX!f0FSbr#WHu1FAKGC4;BxP z#c!=NLWbWWV=e0LLT$RUSQpV=DmM2wgLdyX18oh0C^n+JY&UDN9SXZAieg>bN!(s{&)e>_rmGH8`wAzXT(Km3y< zFvk@1!MAUXNxbJC)yp>?;Y0g`><%a4Il?WNBhYIuED&iOpI56}O4 zi%A!e(7PhwJ{qQkW4N!R`yC|0Q6j^4MI@OA!u-03Q0I-u#IKBxh0wQ1+@RR{>8*CA)9+@BJo;-vMi%!(+JrD}%SsFYWRp^<;n3%PFoy z|1sa28wlOKCcPIb{^W(+5tdAhf$WI}_V#?i^H$x{#x?_@BY|#>AOs_+UmgxjquxOX z5*%etU(&g=h5>akZ8*vmu;!hKwlP%Jg*Tx9V;T@@N}REU$x0gHxXT;*PJHhw)M=$) z|1c%+gzqgyyoac&zDM94Mp85@5SSD-5%G_0-(M53o}1+IJ$o=LSs!C~x@~uG2hTTy z(Eue~8~@jnAOJUy)O9D7eGjRkL9F|{$if-Yg^J_%R4`lbFdhiw z!wOmBH(P?qz2+0fyIS4$!thwPUGL#AMcCK*e@W24|5q;xkIiB6M9@L4f1CP0wl8?% zC09>{@!zJ4N9MjOPPWATFLQu-D zaVbXW?)@*D1Oz&&HP~(-Dl7!tG*npIU4Hf_t6;%8xOh265%FpG%A$nyuHny=lzqUiFq;wU@*Q;oS z-(X&*OAwGIK5vwY_+%1HITK9r$!B^V$Z4}P=(iziJsA&@ZG@UJs@DFv8QQ3)9gQoQ zpq9_+96%j;daU7WyVd)DjUE2~j2%!Y1WS>N=Maz+_K9+S4MsTUtOX1=k||m3Okf=7 z2ivzD64!*+DU>t1->S*Fm5X>9CHpynlKd$(%N>c<{DyERq#^WBTGVlA%wB9xSr4my zmGGajKLpf0|5=2>@|uO@kGyDF=o5GfqgU-YgkZ!P5uLFuL-Krbh$fT5Mw~`reVMDB zMtCZ+mZYZ|;{Wm^7wEy@hafw|@>LD*LQE}_H_^Z2J0mLN1Q^IpSW>p=tGXL4IzuQT zQ7(6+zeShO>j@4o^R1UJrcCrRr zG`dUiGvUa;I(>!K1{M1$05oSgN!^`MQ6ML%URHF2?N^{Q)X3O_sv>>%nm1J#qFFVW zz^H-qL~evpu70g$mtevT4GzI%nHI0MhXNgleRcc=L3wX}1&8jhn*wCYqZphGA=08| zGt?^KvI+782jj%5y<78<<)DkTaEIDSl2Am$TnI$NF4(&V2G&K+h*>)2s7fDHXLaq( z%TU7CT}U-u&CHz*D+q)Y7i?qy3;RBRE!og7|AqK!!_AQ7=oI6a+Xh3+1*8(?L@;RH*yTsZ#ShnGf_Kk5+@w#wK09^UM!**?kB;P!ce9YN! z%Ab-GxQTPLRlhFUq-;eZ`T*bhGpZfez`k_uFx2ZG3pf>_SlY{xOg~-7z;M zTWa!|EQ1nch|zzriY8nPxSX=RQ43xXx}K!Wu3&V4wpYN13_5VeW>ixv5W~=~!N^$s zwQjDGd0rd(TU=oY*#`!FHQ5@6QU_}ydKpFn?*qwMh_K^X3)NE1y>@#X;(1c6Abp>ttghAsG_bG?G6{n10X3V*d+ zvL6EzxA*T3kg1!z+F-GfT%hnd&VE5wG8+&$3cJxlPkf_^gpw~Td6Ek&)m^<)t;uxB z`L(!W`$6tE#V_?zS+w#K3Vjtw+v?8896`#c)6(C4!!TlkF5lR^?0AgpGk1bFYfJ1S zlm~0wi+H~JLSj(b?)Oez7&PYq0a*WlCVBV2b2N~M)T0yzLp4Xzdne4$8X`z&K?eKFZ`!eJL5gj|ABzZFaILPsk42(ao2c^K7= zi~Slb+54Sq#;u?T-cABHpf8iZ@+@E_WjyX9Djl(!l0VNpziqTk#Z`= zTLwQM+LRpX+wOZaHK0_HoqsG4oW#(uJn;N|BM^lwicWw(Bebu|7~-b_;7%c-drOk0 z%1))Fkz{#kyzqxVulYSoL}EfV(~ifb2xiov_143JvVq!?0%NoNpLUbeHf_SCMjcc% zkfGl+^#vpm4s|994+(84$}UoL3mpWB?I9D&GIvS1J> zM=ZLVQK>o%+0&N${jY6SXOq4h8XTv-+gc*oqlSeq=v+DeRbD@L(EikIYt2XHs%{-e35W<=rp=--p z*_0hK_QX2%5K&Jeb@D#EeB8h1Fum|yW3$2s-x0SVW(Ko@6G8ADhydd zZkuF5{18D%ZmNeTlh4psFc6U8v}4gpH=9Hbvd`MpHr3C5*7I*1G#RjDE4Bk6>SU** z9q5IS7d=6EJCNET?-69Af@2clx2pbh`;p$ChoSCDT7@K+xrd+AwxK4C9qLP+w_{4v zC(6u|DOL{g#@jJE%Hrvbq9yHKq8v%vW{zwlnXn(4v9%5$bgvg3YgYvJIAO0OLHuT! z#vLlkY!~ldv@0RGDft0~DU%tio!63WOv^AAy&(0j1vnwv6SBQw9AU53+Zp#?#tz>t z+LK+ZzHaJoWkGSkF2LWh6@Ae0rweNap$GNlaWM+s^hU4q) zuhZ#SPoXT*3KFmonMEY*HH&+-+dMyy_w*p=yZ&Fv!rkJ3N4ReG+cV+Y&{aq3??HDH z3{T+;eD3hv?Z$WWJ74|OW>nodjTyDUh$h#4IO55e43ytaJ#p1Y#_Uxpo#&5VX84AG zu9lhoQ54xMy^5&dMF!1?`LO2Ot8KI7GMoh8H+Nu?p{1FW zA`9<|bttUu7m6P+>t>mbS5+r+&eJ}RVZSUd4NobU0Um7alO1VGXTZ0+ct{Ro8^N`+ zZ&knG@rJz_`#;1P{cgnRE~$IfE1yYx6!P~-q?9wOwyl>6xa5bL zELwGAeoq}cwks8nO^y(Qr?!`#lY-3Ibd>e@r zBhbBis;TQ~wrh^8NEW&dca3z-&>_G2#vUB}U-J$6*(sZ^h&M1XL7I;Yt-=mHIpY6c zTb><<8Z`X-R}5bu^2{|2#ZOH7}!#`pF3dy}O@{h-T4*2wj ze{A9(&Qg+$(QhtrygQ!Nb)GM=z@8||Ik=6mDQoawDKy`{=S3SHp0dM6b2~^uJwxYu z7b{{wp)+2t+$%j}CAtes?AE-^WvF`=^YF3An#Sj?SBu^#AM=WCYou6~bKMMy%Wg6E zb5bRhiL~Hd%*Uh@xRXMgD%&#c!;Bj!$IuoUKyIgW@KenWhEE!h0AVz10#ri^#eoR3 z9PxHBvcinLlOf;4Z}_Knx~^)tDh{Et(I-5HmP1NCk?;}xF`6-(z2E~*0l4FTgOBb2yE-=#sjCfv_freRxS?R_Gvkeyv1sg@9SDCV!+$564|~ zZ*+qKu1%!E>rgkT+Sr+pwa*|3UV{BL^S&f5S~H1#gPvhTqHD*P$~0jXu5%y9pC2jc zz$vBCe$2s3=ir-$Gzl3PYQ(Hj$3$2k@Zyy>udWu{Wu_FC{zyhGK7|G$PzoQUW7F$o z`wDbncWzr9E+P}0OTJ3C%07)s1A-u8xz(z0e5^L1l7V%Tmex4 zKl@J;z7ch||1;-<|7XtYdM9*{WSW`YPH%bEOKIMo>1@bq)}NC@u&UM2pNAzo zT3Zqq#VNVj^g2tG?|!x@7( zu?)7}it5o>YUFD*aR3xY;S_RWw}w`%9xx+eX1~nD@Uzdz8`Y;!>_({{1e*ySB$_7M zTzDxFtaW>oWpSj?syve>mKoefLUEjTyt6fIugB|qODiBHq}~Dsb|}sssQJpC2V6vk z1ulCKDP=#8<#p5iWT#Fp=?(C;u9XwSVNsJSP{WTRV(VSUCW#VYNlalL&ej0tm~u-1 zfK%KFB)DynCtrfc-h4oLHDXO2c3^;si7a?Nj?ap^sR3}7*`4U%gSA$j$X4$UhQTY3i*1^G zIaY$?f)5LA_q&tOd&Xn^twQC;O%yWO^jRafb`W?ScVdwK_r4-YtKNGPi{s2<%6z$R zw*v22!M5}kNhc-CCF3-2`5JqR-qjWPf<2H7E*@X*{W0@kCKn^=DZmf1es7y}xCn`G z!x-Qcta@XWk`?yt=am=pX8}fJ^`q0m)b+vCb@L4aiZzU_^%Za5mf))90RAgO>D0oJ z-qcl@_2)OG-3}gUj%aOk1L2rG7I_+jU1yv9vxvRn_cqhXo>u9$h7p)<6*ZSGT>2*J zLHg=QBbPR5KSBzo2$s|!7IcV-C0RwCnA$YtvpQeAo_jMUk*j+6 zWR{ULd?oieN;F{atlu-J(1sUNMs?6nCn@>q_}l0W1!lc5<9_N3LWw3q!54(==fgK? zqhd@R5%992?G<+jR&J`EVXA?9#KmUk`?=8s7mpAf6kV|skpY^RTfhG}c_6%fP?^p{ zm(LT~`;)+K8Ns`%Fe?z+d)SIPKnhUdkhE&voHi6g?}J$t)ha6yr3hT3s^ z_K~K{+|aO1URvbs@N2!b4M9uOOY(}|I|K0$vEI%RrZ6{*QiAQZ(f7N+!yeGb*QWP3 zmK(T-G}uWVil{e67z|gl*j>6^J^str3;M%-T-0H2%S?E4`-w4EhLSLH-Gq6%cg5gr zCw66H`@)JVI3H){PeN}am>I7psy-)&Xw;_OS6`FSDYNUT2POBZqBdpB-;NKHz8gt% zDMin7cX>f2JfsmDd_&6V&^dtFieZMh`jHcrK&%>ASVXn@VWdv%!&nh~aw>((Z^ z#9)-pd@*a=;F+BhSI5@7SN>`XjiYnF<(o-7&UUX-a`=}#wnuD<>v%$(`!7!FFo$Oh ze~#^0%nz(XCw^KBxcyy^2QOe4pfh}Ew}-xN6oY;1&D&si$3=I?Q{=(dji@H%eKZmi zG&E*jgGiZUq{KmV2{Wr`w2Z8l_3-bCY~7)k|_X`9+C<)g)TrjcW!+ zAb~%EqQ7l=&RFM$%kSN)kAET<57*LJ2GJ@EeRfaF>XK`Rjx690eb?VVyxtw&AGT}Y z4$&<}{JGNb4ij5~Qeuw~zSl*xI3~1+h+4K{>+^3u?%F>XIzGoa+YP9aXxH6GM@##6 z94<}#LJnTD!ksJaMWTT;pC7x$aL~S}R5u-49&G*XkOxO^NC3 znW>&H)C;hnZ- z{8}D2pR*kTHd5HZiebvREc`R4ST5Omcot!l7tNAMDX9q&h^e7D$L7E4%!U$^k`aJcFnh_kb$=0FUjxh~b6yRaze-*o7SY zFsoL{-4`v0fyJzTtfN5E;C)htDVJS^xe^pgb*%5cqf;rZRq`1I*>$1$XX1;V{B&8W4pgKle z?$}-&yrM{QWf#tYrtIJAcXRLXT@7K^SL&8R^Wt4%+X!1TL}u}j30Lff8U|yWrP3P@ zDNVvP2L5nyURC;XzH)kB)wD}J$WE}(~{!cUw?Unr^2vE&&899z#7Z_7yR)8H@;WAG8o_L0di4hl%Ye6{`!ytJWxT21-ah z>dH*Jjh|l}3B86x&6vAAv!r=ZRtEjF_VpDV3Pst1#S>kV(ZevJZMOCSBY8xmSI6X1GK}=GP*^@s%3FZh3||CRxQ~mLaej zmmxrhBHLGbBVEPaCmJJjZ(Eo~v$9RCcSQVW=`Ilq?0QVn!*uPx;k7u0VGb)Ep{{;T zu4;%kD^X-$es!xqBINz+Sl0A~&|l*k`o}d>;=B6z7)N6c#jj}`9gpcjASQKRFW|p- zA&#KR{9V5CJDp#`bE$}&N!z16E2t5 zh}7DrdaMt~^XX_)pVow;Q*|Wh{O|&36nJRwXmzg2yiSWsF_|F#43djY@wpuN_=>nFf2d^Je$0 zFHR@pu9C@B(;&R2vH;_m{jDq3so+SJeF@6xn*&19s~hhHD@iaNJu8oD#-%5=N9m;1 ztjNcRRpkAH?=HHxmmdY}l+Z)qDZdm7K=z|bq&2FV^$PN?RH92|(k))#cxUaBMv_9bXc?0sU1rX&|fY@7)`11 zyju+RHqA6rLpkbH_H)T|8pY688475o7b5bwji!3e%s+F!V00aR`hvD55y3!KYzor> ztkWc^lV!%H+M$u&h42t3a$XKIrW3ciVCdcNV#gN(IC$4wxr*if-WoM2^vlS-%`luT z`1g}P<_HB$2$XA|XTO*U8Uyq0WChG?H0Z26CR2{MV&;v$hu0T@-s@s=bxr>H1?i9H%L=N<{4|Qm6+TIoU)o&e{pZKY zQ@6y~vm|yg&qkQyQJlK(ah^{6yD58JQT^pyPPr?O8~hwG?U+Fx5hW@gKt!p#204Dy zU7I(|YiQ(q4%Drylu))2j%+lCfpObpA+Dv<{AwJhSDl|uFRRn)E+U=m$oR5-vhHs& zzgfzW3;Njgh0Hg)>Y-b3PdxL*T>UZlMCtkUp$m-wux(UeOHp)-qen=AQ|>-!j~Vwp zXfWd5WY^HTYQ6>8Uzv}zfsn5qPyOZ%i(>rQPGK?~XA`4N9bX?>4P#@fqH{666qiJO z3?m+5=5i{swNBi}Dr!oAjeV1RD8+W#%ZqUs%!Gxv4aOoZ^Gf*Eh}fbqr_Gky__&sz zGDHBb_;%$V_l{#jCHu}toaLC`rK2K^Fo`KpGwM)?%p2eqXxinooJW4eC{}cG%&aN) z_gc<&jb5I3#FrQyu*z$LUwF5d`t<7tTaml>JrHo=n@f~pt?-}Bn@UM33rA`rW7UP} z0{6w5lXQ;{yk?8*r=P=}u&xb$U(%hzrCz$q_}I#LIl1Yf^KHy>ZaY7ITqYUl51|`g zpjimYtH(0kM{j1sTBmZqQ>(P1xfMnC&Z3dYl(k2<)ai@ey!kGm@i%MQ0knpLX*cJS z8hTCEbH9u2W)WF7t<$ryz(Th=!ScFsJ-7J7+`jDEkICQojAV6xa-+dsc8!k(Oc+jP zss<+e&uSnQ;G&0m5N(S!mZQWEIkadr%)eu1{ZfgwcdITEQGXAz<0gW^5YQx|jI~5a zU7R_{m4n^$Pbv_REe|(?@))x-6KEmcy;_Oy-OCf7HY7xNl>&afWL!nmNaGzngien* zI?g$LwIdaSO(^8RvJPbACs$d0#DB#<7yawY)p+DpTP42!|m)`yhw9mhTk=;uR_89vAjzvno1tyG2ds|J(tP~8o;l@v&ca9m{ zHZ2$^Bv@49m^nlYN--^d3nKn9A*Ut;%1>5}EeCZM z3Aec@uGZm@1%vqMWK2?sqrQbODpsH@?Vn{LZZsr6(jv_;x_Y)My3;?l9>09Ef)7)$ z-EH5{?AXPznck*LSpFis?{i^C!lfmw=)Y-8F=q&0?EvvEEJI;M_;donq<-mUft zYUw->wV+fb7B>439rUh<`;(+-nqsW?eQ8H#BLJZkmK+LALwq4@^#c?%IR8(5gf0VF zc$zM%^OQtOp9H$JmdZjh$Og=N!c7I7C~5V>fqZ!ho+GO;n8EdBK~`67o!)i+u|wsY%U1QL+N*7N^o}-=<_^B{$C#H$NrOtTWcC{ zQudfB21x1ucVYd{I5z>A0C(p~Zc1LLoTDO5Fkp0I)E!VH+HQh6W5F^@+rF|_#Rsus z;1y}#OCg28q#-5~euB4RgaVm2O(#7v``ZraOBX!86Cal)_a5RSm!wHuwe}q|)K8+K zix|yVkef_4BG6enrfVRfE+P=R#fEm#SRuAcvAFpvZTC0F(i-w1&U&;aoz0PA!;7N+ z8pxFu#xApvH#wL_Ul8)?_2|c+qo3Qmh{{F$iUm3Tx;w9l!;JW4#&~ugOOa?%orxtXs zFe=lh>f5>s^iO8IB@u<|GLTldRcbFG;HELapYWiK*7%_u|OgoDoFTHb#3$%+2_k2H*0r7raA8kI>$_}?`&)5w*#B{BE zKaW?E2O`#pq17P$Nm17JM;i19W`|}7LI`j;wm%L?QuKkDm@N6iCCnR_^4xf~3XGkc z1))Q83XV(7^%39#$GPjjG5n0d1^@g+pICAp{}d5xbs!)$n_hKJs<`c;#>V!@8#wfr zpd#^IQNpk)^*TA@=b*#186`JpmQ2Kp$y-wiTs&QT-dp;~VF% zE@!=9P^{K##?cDd&;reoVnXki@E#*%wlF&Ac{&{Glpg|!CQ~dH4B6>mzq6u34XvK8 zYV6CK)R^@WmY;#*@GnZrEqrv8R@LX;Hogjvc^e z0?))*)crQB)8z@n+?krehNJ|y{_>$@7%>m77V-&IC+7Z~bx>F5WC7q&bLhy29BgO* zOJS{|rik$=+R5G=)H?Nh!HIqL70d^1VM=2YcP6=i6L4> zm#ERp9pCre=l%tEemisad7iz`KIdI)z3)0}0jaz0BN*)`$8qCjuF6q!H9@m)QMa4B zWA?yLXh=N89f(OJ9Dcg33KV@%#q!cvsv7YPd4ITIy{?V$A9JaX!Lk) zd&ULAJb%Ar6D*uLUD*~hV-e|o{~j{zM(8hDR23yrT(?$iJ)>{vz1C^0{*F}C7&o$1 zK0dUCOtT4&5#lsSkUR#hpYMGV_MkngVq5&IIK(SeSCU_QWal5&y2XTBGgooO$Q361 zxdrqd$2aVl;aA=s1Xv>Fb_24(Pq^o+69{F`R!B!j6kRH~Sc@mD#rqeReepJ8`v^Uk z2d-&bP(uo&FJp^4o&8ubH?yqt>%oV3zncojL7t*CeLE6no4$2z!Py)5=y$wiwQ1L% zp<BTWS&uH9QB)0T|9@t{82$fJI?A_Q?u_H8OBVvG>k zKcyt#Jh9vg2u2~(RQ{`B1{(d*bx~;QTcpQle-5Wm(iJ0~UmhfyEj@=sr4N5ID929F zae7?Hpp^Ik$mIBn{+PPOT8w0aTylktJw+o{<0sT4S~^d09JDkiw;3ym#x-}tu>%=f_1}n7zRC86?GX*e znH30{!x5{Mh49Ee&4~7OI<|pt9bjZmV2XTZIq{H$hh2jQ&#Bak)$H>jiD{R}HAgNd zv3*y*X`a9gMdBf!T^aBEIl9CIq_Bi*dgVavhDPJ3M*+mHheBvG#nk+yKP8eGhhw~| zz#Xo#@85tKtr^5GBe7Q5>e0>t@kgoA6nm`R)`r`R#fK2#Uv1VSzl=nNbtWSl%@V06 z!iTxjXdiEa^NwCmvoRei>`CH<(N2c~fj1mEeCj{P1rxe}rVc`X=CW{xv!w(#hKNKo zZl&zlYf3y9-lUo#)NNyRp1SW$;t1O6D zV~nV##zr1vUrV=bJVu>{#4E0NShmtM)ijrrjAph+_Du)MG?zcGnBk@AUyMo$u?Uev z9z5KSNsvd9cdLOHgK@q)QNMgKQwYEsdm z;5q#Xs@_M!=N2oar2xZ>qe!lV`6JB1n{fj**ED|{biNguN;|?c{Zk+#QBlLu?HSH& z2pmD)5Pg>D5-XrK@O{yTPFF@~lw4Vk_*uH1cXytGg$CMYgQ)|sR*iW$vz4XHb_&kt zY*0dJ2nn^+mD()yi%uSD7u8cFVyvsFb*x#tMySkUtU()Z8v>H&FLs8BJS+u3Osm|= z#E}|SSw)tO51Q{Hd+?N;Yxa7n*FCjs+AgzF0}+|?G6yIp*q;IaP5I=u zWdBN2e!NC9`{FPAgQLL5n%K;c%C zI&a5r}1=*C&W8^UfN#hw03O zpUh81RcS7~d2$KbUJoV@7aRFFoGe(I$8~)QyB8{Pkqk!+_dWO2v)pDbCb@)l4FIu9 zM~TGF%QehcisP)~cv7X_Og`vz#kL)nl#68Nyc@3Q_x4G*Ok~quml>9&A(21`j9shq zYFzO(770A;brfMX>%5=pv)q#lPF?bQ4?V%GDJvT4jbz5}%NalNj(=L!ch32H=YHGK zPN~%F;06(G%-Zv%yfK63Oa4sSKHy6^fC(ckSjNr4T#A)IU@san96P=Y9h70r|D|yA zPHle_K*n%!alhlZ@dH@?Oi{y5*KucPppWo226Y>h8EzP0#5 z?xD5LKqsk&z*b>6w=lYxHB;9>-@fX95y)X57l=3dg)%awt z^OoHbt?D_IJ;4g6@=`Or_<9Oir_jigg@`oZ&NKP`v!F$G!eZprxI64d9#uLOMJ6>N zgN)+YB%|z!_Sg9Dtk{|fwpg4UV$ErVsnWE@edEFJUsoiQrB2HiP^pMEP+{AT>p_=; zc9YD0dMssOKk606#L&Av6c_glF0MoV^*C?AWC=EpHFCwLriEN#L3)bQ?vGX2yu<-h zUbkwDARBOD)HqPE^pRVH{C@Ixr#DLOUr)_V%nz90kq~-9&lFo5E4F^)NF zxy+tPXH(C0t0wqrX?zeyrW5zADLq?~t{2aLv&@&g~yz|NbsOW9ts-BYOhLhMKChDNP~mF_XCab>D)1+&iW z)QQ!LE+RH{w($T`9SRa19t5-`VFy0^&~y&ae$@1aEy)!(Usq^Dg%+*Z%>3|dS;NohZuH#iDK~Q4S|x-g^5+a z04w3}J5viGX*Gef`6k#H`lw~Dd7Dn%AF4*LqTp-WkI2tlZv}YkeGTP}$fmQ;-nh?&_qHw@Ul=;~-L|iq-dvVKV zPwB{Uvto4S7V5wkF~puwJ}lmFmN^!4ax+B8D>$Dj$FhEkw=pfaQOK*s{*QdQ>lSJQ z?jYKxm0qYQ`VEfXbt_hYjYagnQtEqGX@j`i;ek^JPBkU?%&IyR)ww0NUs-u5Sa5Z@*;*`9ZS-K<4%4W878^Do&sj*PwZGrM;wW@WoD8b zahXW@^UOLX>z@I;rA3mh?Z$8m}8VqT86T z!$pmeXPxhB9X_Pus(s4w?m%xB1quHO|4hB+^!5+9A)I|1U{dUiB+hE;i;ft-g{U zj=d}kpDnx;kRVF)S6=tF6B6}s);acW>XCC3^>Yk6NMb0Vr?v0kJl-ER^Toa5gEiZ- zsmFQl++(2vjt^!$wIh?vHH!OHvGR;r#rVL?58}`+J&`C{?+u#a68){0DO4{SNgf3E z%Ur)MT7G?o4VzQojz-O=rrU zV;eWh5DVY;QtX9s;^}`!o*-_|4vuD-19^FqYc1bI$;y&bQkmX;jU!QBK1*onQ6g|| zG+b!%@yFX@PTP=(CRC*FLGs*CYgb^&<}NZ*fZ$Q{vQl0$Q?7HpQO*UhwL%n6mOb&R zH@mvp@VZXB>R~n9Vm9=n;MUkh7kZeYN_W)CP6FWWZ&`bWj3}nmDN6!EC-k(?LAzw5 zITHM}jYb|2%C$fE^7zL}$c+yB#BFob24p>&1B2ayb4;l!3w+e`pMNqjO1~iEfgE+Y z)j|nQiE@sSLU!Lem99LwYS&wov#N0(6FjL-!guP+I1lD7BEv7Ts6vjZfxw8GPza@ z*Zz~s`p2->MN&oI8-6L{vCRy--rYN$Bx{-B-z$a*t0p24Q87vW#Ox+ zQkEKY*SejnWBVDs)SbgUth-*^6d{Jm;``c^Ov8D{7K(fwl(9wMAJk_ps_no1r1(pr zuISOE9dQg%)7NA&&NP#6jh~Gs`f1@|+yu(6*78VL9m@faA=OEAOPc5l?)&RQ;+AWI zCF>X+_<7vms!N+L_`ro(bH)B8eSbsr%hc_@+NFk(9^KS*=Jubwovj~+J605Bo1?uQ z=;di}4EV{p8#dHYnmrji(zmWCSj|@)?|N|u!(GX#5wY*x-@eaDf?!GM7~MiggsZ8D zC9FUZ-u^myTcYcqni5`90b%z-kcHEl%8`a6FR)zs+{^+t&u2Z zBb|nnr;OUd@Zt&Y1Yan$t@b&ZE2GlJIjYJS==mqyqqZB^7neLIruYe)wpLhL_Na*0 zL%8&jexxlp+F=YsGj@MC4yfhrQ0gZBn<@S|y?T(N49VE_egCI@Isy_Cx?kOIjOC9f>KI6q=s1$CJLU@X>_VYcl)g3OVH+BLpvG{}YUJEo zo?+n9MvV$&py=K=3ec%^eW|pUW?#W?sadFjI@5_iB#r6kSh?grXJN1ilZFA{Lx}@K zN``dB^0yHW)Hxz_{JDJ9H%F5HrOQ=Hl5AY{?!;-Z%Cnb{j5@Z%3y1znvKG~)t0l;J z;v_zuGG2yE;KOd}C_qAV&q(b$b-)5{N>RI9*>`mlQOWJ2H)KW^UZ3JQA)y6(uMsGJLq0{X?iL+`BFMed|O3?|uMyJjT(|VL08r?*$e2eBx3@x6O*uZ$0$P~g!CXuwya>NPX;~k zF8Fga;K+?`;Y%f6%JqT1sHc@5>9xrXKU;G7HHQ`bO?9-Dr&nkoIo#Y2)`xjs+ZGnQHsgIExjQ9v!Gy;q*J(u;QK-&eP<5;4B)MEK>UJD^TtT)0 zGW&6hMe!HjJd`?}&!oz840^!LT&qMakz6b2^VF_}L+q#^SM0{ZMJ<_IK{zCKtUrs- z!BQzkfUV=0Ete|2x)7`*>A5Doa{J;I#lzz6m5vsMG3m5l0^R9|L$ z2&sgB*};i*sg^F&OKGzMKCsZBRL`HKj~X1Z=)2V@0-8A#qVnkcI&7te06fzx<-Yrkh+xwL^KMOH^-~!}IA{S7e z%~NWeyktTCkE1?AGt2lcKqOIGV4_a?RKvQ``bD}E`R|u;Gc{&n8G}IH!AkM(_?+vW ze&R<}*};-6u07@1t%4zauN)k=hf}7#44g$VFCw)H#~BZ^jBc9d4{z*OSIwOwXF}B@ z=2Zn;dGn05AnneaWx7BPk+&Ra{<&fp77rSKLd^x2pB6}WdAs0m@IVnydXmu%AZ+Ke zD@9&=`AW_ne09(#v;g8YEXjBmS*2Ih&R<&^T%!+QXps-SlhcgQ4p2Yj0@Ga-xa7Xm zSW0;x?^D`P2jZqn9L&iAQoH&JM}2Y(kE|NQEXn~&sbW9LDWO2$0~SWi>=jg5cE69) z9L81sJg0};657zApgH${ZbgzP}Sso#TLi7Hee^WhrjP%B!_25toPoCeAZ;Z@4WWaP~ zEQ^h>?WEy%M7g3dmHqyh!)M*i01n~#PyJDfNrvXIYE8iHiJm2iY5utxisj^95d;vc`=MDGk&pL}J8mu73Yq||m*vC}PX7b?U`;wf z=^L|D_lP}g)d8Cq9W^G|zG`!}Iii9(3vmG#D)sVs#h;4nKWRIhf?mrqcjxd5RB71@ zgkc#3|BHM}I1wpODic>e4G@TbNSA=+OhW&uhoTv#8DZCu^oZ?~R!XEDB#db9UcWf7d$pRnB7YcFw`zzgvbIsqL`&TC%)u--vMqXY}qn)~-8 z93-zd$29QLnMW1a{k4Wa{>B^zfBO}Z157NnSmY&?7E=;OC1+wj&?vqs1v=lJ+y?SY zYUZDqYSt4)1pM#&pofrY|9qQAQWP}G3YKef+H zjqIHE&Cg24Y^H@?2M2F#m$ZbgmQSW$KkEK(jkV;HYD|1pe}t!rF|n(6>)j-NTjX!L z``bXOje3iM$dh?%1kHC3z zi~A(#Z(JabEX`oENQi%q`@-Jf_*Q;r1X&+-*-}i&@l)y2CL7Uxs!>Hx06Kb?N(Ud! zc?CC#xiQ7JvrFKJv{}7=zj2ui*|Qh=vPMYH@Yw-}4&HV8Twg}wbC$KO>%V11;bDal zJw8y(MZ3=Xj0%nh`yGQojyGFar>m^ermT?gkFC{d6iw!l+#$sT;?xBV|8#dm(O(27 zaHSkmD@w&&qPp^x~wS0HN>P99ck1u~S9 zTVX~^`&rAL${x8}84$CcZVpFsLqz-ie8OWq%_Y_i5Y)17hwpCY+3IclHzSoyF+1*$ z1oWzh2zD7cKdkATKThHf5TqpCCM?LebmUTjQx?T7z2wrOQ%G)Ui>ZhnGIagF*eTp zBtc3V9c%SxhL9rh)brPdy!{Yp@HqFw2aljtAMhw=6O zhj( $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH_SIZE, - "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, - "steps_per_print": 1, - "zero_optimization": { - "stage": $ZERO_STAGE - }, - "bf16": { - "enabled": true - } -} -EOT - -ds_args="" -ds_args=" --deepspeed ${ds_args}" -ds_args=" --deepspeed_config=$DS_CONFIG ${ds_args}" -ds_args=" --zero-stage=$ZERO_STAGE ${ds_args}" - -if [ "${activation_checkpoint}" = "true" ]; then - ds_args="--deepspeed-activation-checkpointing ${ds_args}" - - ## old argument for recomputing the transformer layer - # ds_args="--checkpoint-activations ${ds_args}" - - ## new argument for recomputing the transformer layer - ds_args="--recompute-granularity full --recompute-method uniform ${ds_args}" - ## new argument for recomputing only the attention layer - # ds_args="--recompute-granularity selective ${ds_args}" -fi - - -DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT" - -torchrun $DISTRIBUTED_ARGS \ - pretrain_gpt.py \ - --tensor-model-parallel-size $TP \ - --pipeline-model-parallel-size $PP \ - --num-layers $NUM_LAYERS \ - --hidden-size $HIDDEN_SIZE \ - --ffn-hidden-size $FFN_HIDDEN_SIZE \ - --num-attention-heads $NUM_HEADS \ - --micro-batch-size $MICRO_BATCH_SIZE \ - --global-batch-size $GLOBAL_BATCH_SIZE \ - --seq-length $SEQ_LENGTH \ - --max-position-embeddings $SEQ_LENGTH \ - --train-iters $TRAIN_STEPS \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH \ - --data-path $DATASET \ - --data-impl mmap \ - --tokenizer-type GPTSentencePieceTokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 \ - --distributed-backend nccl \ - --lr $LR \ - --lr-decay-style cosine \ - --min-lr $MIN_LR \ - --weight-decay $WEIGHT_DECAY \ - --clip-grad $GRAD_CLIP \ - --lr-warmup-iters $LR_WARMUP_STEPS \ - --optimizer adam \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --log-interval 1 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 \ - --bf16 \ - --no-query-key-layer-scaling \ - --attention-dropout 0 \ - --hidden-dropout 0 \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights \ - --swiglu \ - --normalization rmsnorm \ - --disable-bias-linear \ - --num-key-value-heads $NUM_KV_HEADS \ - $ds_args diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/pretrain_llama_distributed.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/pretrain_llama_distributed.sh deleted file mode 100644 index b7bf89023..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/pretrain_llama_distributed.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# This example script is contributed by external user https://github.com/LydiaXiaohongLi -set -ex - -###################################### -# Change the below configurations here -BASE_PATH=./tmp -DS_CONFIG=${BASE_PATH}/deepspeed.json -DATASET_1="./tmp/data/bookcorpus_train_1m_text_sentence" -DATASET="1 ${DATASET_1}" -CHECKPOINT_PATH=./tmp -TOKENIZER_PATH=./tmp/tokenizer.model # offical llama tokenizer.model - -TP=2 -PP=2 -ZERO_STAGE=0 - -GPUS_PER_NODE=8 -MASTER_ADDR=localhost -MASTER_PORT=6000 -NNODES=1 -NODE_RANK=0 - -HIDDEN_SIZE=2048 # e.g. llama-13b: 5120 -FFN_HIDDEN_SIZE=5504 # e.g. llama-13b: 13824 -NUM_LAYERS=24 # e.g. llama-13b: 40 -NUM_HEADS=16 # e.g. llama-13b: 40 -SEQ_LENGTH=2048 - -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=32 # e.g. llama: 4M tokens -TRAIN_STEPS=250000 # e.g. llama: 1T tokens / 4M tokens_per_batch = 250000 steps -LR=3e-4 -MIN_LR=3e-5 -LR_WARMUP_STEPS=2000 -WEIGHT_DECAY=0.1 -GRAD_CLIP=1 - -## Activation checkpointing saves GPU memory, but reduces training speed -# activation_checkpoint="true" -activation_checkpoint="false" - -# Below configuration required for llama model as per llama paper -# --no-query-key-layer-scaling \ -# --attention-dropout 0 \ -# --hidden-dropout 0 \ -# --use-rotary-position-embeddings \ -# --untie-embeddings-and-output-weights \ -# --swiglu \ -# --normalization rmsnorm \ -# --disable-bias-linear \ -###################################### - - - -cat < $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH_SIZE, - "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, - "steps_per_print": 1, - "zero_optimization": { - "stage": $ZERO_STAGE - }, - "bf16": { - "enabled": true - } -} -EOT - -ds_args="" -ds_args=" --deepspeed ${ds_args}" -ds_args=" --deepspeed_config=$DS_CONFIG ${ds_args}" -ds_args=" --zero-stage=$ZERO_STAGE ${ds_args}" - -if [ "${activation_checkpoint}" = "true" ]; then - ds_args="--deepspeed-activation-checkpointing ${ds_args}" - - ## old argument for recomputing the transformer layer - # ds_args="--checkpoint-activations ${ds_args}" - - ## new argument for recomputing the transformer layer - ds_args="--recompute-granularity full --recompute-method uniform ${ds_args}" - ## new argument for recomputing only the attention layer - # ds_args="--recompute-granularity selective ${ds_args}" -fi - -DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT" - -torchrun $DISTRIBUTED_ARGS \ - pretrain_gpt.py \ - --tensor-model-parallel-size $TP \ - --pipeline-model-parallel-size $PP \ - --num-layers $NUM_LAYERS \ - --hidden-size $HIDDEN_SIZE \ - --ffn-hidden-size $FFN_HIDDEN_SIZE \ - --num-attention-heads $NUM_HEADS \ - --micro-batch-size $MICRO_BATCH_SIZE \ - --global-batch-size $GLOBAL_BATCH_SIZE \ - --seq-length $SEQ_LENGTH \ - --max-position-embeddings $SEQ_LENGTH \ - --train-iters $TRAIN_STEPS \ - --save $CHECKPOINT_PATH \ - --load $CHECKPOINT_PATH \ - --data-path $DATASET \ - --data-impl mmap \ - --tokenizer-type GPTSentencePieceTokenizer \ - --tokenizer-model $TOKENIZER_PATH \ - --split 949,50,1 \ - --distributed-backend nccl \ - --lr $LR \ - --lr-decay-style cosine \ - --min-lr $MIN_LR \ - --weight-decay $WEIGHT_DECAY \ - --clip-grad $GRAD_CLIP \ - --lr-warmup-iters $LR_WARMUP_STEPS \ - --optimizer adam \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --log-interval 1 \ - --save-interval 10000 \ - --eval-interval 1000 \ - --eval-iters 10 \ - --bf16 \ - --no-query-key-layer-scaling \ - --attention-dropout 0 \ - --hidden-dropout 0 \ - --use-rotary-position-embeddings \ - --untie-embeddings-and-output-weights \ - --swiglu \ - --normalization rmsnorm \ - --disable-bias-linear \ - $ds_args \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/README.md deleted file mode 100644 index 004469bd4..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# July 2023 sync with NVIDIA/Megatron-LM -This folder includes details about the recent sync with the NVIDIA/Megatron-LM repo (where this repo is forked from). It includes example scripts we used to test after the sync, together with this README documentation about what were tested. - -We also created a [backup branch](https://github.com/microsoft/Megatron-DeepSpeed/tree/before_rebase) which is the version before this sync. This branch is just for comparison tests and for temporary use when debugging the main branch. We do not plan to continue supporting the version before sync. - -## List of rebase efforts/achievements -* Enabling Megatron-LM's sequence parallel. -* Enabling rotary positional embedding. -* Enabling FlashAttention v1 and v2. -* Fix the conflicts related to activation checkpointing when DeepSpeed is used with the newest Megatron-LM since NVIDIA introduced some new fine-grained partial checkpointing techniques which DeepSpeed is currently not compatible. -* Major refactor to DeepSpeed pipeline parallelism implementation for GPT model in order to work with newest Megatron-LM. -* Fix model checkpoint save/load when DeepSpeed is used with the newest Megatron-LM. -* Fully verified the performance and correctness of GPT pretraining after rebasing. - -## Test environment -We used 128 V100 GPUs (8 DGX-2 nodes, 16 GPU per node, inter-node network is InfiniBand with around 660 Gbps measured bandwidth) for the tests. For software, we used DeepSpeed v0.9.5. - -## Verified cases and results -We verified the following cases (matching training/validation curves before/after sync, checkpoint save/load works) for GPT-3 pretraining: - -* With DeepSpeed ZeRO stage 1 -* With DeepSpeed ZeRO stage 1 and Megatron-LM's tensor parallelism -* With DeepSpeed ZeRO stage 1, Megatron-LM's tensor parallelism, and DeepSpeed's pipeline parallelism (i.e., 3D parallelism) - -In addition, below is a performance/convergence comparison between before and after this sync. - -| Case | TFLOPs (per GPU) | Validation loss at step 200 | Training script | -| ---- | ---------------- | --------------------------- | --------------- | -| Before sync, GPT-3 13B, 3D parallelism | 50 | 5.73 | [script (in the backup branch)](https://github.com/microsoft/Megatron-DeepSpeed/blob/before_rebase/examples/before_rebase_test/ds_pretrain_gpt_13B.sh) | -| After sync, GPT-3 13B, 3D parallelism | 55.6 | 5.71 | [script](ds_pretrain_gpt_13B.sh) | - -At last, we provide a [toy example script](ds_pretrain_gpt_125M.sh) that users can try as the first test. - -## Flash attention -We tested and verified that flash attention feature introduced by this sync works properly for GPT pretraining. -Our code automatically uses [FlashAttention-2](https://github.com/Dao-AILab/flash-attention) when avaiable. - -We compared the training using the [toy example script](ds_pretrain_gpt_125M.sh) and the [toy example script with flash attention](ds_pretrain_gpt_125M_flashattn.sh) on 8 A100 GPUs, and found that FlashAttention (1.0,4) increased training throughput (TFLOPs per GPU) from 25 to 32. When scaling up the model to 2.7B using the same script, FlashAttention-2 improved the training throughput 121 TFLOPs to 132 TFLOPs in comparison to FlashAttention 1.x. - -For installation instructions, please refer to [FlashAttention's repository](https://github.com/Dao-AILab/flash-attention). - -## Rotary Positional Embedding (RoPE) -We also tested and verified that the Rotary Positional Embedding (RoPE) introduced by this sync works properly for GPT pretraining. By comparing the training between [without RoPE](ds_pretrain_gpt_1.3B.sh) and [with RoPE](ds_pretrain_gpt_1.3B_rope.sh), we are able to observe that RoPE helps improving the model convergence just like [previous observation](https://blog.eleuther.ai/rotary-embeddings/). - -## Notes/TODOs -* After the sync, DeepSpeed still relies on the older activation checkpointing mechanism (see function ```_checkpointed_forward``` in ```Megatron-DeepSpeed/megatron/model/transformer.py```) since we didn't have time to integrate with the new version yet. Contribution is very welcomed. -* (Aug 2023 update) With the contribution from 3P users (https://github.com/microsoft/Megatron-DeepSpeed/pull/225), now it's also possible to use Megatron-LM's newer activation checkpointing mechanism. However, currently it's still not compatible with DeepSpeed, so you won't be able to combine it with any DeepSpeed technologies. We DeepSpeed team compared the [older mechanism](ds_pretrain_gpt_1.3B.sh) and [newer mechanism](ds_pretrain_gpt_1.3B_megatron_checkpointing.sh) on 1 DGX-2 node (16 V100), and found that the older mechanism has less memory saving (older max allocated 15241 MB, newer 12924 MB) and higher throughput (older 23.11 TFLOPs newer 17.26 TFLOPs). Thus currently we still recommend using the older mechanism both because of the similar checkpointing performance, and (more importantly) because only older mechnaism is compatible with DeepSpeed (and in this case you can combine with ZeRO to achieve more memeory saving). diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_TEMPLATE.json deleted file mode 100644 index 3526aae85..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_TEMPLATE.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_slw_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_slw_TEMPLATE.json deleted file mode 100644 index f1abcedcb..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_config_gpt_slw_TEMPLATE.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false, - "curriculum_learning": { - "enabled": true, - "curriculum_type": "seqlen", - "min_difficulty": CONFIG_CL_MIN, - "max_difficulty": CONFIG_CL_MAX, - "schedule_type": "fixed_linear", - "schedule_config": { - "total_curriculum_step": CONFIG_CL_DURATION, - "difficulty_step": 8 - } - } -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B.sh deleted file mode 100644 index ccc2e581a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B.sh +++ /dev/null @@ -1,332 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=512 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=2 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=0 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ or -## https://the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_megatron_checkpointing.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_megatron_checkpointing.sh deleted file mode 100644 index 343dc9f0e..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_megatron_checkpointing.sh +++ /dev/null @@ -1,345 +0,0 @@ -#!/bin/bash -############################################################################### -############################################################################### -############################################################################### -## WARNING: This script is only for evaluating Megatron-LM's activation -## checkpointing. We do not recommend using it for actual training because -## you are not able to use any DeepSpeed technologies. -############################################################################### -############################################################################### -############################################################################### -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=512 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=2 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=0 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ or -## https://the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase_megatron_checkpointing" - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -# test megatron activation checkpointing -# we fixed bug in the code of this activation checkpointing, i.e., --recompute-granularity full --recompute-method uniform -# the two arguments can be found in megatron/arguments.py -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --recompute-granularity full \ - --recompute-method uniform \ - --recompute-num-layers 1" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -# disable the deepspeed activation checkpointing - -# if [ "${activation_checkpoint}" = "true" ]; then -# deepspeed_options="${deepspeed_options} \ -# --deepspeed-activation-checkpointing" -# fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope.sh deleted file mode 100644 index a3d6918ef..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=512 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=4 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=8 -no_pp="false" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ or -## https://the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase_rope0.25" - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-rotary-position-embeddings \ - --rotary-percent 0.25 \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope_slw.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope_slw.sh deleted file mode 100644 index 209021a39..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_1.3B_rope_slw.sh +++ /dev/null @@ -1,347 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=512 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=4 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=8 -no_pp="false" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### curriculum learning (sequence length warmup) configs -# The "divided by 3" means we use 1/3 of baseline's total steps for sequence length warmup. -# This is not always the best config, but usually a reasonable choice to start with. -cl_step=$(( ${lr_warmup_tokens} / 3 / ${global_batch_size} / ${seq_len} )) -# Starting sequence length during sequence length warmup. If the train/validation loss is -# unstable at the beginning of training, need to increase this but also need to keep as multiples -# of 8 in order to enable Tensor Core acceleration. -cl_min=64 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ or -## https://the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase_rope0.25" -jobname="${jobname}_cl_step${cl_step}_cl_min${cl_min}" - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-rotary-position-embeddings \ - --rotary-percent 0.25 \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}_cl_step${cl_step}_cl_min${cl_min}.json" -template_json="ds_config_gpt_slw_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - | sed "s/CONFIG_CL_MIN/${cl_min}/" \ - | sed "s/CONFIG_CL_MAX/${seq_len}/" \ - | sed "s/CONFIG_CL_DURATION/${cl_step}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M.sh deleted file mode 100644 index 8235b6c1a..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M.sh +++ /dev/null @@ -1,331 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -model_size=0.125 -num_layers=12 -hidden_size=768 -num_attn_heads=12 -global_batch_size=256 -lr=6.0e-4 -min_lr=1.0e-6 -init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=16 -# global_batch_size=512 -# lr=2.0e-4 -# min_lr=1.0e-6 -# init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=2 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=2 -no_pp="false" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -data_path="BookCorpusDataset_text_document" -if [ ! -f "BookCorpusDataset_text_document.bin" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -fi -if [ ! -f "BookCorpusDataset_text_document.idx" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx -fi - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="output" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -tensorboard_dir="${output_home}/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M_flashattn.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M_flashattn.sh deleted file mode 100644 index 3a26aab26..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_125M_flashattn.sh +++ /dev/null @@ -1,332 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -model_size=0.125 -num_layers=12 -hidden_size=768 -num_attn_heads=12 -global_batch_size=256 -lr=6.0e-4 -min_lr=1.0e-6 -init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=16 -# global_batch_size=512 -# lr=2.0e-4 -# min_lr=1.0e-6 -# init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=2 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=2 -no_pp="false" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -data_path="BookCorpusDataset_text_document" -if [ ! -f "BookCorpusDataset_text_document.bin" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -fi -if [ ! -f "BookCorpusDataset_text_document.idx" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx -fi - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="output" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -tensorboard_dir="${output_home}/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-flash-attn \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_13B.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_13B.sh deleted file mode 100644 index 931886b34..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/rebase/ds_pretrain_gpt_13B.sh +++ /dev/null @@ -1,332 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=2048 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=16 -# global_batch_size=512 -# lr=2.0e-4 -# min_lr=1.0e-6 -# init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -model_size=13 -num_layers=40 -hidden_size=5120 -num_attn_heads=40 -global_batch_size=1024 -lr=1.0e-4 -min_lr=1.0e-6 -init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -mp_size=4 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=8 -no_pp="false" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=2 -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -## Public the Pile dataset, can be downloaded at -## https://mystic.the-eye.eu/public/AI/pile_neox/ or -## https://the-eye.eu/public/AI/pile_neox/ Change data_home to where you -## store the pile_text_document.bin and pile_text_document.idx. -data_home="/vc_data_blob/users/conglli/the_pile_public_merged_nopreprocessing" -data_path="${data_home}/pile_text_document" - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="/blob/users/${username}/project/data_efficient_gpt" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -## Microsoft internal constraint: because tensorboard is logged by last rank, -## it's better to put the path in NFS instead of Blob. -tensorboard_dir="/vc_data/users/${username}/project/data_efficient_gpt/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size ${mp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} &>> ${log_path}/${jobname}_${host}_${current_time}.log \ No newline at end of file diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/run_deepspeed_example.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/run_deepspeed_example.sh deleted file mode 100644 index 909cdf671..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/run_deepspeed_example.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -ex - -BASE_PATH=/vc_data/Megatron-LM/data -DATA_PATH=${BASE_PATH}/indexed_datasets/megatron -DS_CONFIG=ds_config.json - -TP=1 -PP=1 -NLAYERS=24 -HIDDEN=512 - -GLOBAL_BATCH=64 -MICRO_BATCH=4 - -ZERO_STAGE=2 - -OUTPUT_DIR=ds_z${ZERO_STAGE}_nl${NLAYERS}_hs${HIDDEN}_gb${GLOBAL_BATCH}_mb${MICRO_BATCH} -#OUTPUT_DIR=baseline_nl${NLAYERS}_hs${HIDDEN}_gb${GLOBAL_BATCH}_mb${MICRO_BATCH} -mkdir -p $OUTPUT_DIR - -cat < $DS_CONFIG -{ - "train_batch_size" : $GLOBAL_BATCH, - "train_micro_batch_size_per_gpu": $MICRO_BATCH, - "steps_per_print": 1, - - "zero_optimization": { - "stage": $ZERO_STAGE - }, - - "fp16": { - "enabled": true, - "initial_scale_power": 12 - }, - - "wall_clock_breakdown" : true -} -EOT - -export NCCL_DEBUG=warn - -ds_args="" -ds_args=" --deepspeed ${ds_args}" -ds_args=" --no-pipeline-parallel ${ds_args}" -ds_args=" --deepspeed_config=$DS_CONFIG ${ds_args}" -ds_args=" --zero-stage=$ZERO_STAGE ${ds_args}" -ds_args=" --deepspeed-activation-checkpointing ${ds_args}" - - -deepspeed pretrain_gpt.py \ - --tensor-model-parallel-size $TP \ - --pipeline-model-parallel-size $PP \ - --num-layers $NLAYERS \ - --hidden-size $HIDDEN \ - --num-attention-heads 16 \ - --seq-length 256 \ - --loss-scale 12 \ - --max-position-embeddings 1024 \ - --micro-batch-size 4 \ - --global-batch-size 1024 \ - --train-iters 1000 \ - --lr 6.0e-5 \ - --min-lr 6.0e-6 \ - --lr-decay-style cosine \ - --log-interval 1 \ - --eval-iters 40 \ - --eval-interval 1000 \ - --data-path $DATA_PATH \ - --vocab-file $BASE_PATH/gpt2-vocab.json \ - --merge-file $BASE_PATH/gpt2-merges.txt \ - --save-interval 1000 \ - --split 98,2,0 \ - --clip-grad 1.0 \ - --weight-decay 0.1 \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --init-method-std 0.006 \ - --fp16 \ - --checkpoint-activations \ - --tensorboard-dir $OUTPUT_DIR \ - $ds_args \ - --exit-interval 5000 | tee ${OUTPUT_DIR}/output.log - diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/README.md deleted file mode 100644 index 96e0ef8a8..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Sequence Parallelism - -This folder contains examples that demonstrate how to use DeepSpeed's sequence parallelism. - -## Setting Up the Environment for FlashAttention - -DeepSpeed's sequence parallelism can be combined with the following types of attention. - -- Classic attention -- FlashAttention (enabled by `--use-flash-attn`) -- FlashAttention + Triton (enabled by `--use-flash-attn-triton`) - -For the best performance, we recommend using FlashAttention + Triton. Here are the installation steps and the versions we have tested. Note that FlashAttention is compatible only with Turing, Ampere, Ada, or Hopper GPUs. - -```shell -# install triton -git clone -b legacy-backend https://github.com/openai/triton -cd triton/python/ -pip install cmake -pip install . - -# install -cd ${WORK_DIR} -git clone -b v1.0.4 https://github.com/HazyResearch/flash-attention -cd flash-attention -python setup.py install -``` - -## Enabling Sequence Parallelism - -To enable sequence parallelism, set the degree of parallelism using the `--ds-sequence-parallel-size` argument. Ensure that the number of attention heads is divisible by this value. -Ensure your model configuration is compliant with FlashAttention's requirements. For instance, to achieve optimal performance, the head size should be divisible by 8. Refer to the document of [FlashAttention](https://github.com/Dao-AILab/flash-attention/tree/v1.0.4) for more details. - -Some working examples ([GPT1.3B](ds_pretrain_gpt_1.3B_seq_parallel_32k.sh), [GPT30B](ds_pretrain_gpt_30B_seq_parallel_32k.sh)), that enable sequence parallelism, are available in this foloder. - -Please note that our sequence parallelism feature is currently incompatible with Megatron-LM's tensor or pipeline parallelism. diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_config_gpt_TEMPLATE.json b/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_config_gpt_TEMPLATE.json deleted file mode 100644 index 3526aae85..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_config_gpt_TEMPLATE.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "train_batch_size": GBSIZE, - "train_micro_batch_size_per_gpu": MBSIZE, - "steps_per_print": LOG_INTERVAL, - - "zero_optimization": { - "stage": ZERO_STAGE - }, - - "gradient_clipping": 1.0, - "prescale_gradients": PRESCALE_GRAD, - - "fp16": { - "enabled": true, - "loss_scale": 0, - "loss_scale_window": 500, - "hysteresis": 2, - "min_loss_scale": 1, - "initial_scale_power": 11 - }, - - "wall_clock_breakdown" : false -} diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_1.3B_seq_parallel_32k.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_1.3B_seq_parallel_32k.sh deleted file mode 100644 index da028dc73..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_1.3B_seq_parallel_32k.sh +++ /dev/null @@ -1,341 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=32768 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -model_size=1.3 -num_layers=24 -hidden_size=2048 -num_attn_heads=16 -global_batch_size=2 -lr=2.0e-4 -min_lr=1.0e-6 -init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -## Currently we only support MP=1 with SP>1 -mp_size=1 - -## Sequence parallelism, 1 is no SP -sp_size=4 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=1 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} / ${sp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=1 - -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -data_path="BookCorpusDataset_text_document" -if [ ! -f "BookCorpusDataset_text_document.bin" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -fi -if [ ! -f "BookCorpusDataset_text_document.idx" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx -fi - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $sp_size -gt 1 ]]; then - jobname="${jobname}_sp${sp_size}" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="output" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -tensorboard_dir="${output_home}/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size 1 \ - --ds-sequence-parallel-size ${sp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-flash-attn-triton \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} 2>&1 | tee ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_30B_seq_parallel_32k.sh b/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_30B_seq_parallel_32k.sh deleted file mode 100644 index f23e6f958..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/sequence_parallel/ds_pretrain_gpt_30B_seq_parallel_32k.sh +++ /dev/null @@ -1,351 +0,0 @@ -#!/bin/bash -dir=`pwd` -############################################################################### -### Main configs -## GPT-3 models use 2K sequence length/context window -seq_len=32768 - -## The "GPT-3 XXX" below are configs from GPT-3 paper -## https://arxiv.org/abs/2005.14165, choose based on -## your desired model size or build your own configs - -## init_std is standard deviation for weight initialization. Usually larger -## model needs lower std. We used a heuristic equation of sqrt(1/3/hidden_size) -## from the MT-NLG 530B work (https://arxiv.org/pdf/2201.11990.pdf) - -## We changed min_lr to a lower number (1.0e-6), which we found is able to -## provide better zero-shot eval results. - -## GPT-3 Small 125M -# model_size=0.125 -# num_layers=12 -# hidden_size=768 -# num_attn_heads=12 -# global_batch_size=256 -# lr=6.0e-4 -# min_lr=1.0e-6 -# init_std=0.02 - -## GPT-3 Medium 350M -# model_size=0.35 -# num_layers=24 -# hidden_size=1024 -# num_attn_heads=16 -# global_batch_size=256 -# lr=3.0e-4 -# min_lr=1.0e-6 -# init_std=0.018 - -## GPT-3 Large 760M -# model_size=0.76 -# num_layers=24 -# hidden_size=1536 -# num_attn_heads=16 -# global_batch_size=256 -# lr=2.5e-4 -# min_lr=1.0e-6 -# init_std=0.015 - -## GPT-3 XL 1.3B -# model_size=1.3 -# num_layers=24 -# hidden_size=2048 -# num_attn_heads=16 -# global_batch_size=32 -# lr=2.0e-4 -# min_lr=1.0e-6 -# init_std=0.013 - -## GPT-3 2.7B -# model_size=2.7 -# num_layers=32 -# hidden_size=2560 -# num_attn_heads=32 -# global_batch_size=512 -# lr=1.6e-4 -# min_lr=1.0e-6 -# init_std=0.011 - -## GPT-3 6.7B -# model_size=6.7 -# num_layers=32 -# hidden_size=4096 -# num_attn_heads=32 -# global_batch_size=1024 -# lr=1.2e-4 -# min_lr=1.0e-6 -# init_std=0.009 - -## GPT-3 13B -# model_size=13 -# num_layers=40 -# hidden_size=5120 -# num_attn_heads=40 -# global_batch_size=1024 -# lr=1.0e-4 -# min_lr=1.0e-6 -# init_std=0.008 - -# GPT-3 30B -model_size=30 -num_layers=64 -hidden_size=6144 -num_attn_heads=64 -global_batch_size=2 -lr=1.0e-4 -min_lr=1.0e-6 -init_std=0.008 - -## GPT-3 175B -# model_size=175 -# num_layers=96 -# hidden_size=12288 -# num_attn_heads=96 -# global_batch_size=1536 -# lr=0.6e-4 -# min_lr=1.0e-6 -# init_std=0.005 -############################################################################### -### Training duration configs -## The main termination condition, original GPT-3 paper trains for 300B tokens. -train_tokens_in_billion=300 -train_tokens=$((${train_tokens_in_billion} * 1000000000)) - -## train_samples is another termination condition and also affect the number of -## data samples to be indexed. Since we want to reach the train_tokens -## above, and data efficiency techniques may change num tokens in some samples, -## so we just set this config large enough to make sure we have enough -## processed data and don't terminate by train_samples. -train_samples=$(( 300 * 1000000000 * 2 / ${seq_len} )) - -## Another wall-clock time termination condition in minutes. Set it large -## enough to avoid undesired early termination. -exit_duration=30000000 -############################################################################### -### lr configs -## lr warmup and decay duration. -## Original GPT-3 paper uses 375M warmup tokens and 260B cosine decay tokens. -## Here we increase the warmup tokens to 3B since when batch size warmup is not -## used, there are more tokens per step. Thus we need to increase warmup tokens -## to make sure there are enough warmup steps, which is important for training -## stability. -lr_warmup_tokens_in_million=3000 -lr_warmup_tokens=$((${lr_warmup_tokens_in_million} * 1000000)) -## Here we changed the LR decay tokens to align with total train tokens, since -## related works (e.g., https://arxiv.org/abs/2203.15556) find that setting the -## learning rate schedule to match the number of training tokens results in the -## best final model quality -lr_decay_tokens_in_billion=${train_tokens_in_billion} -lr_decay_tokens=$((${lr_decay_tokens_in_billion} * 1000000000)) -lr_decay_style="cosine" -############################################################################### -### Parallelism configs -## Model parallelism, 1 is no MP -## Currently we only support MP=1 with SP>1 -mp_size=1 - -## Sequence parallelism, 1 is no SP -sp_size=4 - -## Pipeline parallelism. To disable PP, set pp_size to 1 and no_pp to true. -## Note that currently both curriculum learning and random-LTD are NOT -## compatible with pipeline parallelism. -pp_size=1 -no_pp="true" - -## ZeRO-based data parallelism, stage=0 will disable ZeRO -zero_stage=3 - -## Total number of GPUs. ds_ssh is from DeepSpeed library. -num_gpus=$(($(ds_ssh nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-2)) -num_gpus_pernode=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) -num_node=$(( ${num_gpus} / ${num_gpus_pernode} )) - -## Data parallel size. -dp_size=$(( ${num_gpus} / ${pp_size} / ${mp_size} / ${sp_size} )) - -## Micro batch size per GPU -## Make sure that batch_size <= global_batch_size*pp_size*mp_size/num_gpus -## Reduce it manually if GPU OOM -# batch_size=$(( ${global_batch_size} / ${dp_size} )) -batch_size=1 - -############################################################################### -### Misc configs -log_interval=10 -eval_iters=10 -eval_interval=100 -# num_save controls how frequent to save checkpoint. num_save=20 means that a -# checkpoint will be saved every 5% of training. For longer training you would -# want larger num_save to save more frequently, and vice versa. -num_save=100 -estimated_train_iter=$((${train_tokens} / ${seq_len} / ${global_batch_size})) -# save_interval=$((${estimated_train_iter} / ${num_save})) -save_interval=100 - -## Activation checkpointing saves GPU memory, but reduces training speed -activation_checkpoint="true" -# activation_checkpoint="false" - -## Whether or not log optimizer states (norms, max abs values) to tensorboard. -## This is not required for training and might save GPU memory when turned off. -log_optimizer_state="true" -############################################################################### -### Output and data configs -current_time=$(date "+%Y.%m.%d_%H.%M.%S") -host="${HOSTNAME}" -seed=1234 -num_workers=0 - -data_path="BookCorpusDataset_text_document" -if [ ! -f "BookCorpusDataset_text_document.bin" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.bin -fi -if [ ! -f "BookCorpusDataset_text_document.idx" ]; then - wget https://the-eye.eu/public/AI/pile_neox/data/BookCorpusDataset_text_document.idx -fi - -vocab_path="gpt2-vocab.json" -if [ ! -f "$vocab_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -fi -merge_path="gpt2-merges.txt" -if [ ! -f "$merge_path" ]; then - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -fi - -prescale_grad="true" -jobname="gpt_${model_size}B_tok${train_tokens_in_billion}B" -jobname="${jobname}_lr${lr}_min${min_lr}_w${lr_warmup_tokens_in_million}M_d${lr_decay_tokens_in_billion}B_${lr_decay_style}" -jobname="${jobname}_gbs${global_batch_size}_mbs${batch_size}_g${num_gpus}" -if [[ $zero_stage -gt 0 ]]; then - jobname="${jobname}_z${zero_stage}" - prescale_grad="false" -fi -if [[ $sp_size -gt 1 ]]; then - jobname="${jobname}_sp${sp_size}" -fi -if [[ $mp_size -gt 1 ]]; then - jobname="${jobname}_mp${mp_size}" -fi -if [ "${no_pp}" = "false" ]; then - jobname="${jobname}_pp${pp_size}" -fi -jobname="${jobname}_seed${seed}_rebase" - -username=$(whoami) -output_home="output" -log_path="${output_home}/log/" -checkpoint_path="${output_home}/checkpoint/${jobname}" -tensorboard_dir="${output_home}/tensorboard/" -tensorboard_path="${tensorboard_dir}${jobname}_${host}_${current_time}" -mkdir -p ${log_path} -mkdir -p ${checkpoint_path} -mkdir -p ${tensorboard_path} -############################################################################### -data_options=" \ - --vocab-file ${vocab_path} \ - --merge-file ${merge_path} \ - --data-path ${data_path} \ - --data-impl mmap" - -## If CL is used, make sure to set "--split" the same as what you used during -## offline data analysis&indexing. -megatron_options=" \ - --override-opt_param-scheduler \ - --adam-beta1 0.9 \ - --adam-beta2 0.95 \ - --tensor-model-parallel-size 1 \ - --ds-sequence-parallel-size ${sp_size} \ - --init-method-std ${init_std} \ - --lr-decay-tokens ${lr_decay_tokens} \ - --lr-warmup-tokens ${lr_warmup_tokens} \ - --micro-batch-size ${batch_size} \ - --exit-duration-in-mins ${exit_duration} \ - --global-batch-size ${global_batch_size} \ - --num-layers ${num_layers} \ - --hidden-size ${hidden_size} \ - --num-attention-heads ${num_attn_heads} \ - --seq-length ${seq_len} \ - --max-position-embeddings ${seq_len} \ - --train-tokens ${train_tokens} \ - --train-samples ${train_samples} \ - --lr ${lr} \ - --min-lr ${min_lr} \ - --lr-decay-style ${lr_decay_style} \ - --split 949,50,1 \ - --log-interval ${log_interval} \ - --eval-interval ${eval_interval} \ - --eval-iters ${eval_iters} \ - --save-interval ${save_interval} \ - --weight-decay 0.1 \ - --clip-grad 1.0 \ - --hysteresis 2 \ - --num-workers ${num_workers} \ - --fp16 \ - --seed ${seed} \ - --load ${checkpoint_path} \ - --save ${checkpoint_path} \ - --no-async-tensor-model-parallel-allreduce \ - --use-flash-attn-triton \ - --tensorboard-queue-size 1 \ - --log-timers-to-tensorboard \ - --log-batch-size-to-tensorboard \ - --log-validation-ppl-to-tensorboard \ - --tensorboard-dir ${tensorboard_path}" - -if [ "${activation_checkpoint}" = "true" ]; then -megatron_options="${megatron_options} \ - --checkpoint-activations" -fi - -if [ "${log_optimizer_state}" = "true" ]; then -megatron_options="${megatron_options} \ - --log-optimizer-states-to-tensorboard" -fi - -config_json="ds_config_gbs${global_batch_size}_mbs${batch_size}_log${log_interval}_zero${zero_stage}.json" -template_json="ds_config_gpt_TEMPLATE.json" -sed "s/GBSIZE/${global_batch_size}/" ${template_json} \ - | sed "s/MBSIZE/${batch_size}/" \ - | sed "s/LOG_INTERVAL/${log_interval}/" \ - | sed "s/ZERO_STAGE/${zero_stage}/" \ - | sed "s/PRESCALE_GRAD/${prescale_grad}/" \ - > ${config_json} - -deepspeed_options=" \ - --deepspeed \ - --deepspeed_config ${config_json} \ - --zero-stage ${zero_stage} \ - --pipeline-model-parallel-size ${pp_size}" - -if [[ "${no_pp}" = "true" ]]; then -deepspeed_options="${deepspeed_options} \ - --no-pipeline-parallel" -fi - -if [ "${activation_checkpoint}" = "true" ]; then -deepspeed_options="${deepspeed_options} \ - --deepspeed-activation-checkpointing" -fi - -## When saving checkpoint to a storage with cache, their could be consistency -## issue of the pointer to latest checkpoint. Here we find the correct pointer -## and broadcast it to all nodes. -iteration_file="$checkpoint_path/latest_checkpointed_iteration.txt" -iteration_file_2="$checkpoint_path/latest" -iteration=0 -for (( node = 0; node <= num_node-1; node++ )) -do - if $(ssh -q worker-"$node" "test -f \"$iteration_file\""); then - local_iteration=$(ssh -q worker-"$node" cat $iteration_file) - iteration=$(( ${local_iteration} > ${iteration} ? ${local_iteration} : ${iteration} )) - fi -done -if [[ $iteration -gt 0 ]]; then - iteration_2="global_step${iteration}" - ds_ssh "echo $iteration > $iteration_file" - ds_ssh "echo $iteration_2 > $iteration_file_2" -fi - -deepspeed ${dir}/../../pretrain_gpt.py ${megatron_options} ${data_options} ${deepspeed_options} 2>&1 | tee ${log_path}/${jobname}_${host}_${current_time}.log diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/README.md b/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/README.md deleted file mode 100644 index 341b0d113..000000000 --- a/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Universal Checkpoint examples - -This folder contains example scripts that demonstrate how to use Universal Checkpoints to change the number of GPUs when training with ZeRO. With Universal Checkpoints, training can be resumed with a different parallelism degree on any of tensor slicing (TP), pipeline parallelism (PP), sequence parallelism (SP) and data parallelism (DP). Using universal checkpoints involves the following three steps: - -1. ZeRO-based training run, optionally combining TP and PP or SP, that creates normal ZeRO checkpoints. -2. Converting ZeRO checkpoint into the universal format using `ds_to_universal.py` utility of DeepSpeed. -3. Resuming training with the universal checkpoint, on a different number of GPUs. - -## ZeRO stage 1 training -For ZeRO stage 1, we provide bash scripts for bf16 and fp16 training examples corresponding to the steps 1 and 3 above. The step 1 scripts launch a training run of TP=PP=DP=2 of 200 iterations that creates a checkpoint every 100 iterations. The step 3 scripts load a universal checkpoint of iteration 100 and resume training with TP=PP=2 and DP=1 for an additional 100 iterations. Users can modify these scripts to try out other save and resume 3D combinations (e.g., save TP=PP=DP=1 and resume TP=PP=DP=2). Tensorboard logs are created by both step 1 and 3 scripts to enable visual inspection of how well the loss curves of the initial and resumed training runs match, especially at iteration 101. - -1. bf16: - * run_bf16.sh: step 1 - * run_universal_bf16.sh: step 3 - -2. fp16: - * run_fp16.sh: step 1 - * run_universal_fp16.sh: step 3 - -Please note that these scripts should be run from the root folder of the repo (i.e., two levels above this README). For illustration, here are the commands for running the bf16 example. - -### Download and Pre-process Training Dataset -Before executing the steps below, you can download and pre-process the training set using the following commands (see [here](https://github.com/bigscience-workshop/Megatron-DeepSpeed?tab=readme-ov-file#quick-pre-processing-to-start-training-with) for more details): -```bash -wget https://huggingface.co/bigscience/misc-test-data/resolve/main/stas/oscar-1GB.jsonl.xz -wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -xz -d oscar-1GB.jsonl.xz -python tools/preprocess_data.py \ - --input oscar-1GB.jsonl \ - --output-prefix my-gpt2 \ - --vocab-file gpt2-vocab.json \ - --dataset-impl mmap \ - --tokenizer-type GPT2BPETokenizer \ - --merge-file gpt2-merges.txt \ - --append-eod \ - --workers 8 -``` - -NOTE: Make sure to update your `BASE_DATA_PATH` path in the `run_[bf16/fp16].sh` and `run_universal_[bf16/fp16].sh` scripts to point to the pre-processed data. - -### Step 1: Create ZeRO checkpoint -```bash - bash examples_deepspeed/universal_checkpointing/run_bf16.sh -``` -By default the script will create the checkpoints in folder `z1_uni_ckpt/checkpoints/gpt2/z1/bf16/tp2_pp2_dp2_toy` - -### Step 2: Convert ZeRO checkpoint of iteration 100 to Universal format -Assuming the DeepSpeed source code is cloned into the home folder, the following command will generate universal checkpoint for iteration 100. - -```bash -python ${HOME}/DeepSpeed/deepspeed/checkpoint/ds_to_universal.py \ - --input_folder z1_uni_ckpt/checkpoints/gpt2/z1/bf16/tp2_pp2_dp2_toy/global_step100 \ - --output_folder z1_uni_ckpt/checkpoints/gpt2/z1/bf16/tp2_pp2_dp2_toy/global_step100_universal -``` -Note that we chose to create the universal checkpoint in the same checkpoint folder as the ZeRO checkpoint. This maintains the normal checkpoint folder structure expected by the Megatron-DeepSpeed code, which makes it easy to load universal checkpoints with little/no script or code changes. For clarity, we show below the contents of the checkpoint folder after creation of the universal checkpoint. Note that the conversion script creates `global_step100_universal` folder and `latest_universal` file. - -```bash -ls -l z1_uni_ckpt/checkpoints/gpt2/z1/bf16/tp2_pp2_dp2_toy/ -total 48 -drwxr-xr-x 2 user group 4096 Oct 21 08:51 global_step100 -drwxr-xr-x 3 user group 4096 Oct 21 09:28 global_step100_universal -drwxr-xr-x 2 user group 4096 Oct 21 09:01 global_step200 --rw-r--r-- 1 user group 14 Oct 21 09:50 latest --rw-r--r-- 1 user group 3 Oct 21 09:50 latest_checkpointed_iteration.txt --rw-r--r-- 1 user group 24 Oct 21 09:28 latest_universal --rwxr--r-- 1 user group 24177 Oct 21 09:50 zero_to_fp32.py -``` - -### Step 3: Resume training with Universal checkpoint of iteration 100 -```bash -bash examples_deepspeed/universal_checkpointing/run_universal_bf16.sh -``` -This resumption script effects the loading of universal checkpoint rather than the ZeRO checkpoint in the folder by passing `--universal-checkpoint` command line flag to the main training script (i.e., `pretrain_gpt.py`). - -Please see the corresponding [pull request](https://github.com/microsoft/Megatron-DeepSpeed/pull/276) for visualizations of matching loss values between original and universal checkpoint runs for bf16 and fp16 examples. - -Combining sequence parallelism with data parallelism is another good use case for universal checkpointing, see [sp pull request](https://github.com/microsoft/DeepSpeed/pull/4752) for example and visualization of matching loss values. - -### TensorBoard Log Analysis - -The Universal Checkpointing example includes a TensorBoard analysis script that will generate `csv` files and `png` plots across the unviersal checkpointing training steps for comparison of training and validation loss curves. - -After Step 3 is completed, the script may be executed as follows: -```bash -bash examples_deepspeed/universal_checkpointing/run_tb_analysis.sh z1_uni_ckpt -``` - -The script will output the following `csv` files: - - uc_out_tp_2_pp_2_dp_2_sp_1.csv - - uc_out_tp_2_pp_2_dp_1_sp_1.csv - - val_uc_out_tp_2_pp_2_dp_2_sp_1.csv - - val_uc_out_tp_2_pp_2_dp_1_sp_1.csv - -The script will also output the following `png` files: - - uc_char_training_loss.png - - uc_char_validation_loss.png - -Below is the visualization of the `png` files generated from this example. - -

- - - *Figure 1: Training LM loss curve for first 200 training steps of Step 1 (TP=2, PP=2, DP=2) and training steps 101 to 200 of Step 3 (TP=2, PP=2, DP=1), which was loaded using the Universal Checkpoint.* -
- -
- - - *Figure 2: Validation LM loss curve for first 200 training steps of Step 1 (TP=2, PP=2, DP=2) and training steps 101 to 200 of Step 3 (TP=2, PP=2, DP=1), which was loaded using the Universal Checkpoint.* -
- - -## ZeRO stage 2 training -Repeat steps in ZeRO stage 1 training above with the following modifications to your job batch scripts: -* Set ZERO_STAGE=2 -* Add `--no-pipeline-parallel` flag to deepspeed options - -## ZeRO stage 3 training (**Coming soon**) diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/assets/image/uc_char_training_loss.png b/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/assets/image/uc_char_training_loss.png deleted file mode 100644 index 4df1ff1fc83ca2284f826369bb43185fa7a1e3da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 54558 zcmeFZcR1Jo`!@cR86srw5oKhRy~z$qNVbgZkv+3zRT4@lNwO2!dleZ)$R?D%_xN4U z>U00T_woCFkKb|se}9hSeSG5e@*IzGUFUgT=kiAn=8E|KjHsG?_Jfucf-l*o~N0sC34N|p0mBvJ^Q=nOdgi5Zg-s=1uqF-y3EaF zbMKzBn;0*z!~goiB_~&FUWyQrF1W~PXXRUN2tsUz{)h2aI{Pky^hK*E$m)2dE{=M7 z-Iy81U3KzMdPel)E5;K`f_9r*mLY_XW@IohWY1xT1{089m-QkcZ)niE#mT}n6~w3& z9+Jhv)P(=u8u!D!H!3A=W6~O3cQu}usXQ;c>l0Yeq2{`t>XyclS>r3M*33tUDTl`# zdRE&h(yr=M=vnyTwbgY*`OojVwf^rvwEuqwL0OXDtPx8+neSy;?sdzyIC^ns_3ury1L;)TKFY7mu~q{rTev; zQrRPOQ*DfadEw7U8Xtf6z*gQ1I9yYzKHgtIg04Lmdg(UT6+A~BE?XCHN$B_d+ZOPIyT7y3IjdJ#<0XRO zgYHQ;x7p5yTK@y*{mr`(-5hLermGVTmT8)*RqhKE@HY|C$Ge!fZrytL=}u&9Y|BGt zc|NO7s@>h)J`eJRRQU)Z=JV&V1+BaCmfn?j(vRP8@>m??S5FloN%vbn&EPe0|3?8) zj^a7wI|+W+sVa{pPI81nI)Kh|dE~y~rNXRdiBA&~!xFjlSj5DBSYXzVj2KF6bP9@i zF4OsJ&4;4a;v|zZUuJxx&|RUWmC*#S7p4`nvVU-9n-F>ZMB)4Iy5~ zdvnZ59cw-E#LPlM6zmD7IW*InaT)!BxbzApPL5YjY^$D?PNDy(=Tpz>lDzlptdPTx zpL~eAipu1P^SZ*7Ey<;?Sp?eJ+Dr+b45fs{#Aq&Fy!cS{x8CI(smWEz0X{Q6Nh^n`o-qh%XKFK4Q@BG zHCpOUj-wX_3X{YNVV!vWdKt0XMlbHs?A9$A!E?LJ6jq_Kvhqaw!OnPKF}!ATTN^=F z4o~TtQt*&{c)x9aqHp$_H&ZQ?CJH(_=Y;GBgH~6ak1N)u+oor~JQwj=HH{tUxS_+& z=MpaW^6k@wN;Nq-xvbjkt5>hCZfv;iOPLp!e=Q7CW?SDKc+}^A8K)?YO30pSePd${ z*7Ir`ti|o6;Rq5|MG=WM+tR7r!rI0nv!h$T;zXpq0)4XS?4a=SH^SZ%8lGq-! zK!OHL0_j~`T!M?a$=KEFS0|fXwij>3FiJDfi`}b_U<|-R+CP127^!gU%~T@(U^hTS zEVXNzQ$TaTf8+K?#};(^lsVj%*#DKOtfJyFUn=3he>vdf$TNmPvQlTYZF1ShCP z+Sa=>u-I9rrly?Nf2vd;ELZ*do=-b9J#H!it8IO@SH`^O=r@;!DIJE(S~6rqimblS=;!O!#vJ)>^y`_JP1N7F87QDH zFs{Kz2FqPo!_Ux$6VZ!hs3l#Z5_M(sSRVP5o}QK#_i;RcHUyVg@nC!DQF3zRtrDx7 z`=v%79qAuDc(B?fzJeDlt)Z??nC7?sNxLm@Z-$Z(wgP%j7=7p6)k$qj0YSmF{e{9z zrFaIvij}&PS{>1lh9HPZ=T9P=SSuW)>HfN`#_n?obD zzS?`^A>19J-V{7M&z3*V;3P{^_!;@(GKbz=Jzj48BE+`TKd8Uquz9YG z=~sE=PZ%m$Aak85A)%pI5HZd7H-E#nkoo91x_W%D(qvwY*VWz8(Zp3)9SraC3c{_| zQt5#6WFz*+I4<>T*IM#!mGqR_>D~L)g8>8Pv5R~2>bP&LVuf-T z?r@y|!Y9UM+djOZp`pP_cdj(Asnd&NHMD1Fg%qMGc=+tT=V=>pLPYpkn2??xZ=LQ> z0n^hILM#=DbB0C+Mz=BabUB3a)_BHv#aJ4%m9ZLh9{OzEb4>am76>MS*v@K+u#EpXCl6eii)zPW*e;BBIijt zzr(dQqK=60@K>;e2TQDpA3uJK)qS3WL!RZ)>U7{eF?#fw(KH4TL+Q>PHjQ+tSYqkJ zvoLinv`*E`A}0_@LK{r$fAwZUASR)riiGhXg12jg@XYTrr9ef9V_5A)a&ovqe1hAF zM#OK~nihjaW|5JRGCn?HW%fhCQDhwEJIkY&tfPqOM1oplX$ho!$wj@_*|t~4<6$-$ z#>W|Hg&mt=qMaw|F|crPF-=3}AXvz=k(wRuuHl_N-7wh{vDQlKB$L@|`t`L6f<(6j z=iJDig6BeG9=k>=UcPQLnhTRI8MzL&*c9vYc`h5Wad247d`|o|P?$ck7e+w2zOn&m z0L?{r?%Xk*ZjC(~rGN|u*`eR*&BSeuqwh#U!!`E!!F!#}Q5r!j!uRjrufiahetLXf zqppm%di4~|72D7xW1rI}M>~vV4#VLOAD*pV|IFo=BI-74+MOneW&$*rri!?ny75Nc zWuIZW%(UTDXlSS@WWEcWoTs$5y6?)$VxXavkmfR;d^jPR;*0M6)^2WXU6&s2Fo(zY zUcd9fX`G?3cAqnjLGl4_?e3{qTH)4mmuaLZut$1l#0{cl!(g#RZ?(7EAw&~&+l*|y z3OXW^INqJ^`F$^$&+JriNXXhZP3g&sxwNMBnGSxN9wx+j-d8&C*l%(8BeAB$8kWHa zoBM}{eyI}Pq_S3MVO5#c?#4q&LMk#KQfSqL{2pvM5o4>zd&*NF}Z_s>k}Os`zBIv>k`5O~hG0Hk!#a|6K7Y z&uy%>y8U@B^l^rYEpP)nZ7vH~w8of0{7-3YlsgW98>g zSfuzLY$sM~dKb#yGWK1%!Y(CM+c`HqJ&nC`dS@^B@?BDkQiHweXi8oygO83pHq&I9 z5)Be-&18quPIX65qhez%FBw-~C~t$skOF&PJzg*0paoEX;6R}X<&)c$_bz^`bhl$? zWeqV)o*H;-(H5t!p%Is{2mqpukX9&fq|7bd&No{#{m;Xs2kb{lZ$qx-M~B!X@aX1j zDLw%~%HW{>EX*#~PTDs5%)To%Nfq83?177bSI~DZahX;;9)#?hk|yaZP*G74ykX+E zcJ&P)7(emalVe|WNcYw|c-2OueGAJ6>>=vKe7a+9>@|t~21cOs5vvl`OvTRLBP9ki zz!ZUgOFv4{3vYDr)}iw+Dk|Cui>5>3X!Bd}%DnKnUlTg?o9apB z8*$P7G}po%Snbp$L);fk61y+%Y;@PAHkg|FY%hL-*q9*QT0^<*?gN`38HS{x+aTO5 z8Nf%f_s>U4yKRuGEzseC?esv}s#rf6CPZv&BQkdfM&&d22Bhsj3k`m~Ye5qm{(HPm z+6}&Vd9D~^VPR!h zcc-;~`7(1-1^9h(Zm!K!jZV_%oK;E6$-*OaJ;U1=UW(A~47kHhL9o8QJ~1~JVfE$N zB*ad)N_y7{I##8)b9CZ>`kGk$+UEILScvDeI}+CI|r!y(8AGG z-PYDNY-hmcGNNTG(G9z#2QuMt-Q&X3^;UY7o-1+icI(s~LnYQpLy3kr%45!DLxL^R z=Y}l7yOAmzf+NeB$RaA5BJJ_}>v@#30A-koUmC5Vgm(^B%_8-C>a+O^Bf3R1OAS=y|dJ| zA73@=@rfZnVF+aF%)palQ!2$(pUwGxg0QeKzkzp<4PK;fa`~ma3tWL;6 zfWfKp*>csd19*og;03!QOoy}OrF;Z}1Rn2ksI!wn)Nc>e!!Yf8dtHk&(XV1%XQt|A z8U$;@)pB^t_U`Vv@mv^9$d%I3wx1!wt6j6nnV;+H?$+e{u5~FX6V{tXT`gRiTw=K| zCn4itul*4#P$?=&pUZ*j+!Pj&eKd^5sUwK!kF<4P5%Q<)hC{LC7F6Xg2g**y&YKkrv%4%w4G2$*+1RsDOX0c<#%~;vl6<5^=5Qx>x zZ#g&w1hiL^bGd}O0HbBXCAHYOxd~-lw9q9I6oee_HH6@@0%Jl}$9z{#w|;a%DHkq- z7sB!gMG25QG-EZLFHlP$GCDd>CW=+FP6cUC*Di*IhGqaHN1+0`{So_}LLJVTDraQ7 zy-{C24ax^J!?ZXpa8DcQWFP}NGrD=1x~=2=T~V)7F-s-^IW$3-kFn<3JhEE z`gM};1xC@$&CLQGEzQk%3DlkmgGe2ELX@ngOml?5AAJpDXy!qYw zLq-Myi za4ze(9G{TzTny%y_I4S4ef^vQ8_xo%hmRiJ2PFGMtjm8{KtSLuE$u?zPQdo!5DL@w zw-zEG*iu4>Y-u8vjsaX#m0#P;kB=-Q16rBR|2QV+;UNTYx9Q|8&nMf`SFqdkEAQnr z&8Z!qb!nYT4~)~4+QW*M+P{z%cp1{2hK<%W1myhqPx3_mz0V5e%2!o`LPQ|h1qmCIqnpjh+H+{UP!_7?j zsOWG~&tqC2xP2D1ALK(N4UxcO2}oA4lD^yML3*buu<`Klf|S`Bqj_t8 zt-GFw(Vv{0Ece@cuQqpcvtsJ_g4b#T?pix*ya>0QG;`xEVB4*bQcy_C#?JnIp0MYE zameR)i`IR&&X>FacC1rU}*d^RtLc`WKX*0&tp41hJ( zU}A`loE7CVAfTwIDE(@$;)!yD{8`*~%s4?E@~F7D)|O~Wvr+@>x(fwHRqSuGOHe`v zpyKPK+tL13)3YmfMQ(E%vdFK~uQD^ugr39F&DTzehaKK%Ud#kqiNg-9h%*xmTAL^4 zc)1HEWLxqlAa9%oj`)slcYkls>{riADzSSPAf;M$rABp2pU}aV!sK4o0~TBb_(Nco zMoWm3R1%m9RF@cK%D=y5V|N7TTvb-aJ3Kmq=i~B|e0KS6FziEI_Jo|qC`y!ZPsdBg zq1bqDF1-z66ulLZ(^&PVZVB8$!nx5Z&)#fJMxYR5Lhi=#|;_S$`C0@#u z^536;uLZB!Sd~x%V+c)8XM{a5wONX6dFoexE+y>a5Eu`6_5_sq2MeGu97cGL-(VZGf=(;9u$_x3*AhEs zY(1KH7( zo1z{Lcprax?PzBlVtkK7ofg*yeIZ^_!*I2C$wYbPN-*pBx|O)6$1b>@ff}Rv0N(jh z)OP`G?ON#O_D{|gy+_rpCr{2b#DQjX75(HaJ^dHladQ6^9tyRb`U;3hc*MjJy*Br5 z`$t(k!N?2wJn!><0N6IF$w3fq-P@S!EwR>YzauaLtJrsE8Rdod(NyyC<;!Fd7na)H zDFXH9LTAuz4V%Vgchw9C9KrF|m4QM=-F$;ER2_nZSbpy}DJUqnZ1Zr0nf5_E2&=1; zK0G|^fz|HXb`QNDy2MjO-JF+4L=}{jEXHg7Mcn7l&33)$i2v=H1fmhrEwMpp)tP)B z7D5JSCoqwR8cZODINN2+^GK!IJLLM|J#*QA_}AT7K6>S2zdxq_c>Q0lrMiN+8; zV{dHOr-~qXsh;6Vmhct`I9L)!Um;#%9S>Fie%%u@th{#`5`I%*-Lb*s!{AQ|9ITfw zp93zh5Q1~YdFjU$zl|@#zlJ~Zo{VjSAS~{&7^Iz}0BhG}wv+l0c3z^fc{aozi1!zb z0c{gP78yU?Y;$=Ec+eo|30akRCINevbmAU_AZoO}y`I_g-cs${xpNbM)X|6)$0$vN zcm{$|$!UtUg*&wXAe&#;0N9mM^emF^@r4+k}C=MFbiI{>H^tiaL$ z3e~%NUx3w`gAfI)857jRJxf!^k#dX-pZ9Tzlgr&XRcGi#&MpmqY&%$~i-$RT>Af+# z3tM}af!^Ar4KUsYwWKdPmT%Nkf?H+*2D@ZLMIZpVxaPE)g_S>R@ z9IB?}>gJp}R7hVN7OZZ9@KWPLfX z0Khc{9j)JAvw~BKuH1!(MP+!na6Rwi(Y#xN)qAtg9bbHEL-~ZCpI=75Eq&S0zyKwx zcg*+a76WOpAFri{tO*a;0>KfR?Cxg)K$wdo6=bkN&O^R9OG0uVxV9-k3BMHKF)vK> zVk&XZWzc10p!BgelVtK#bi0Qd6Nj)KW!`xvbqsaK`&=NsoT@O4D=!y;%PR5R_<9Sr z>|mJ#ExIc~uBX0TW-qIyb=IKFJ_-oJ#ta+jX%ujEOC1n``qYTx@^j}xL6mzcuCe+u zf&tZ|g0TrER#tfR>m0_ay$37YIG*yF5G8yH1*AIpM?=?67jMhEu z^5VL4#8ucx^Ned~R;o9M0``A%M7EftiC)mEvmOZH3kH))-yO$@HhAVg00Cl43tA&^ zs@=oE3BdqmdKUmXww<2tX3Y*P7Wf-fAU6e(+ms6D+Ox|Iz?A6=8f8Y*n)`H3DA#Ok zOKFTM?==wcRtG_aqo6ETUb$u)(&`id^fr)(;yspr^n1A5_Ca)9gG@aQtI_#3PKne$ z%mp8yBq=*}yjTpjo`tQfI2Vh|N{=OjZ~e@neK#zRxJjd1F2!RObgS0JQAA^-Sxh_U z_2?D6_{yNuj}sD5wdSQ}x)ezKO_0TlfI|RPiayq63++pGU-&+`I8=h_-Y{7pu*3l7 zw&+aeCmOI}i#r-WIbz)Okt8K0HN8wz9C#9tD&yb}++YmBAF;h-ia{9Uj0$3vo0>UJ$7dcQ=u?`wt;jQ@H zX4N6`;lgC-*9Fvk8|d!rY=&j|=;6a4Kv+|;!ebLVFWJyiXb!IgCsOUVCjiJh6qrQ- z@!;d_<~H)|lnXax#nFdW#+;iwi5%vszRf~jy{>Yq~T^6FduAc#YAlu9oE zsCT7F-hS5!Gv3K7aO=C(VjL=bCGnalqR^T$Pc^IY=cj~6j~@p^!mfIW{iht8)}1L3 zWa&V$Y)h?#4l|6K)GlTa01K827w{mByz9Gi8hz<`miW(zZZ-<0f~A8-Nwc9EK=iT7 zg`$bC$Y4}BUXh!kV#_Bj@`wJ8x91yy^S$8-a2oauFt)myo7D1R&yHZw=Her z$vuUMxhF}=)nsd#p}X)-`%0PuOStL`LB1|=bk?Tg>R8bK_bT@3@P(+*Q!?7vprki6 z96Q#$QJq=%zTnMbYcV>vw1oLKcC3DQ7}qP1wx-6vG;0hih%@o?fyPQoXF-JnI+hZU zhunJ(B||lg>d`Ip+jc%Q|HFMl>!2Fg03xjObnBVpL6CPJfhrE!|GRMuTKJZ;O;)l* z+{NhY*wjEczD#olW}N`}^`n#v!in?DXL67y!ldWEW~o5}<@NLT-;=o{DM=3rxe=BY z)KIYy6n5zr8pi_upUO%y4%WzSdVF53r_e;&0Q2EP&i4lSMf#nkMrS@28(i&KIL}9O zrbxfqEj;Vv$KJm~#5F7xrC{*;{Pa9oJU-&)>beS{gCNc&9s;Tz&=qrdUHuBz^CyR} zSkTN2KxG=%AZ9&mac+u1CpYedPbjrPb1LjS(%0G-LF^B$aGU!s(}B09LD(dd-xa1Y z=YN~i$~xMt)W`E}E3N)2`=G;?$ymW^m|lKWaBPTj-9zhL&O>tYk{ zrW+O@h|OhhAb4+V>~N_gFagnujnxa+*1@*`vej#z$V} zX7QfxtOx(g%+Nz6Bd1*RJy&lIzoYYoIHP<-+}pmbgvW2{ySs_ip873*^^TVmIwk3| z*{H@_qnmS3Ip3F4$8Y!j@^Nu|Jn^enulW6To4huAdm-^((T;!oz0f3Z-BvB^K69zI(s$ zpsGluowHCUwj^z>opbKdqwBv0oDg;mKbqRwMO|wZl^f2pofi>%94>Bdx~1VZiyt4@ z6cozKEczk!2Ok|IfM|eyj^bVK+v{Xd|D+| zLss5-+#eHbKwV=j3Sio5e4^@kbMt4DQ5((il73NDz+u;INl|rtc=OV z?$IXLEfE3NNGiA*dI5pBZwJ{rYhvxgUCqhjw$_RJxZ(BoR#g?5nY1_e_j&6ivb9Uj zQ1Tul7#LTiE3B4B_H4eszWBl544|X1nU16eQ$EwiP?amjln537lo=cQ_5|Y!jEnS1 zH6_>pL<7|?C9R#rJ$TSs?H!HqmHUgIvHR{9_xkn4-Pz|WJsGlZzwL|`Pb*Oo`lzQ# zke{Jd1dv(+ar_G4!Z=vkdhd8(5^*qbrI5_tpt=*OH~=#sZ_yJ_aL;x?#TP@+sv61d z4~8&ly7Z&l`S?)OesGls08wh~!1w&;fN-K-zotBU?pzERN1WNG$DR)p6Qi=z)3kH; zHD8Efo<98oNcO`NZ(OA~x@M?%^yKO>!WMh2V_I!Lgui>h1+gR||F$p`FRd+x-jge< zFD`FN#xtG}oE*C05KD=W^4uN^>FgBO|G4Z1S&`Hvpn}@9qFUR5`5fZ&`cMw@@kwP= zz}j^@;_cj}`Lk2~Kgt~R=Jmb2u;3pivy1W}vczGH6(+4;o?R)_sWG>>eRID0o$XjP zwn3>aCi3g+c@jt|3a;xXwmb+Su~X#gYWU%_McZ+xKDWniEk=;s%Zc>p_;@8RFE*=A zFA5+R4dpImskJMMLxlta>Od`Un)Hi3maLPdj`mb7(+9h6iXZ!kiVh9d90Yo;wdn0F zU}IseTK@PL9}yqVOTqIjK2s^M&41_2y?tXSAwSB91H~4!L=2GALRAvda{4+4L7XEn z-f0MlFmH=vTx-CMF24H>M`G{i9ofC|^6!51Ad7@PVvT@$M_J`L1SHfFzF>daed=Q8;{9yN{z_n=4-YV^elQZ#&%cgN|MF4 zE^e-jV@e+uok3QIthxAHrl<&M`e=!zC9pu0x$5CjHgO1x3Men7s?o}PKbO~X#V0Dh z3=Vd6s#p+V@w zoD!>R@;*M#I=0?E03eKMTE%ZPd7-{)E=`1wFeEi~@hgYUDN391+84_TDenfPg@jP3 zB0Q;dU04gga5nhH9Tg-R3N3uwBdRA$uTpqLQ%kkI7oX(2QVW|Sn%;{ha0@pT`z`} zRiB+IsW{By08&D6$&U;IX8lx)8W4?OsH~7(8|6iG>h|$jJ{ak3&F%U1lv?G@tk6iQvPInaG&Dk%jA|l)GW! z;i5*f5T?OqV9my{a&Ir-;zjGT6cj|#(kH5YW%dA`fmPaa#tf-+cRm!Pka%kM8hctLQ3XqI#S-2>X_SN z9tQRt-Q0%dQE?XtRMkh@;u`5rnwb}yDNXF-^|?+@AIH1U=DY~jPgB+3#%UWj{=_l@^=?@FJfHWbz^D`|fvNPOq( zUA`m0!J))=tHrjo-d>1RTAC56g&?|>nI4*1j`@O>B#}#6*501(0~{{m#VSU;xg z1=(I3q)4p`=h{Di=KM(X@}-&ID(<;>-fG{ZrB^l1lhO#7hy1a3=7&mzA@b-fmx_Xn zNh%RA6{V7O%EY?Vp1K1!5b61F@p<1uP`udST?wO%luu@&}+Mw*}^s)2l(&pA) zmSREUcskK`#u9`4-RXGN#L6^~dD%779?%LGOs<-3KZE)P2(7_G)!;@9YMJtv&Xqb` z^#|3D1avxqB+ADH4xmpv)oh75j#b~c?9jz)rLO!IM=w3q7$|@)&+i_rOz+O;x(6Fwu$AAK z<>|@p_TT-PkTqC)Hj|&^+CRJ>>DsC%i*x=*23?@hf0hIt3Skl_@LA3MY;B^!U zgNNYsPuHti1>o1|HtS4|;Bt4g1i_Kk|A1Y$ys90v+WSkxf%O*gvG>*4+lER`)=%!1 z($tPX7Da=7QqtX=JTUmYavE>Om$vJ8THLVE9Hl9n+mAoreTEttoC!-cV!6P>LvG!D z936Np8VfWWf~-jsE+e;My3|g^hgKGAxG!Y8?!?=WLlV5^UVQ8=wxDco^CbPi%EeVO z;-<3qmS9kR@8Gj3_hp$}K@8bMMmL}y45L*$jc(mvnu(?TB7XjS0X{j*r`&r~fE@Dr zeQJ*PB|sk}%`@17R}^LNG8K6y>~xGWb2a*@>CET& z)%}GMmu|^ouZ=I*$)awySp`nT`nH#S+xZBgJfHV{y(%0HcRLJkO9xz}eWB5&$(Pd(N*_`* zx+p8#0P0}V$9PtEczVkWO{NAoJQl406`N{ns zC|7V(aJjmbl2hIb*5OoiE|Okf|KR2J#iTqjt06j5Jj2Lst~S;EzCUfQfKXe4w=ZqQ zwx!i93l|sW85#vS_Kn)57UyBVoOhxAW(6g0Y)1OlbPLU;9SqZOUxsAY&}y02kf ze09lGE^>p2;gCEf#j~f{TMZb(;o*icOzvH7m~Jrc#uBr zzgBwTOr%HtI?h8>9qJD|B}06bS_U=X>rmn&lIK3s8!~@6#SRKY+ z0Jxcg8s)Bl|1G$Ufi}dfiQ%62_}KqQd>da;^?jNlulYc zgP8c25?ObpHii+L$bzYFw6d`G-A&r0LV2LPn_A#gP&J`mzoKa4ZWW62-6IyHlmCtc zN0=xm3BTLiG70e3$l9hes@}!K!g|Z;KmPfdxTh400PNLu<;+k&amfkd=Iegsf-$X+RRUtm{IP++dDyt+kz zuX!D94e6KJhjkDnt_-fRzRIM9HiiPQlz&w1fAK^S^7{0tdbtg^4i<@@NAm{M0PYnJ1BpRDo!7flT_|b{DHTL<$roVM}06nnN!)d4| zgqY=_@2{SITsBV4u?$(?efj5v}ia>8eT+|wR0_sEi(*7^4 z{Yz^RPV&6R4|;Rw8=f4R_skj;oMzs74kbnuryUd8-8Y@H>vbgng@CL_y_8onU7O0nBO+>e?o|6AIQY=DyW`83299*@mce4O zV&_R;@UA3_dzMWEfTglNJ&>Vc^LNQ=EA0kYgx&3j%d);Y)Va+Gfh5T3z6`P!nv$Vx zO9oaiB?ALaAj@i6>~L2G-#pi8t``m1bT41p@2o5hQ$w}Cgh}6BSM@V>`wXCE-Q}$d zQ3Qu_@=d9i+~)hn;d8;q37(aHb-;81Pe!Dt2cEaRU3ST+l9_{p7IN05l9(2AS>P2X zQph5ssESJBLLB?ex$8#tXuk_sf!4skF#!f!$flxD9lH5$1j>S_5DZ4+45$f!f~d7Y+v~(rP_u5H zXprxa(9GAhXnAl29tj)qc-5MB>l>8TZ$XbtJany0u1;d2pa)uj%#fhwW(F$ZW&Z48 z+hToC+FthC7WwXIpaBX*&$w@M!{2Q1;SHXjrK2N6z*-EJ8I1Su1@;_`q7$|;seBcB7YVA|i= zj3}QE4GSAiG&2=?-oQr)-50?(bDzrR5X#JT8DVdEZV4WDOIxol6dpW!vMHA}Z_Kb< z)A{KWiMnpF0T18vFJSWnzx%?YgajU{Kz;fZNtZMw>>v>1TEQ!b`mmwJBVu8E37q)= z$tlo22IwYX78RuiTl3(DyJX-X=BjRkMy}}DFVSs3ou!i{Nb0CATOum-@H`KWcZl** z^lbBq9*6JL?=)065}&Ft{W8{@8ED!RzEvJ@#QDL1Cs5+bmQ@8thCa9ATvycLbbI@S z3+M15dMDnF=;`5u+R=kH%CVXbk6z>QRYoL3udp`QcWZHo7V77mJ3mr$Zg8AtkX~zk zt$n|+#GovO{4mDh#|LTzG=^wm8mYaXC8|bM+`!M}>jGK|{cUWhX20AHo&7R&tDc19 zk;uJY$!Y^&TJf5W)jJEB9TtsVrDM8(*(vxhIB+COjm6l1>x^tlrqapTzB-p_c74c{ zp`n)%gpKWkdVDVu5sEtE(2e+atMGzN!uRJ%c_SR=mL;fH4All>M47SodgHn9Efv0sk z2r>fRW}AVowFxuLCSIIt2{Ov`EWy5*yqXi!(>FL5+(2*XOxYmy*;$U}Nb@3fB$%BT38y6B)0h06-nJHvF?Pl(8%N&b+AeEexlY^EpjlE~c z8`fZsvxhusD1tupJC>r$X zjvVSw9!M6j#7A9+VB47hH&cXhVd2xtN=#{-hfueSh*V&Syz}74LHCyilUsy1lCQOM zCIY~6nDfXt>HMX*7i&I0+hvcFK^|ZXq|52s0t5}s5jx;UIuk)$GC&9hrcZZBJ7*<< zNO4oBowr-6_&?|4IZ_OX7{OWCeb5yNz&ei& z1?%wsT%+_eR+^mdnsi@yC_0Wzez#c(ZYf3&FJB*4E$gKP8V2Q7nKk`4>*k-JT)H;u zc^f=mYAAIV@fE^8*gij?j@<#IiD2`$8>7_Tos015AbK=|g@X(vq@4P}+SnNfVW~Uq z&VEN*Nf4vduB2gE%o*c^$Hqarx#WGIiN~$*OnYeV9qrU{ReM76 z?MgeMq==>l8VC(;vUC!=T>G09-C0*CL@in+Vj?5k1={Yru1ux~ay&{(idYyJV(S4< zmGglI4pikhJD;kuu@vqd{dRQcMDiYhdXaJM`4q7#c&JGj7~-I2%mQ7k;2kRLb8j&x z2@V+*&oa(3Z%f9a{88Dfuw19)F#Y4#m+G2W)huEFqI$7Bd$sHNqN0DkH@^MgSX{}m0W<-sS;5>}+96v%jk^-UL zYdxLjYtA2dd}6l0X-(g!H>jbsQ-s!^Q%_hwRu6Rd+UcwllNamX>ChFJ8QW++?eVH* z_6uz>fSLw3@IrytYq9cyVkQH01jx$XCog;XYUM%ZKptDcG0XM6z?c^zYm^A8V5mv% zOvhL4X8FVVk&~l&;;Xs556-<)$Zg*c63PL508_ zVwGcl;YU`#dPtH%DQR-c(M|HRph1v8Daq}EurMW9z1qM@18QHnYX3L$V!3>e?m@~Z1;WJ7l(wZr z6w&0uj=B6!M=qgQ7gpCcvM7!7qyKFcP%NzHrni?a1Is#{(&^hec4{K9!hrj`Yl($> zOAK{4>WprAFf~TxWM`prrN8#~w{%%xB>cYHsq46&H*Uk!d zb+3r+dKQ`ux_kk&7JO+ut`Euf6m7%^Ofz zq1ob1vhd1DPqt>=kAi0dk3QWQtUJ+i(98VF!ou<}_0Y~iFQktY91LKmAfh*~Q~&+@ zH^gt$H-Th85f<&J1NRLKpZ(>#U$MXgt*oJ;2(4FWm)Efve47ep;*DlH6oJ{0dPr_0 z;4+7OxtbtT3td=*bRs<#m@-j3$OrVa0vBy_b5Brr3zuxq%1NXG`~u+;ZQjOiD{q5N zr7NUlWKhO;24iKOQ56L=M#`30eSscbRWKc`&cSgA(Jd^4DnHYFmm9$14M5YR{*C}M zG;o3;z3Iutv?^j|%RhpQHa)-&`@Xj%phf7LRb0Q!N^gkDSR;&)F$?_154XtzPEFBx}?OfwZjmE z5zB|HSf4PrziNPZV_V1voIxv27M8q+I*IA}8d}pUF!EZzi z2>$41%P?jUnV@Q+A2^~Pz_5b&)jSFh>uiYAqmJ>%1vUa+Bx0q%`dT?Hb`XPx+`kWE z;Da_EOnFqS`_ILrLjT^}g%=*%a7}>k!x1?EQU^Cx5?H1v1^_E+Lcrzk8#X<9PYkr}pQMI4DY-Nd8^23bgmh3xAf( zSx=A(n|j&M3x@uA{5X$O=$#_}T=6YIGWyA%u^@~7_a{5A;7?T8Uo?8Ly}^NSgrs5nq<^wf8RUS-sy^rO@+0xXJWaf#@h=l7vn4bn3^*!Ivq=W6J4 zg}u6LA3WTv>kPNXa}kD}49#*e=y4A0>^K0eob|0BIieOk)ar|FDrm$|R#j~T=hNq! zGN9<-AqfFj54DY5Q; z%2@)36+r(XA=>PUplu?kcN;a0mI8|RUV?^~NnpBY#DFdw3Z*Sm{AeV?qEE`kO6BI z>Nka#yqQ1MAK8r_7 z;Y!9~6X#zXSX#>E0VqRANQi};qo81ct|jRL=soQgf2Ey6cWF;sODm|oyjNFm zai8>Cc{ebagu@Gedey=Z$+9QB1ifS?%@>GQ1An-g^U=OLG-s8lZV5aeJ!tr#pO;l| z*VQ@ik{cNPrXj(B9r2@_TUW3@lDvm3{unJM$`w>OY53Mah4Ckgd+q6$Of*^2+c z!>6m4n5%C7ocbDpEUJh%8^OfC{q}qJ{H?FwzP;?V*)=WvKTkW*VqWaCWs5exqgFa- zSAWWVD>&e2-+PfJ2;sp3S4HUAosG8$9|s36=r7e1R6Dy1g>{PN<}7gF3egLV2w1V^ zurzGG=TYkx7)HSn1jlsH8C9%bKgg5zKU?+Gr#YjZ-v7Jry>*bmx7S$>p-HIG6?`#` z;MF&*)qqz1V)K>PKGAaK7MK9Rz&`3rAfjDma@00wAWdGSHXoQaY&#R)3_+LX@Ce(NSR~-BosKMSd@u5-u$~OHTr1(^{&cnQ*9VuwfX+UL|o{zL;jPS6fjKbWeza@ zEWH;OkUY%hbAOXlEc(NLLi;PEh$w^qPqK62xa**oApi}$L7*f~qrzQJ1_p!w=5;Vr zW0FR_9@#*Tv4CEtB(b#A)EEjL106m0T=oCtvbSB>K~TK_h4LFe+FG~yRjnM3zK}ZJ zwE&P)z5TCAr&)a5;3FE8+xc)U%l0Bghtt;RrT7{54pq>9t&*lHk zJi>3?FftlUgA6ZG(oHj%)cBM(Zp1T6$L;QVp$D&^?fx36BG-Ag?s5EEDlg^@DV^c8 z5;!6Us|^ky!bC2>|8_<|`)r#86FRrEk;Z|~D+K;aMV_pxK`|J_Zp;3m-S+nlAA z1SK|7s6bKyzDHF)Xk=+@YBH*P3YC1cCkj&7%}Oa$(7rH3l3W5xgQN??iROmwR0=KfxRXaO|=rxTjNA=Z(QlVgKw3r(GP(2Msr zQ7q-l@{y?lw{F7~2g!QAlsv~exaN?e)6f4l^jbHKL;vmv=$VGB3h7$`0uB@jiIWPh zLu)UXnqC2S>DN))nV<}|vLXGmYu*)fSt2*IwVe-lT|PLD()iRcz_ACkaM06z2!Hsk z7yzj$hd(;CLF)xB0=jEmG3e502SsmhFSvRd;n=1qXp)^u(3Hx_pA`wC2>x5@gr%wl z&2i8t0fr26Bm*)tk@cEEzAkh$AG_%5Q-NS6wf~D1icU??(gICh`bDM~00K-h9h?*O zJK1gT{!7R5F9;C85wXzSSU3m3-=N z+#dAYenm_b0?e zovIg;VR@B5fsWmj6r6m8;fK@nZ(qNQgHMD0g+uY{h(2zQT+g2k$dR;j3QSud^(jF6 zZ6LRxAlZ1p0S`29;t(^$pq&GykGJU4F<$?RFPKS~_0R^v49!mHkynTi6CE5T0_y}5 z>^Y#&(?G+hyt(mzJ*5F91o_>6Mr$>7^ux5ibP7l;2dP9Yr_ z8LC-mTQaz|2Bx9!0RjrgNwoqC6y-fl&Yb=i%XKfsW2h@D*8_${PZvRZfT8IF8jL2= zK|Bs;l2?BBpQIsPk&2Nj=_>?AceITaJst*1_l=S}BbW$uc%qaOZPDC*0L|I|64$30 zLQ|M|w5bDx3b`DeSHR;A-Qct|RGdbFSc0x=S#nArfm-24HDULW;7NKFd{ z5n}%(d}2wMGJ%1C+_%d*C*1-90z}}<3E+wE4EeSHaYC1DRURw{&B6gp1U_J*h$cCE zR%=F;ggMGV{ro@nB(B4WbhfwGKOpCFhF1s20R{r6&h$bUU^{;vNc@ld#Jm-oe-J4F z3a@Y+9Ee0B0E}Nj6DQhTi$F=-4*Ify4~YbUCq5Ilj$@&{|J8rg4t>2Op4+Xwhjf&b z9A3-N1A?Bvf;Ir%%+;;Pe|(-^#2JJ)-ZseQy&1BY=wV`TbP^6U5r@qI>pFvOH$_Dp z;1A)HCVnJvW{pW{e?R?yU{FE_!S)L{uLlk=yFdiDj`ss^dub$iytTDSA(P&4*mB4HPuhoFa{2j} znXdJ%tu3&(6liCCMvt9=c>Roc7s?%Hz?}&;Z1eS>pAH{ij^j3|r9%&kf)=ztMlM=N z$D#15W^jjo-MM$D^9Eu-c|r$uJ=*sQyb?ka_f{%QHrk*DQcXGf;L?G{MH_esde{b> zo(GLmzA1BAaOfPQaVmj3k^c`{?;TI&AO8_M|NaWR>b{2eZIf@{@wTe`_se2ah>aWU+?$p^<1yHBS4V30Ourg zRJ(0n@ePgtsXv!o5x@kI{PbYLS`74;k?hy-_Cz8G?Tfu zYR4P)04-Nj7{Jrk+RDkpqvRaP)&e4WG=4mrb`ywKb$;h<_`QZ9PIadIuoU07rC&}g zeQi*n)Be-@F$=;hu2Xp5^<&xO>iz%V80_RKMg5l-*h9RvM|Vn~f)WF4*CoKRGyTPz z2?Rty*xXz`z&zGexJCm+oYJX&7l9AqdwKCW3CO@}0*0}TVdUHzSoPHG8{4CHpR^mA zy*tONy1GS4$o2wr18|PpLu^N8E7yAcKCo>$e8&}fxFx=F_^teZ><1rBl5z_P6=)Au zfu5#7uap1~W`&w<-QuoGcky=PdkI2K3vCgXUU$WUI7m<=ao)UH-}?)A!e9`fqynx3 zuWZw=qB4D!+!3Yt>s8$U8?qWNS2U7*{FO?s)DT6sR}FRV(t4VvPC6UekwZ~DdKG@yt8TN2){hmRc*5Fi8mJs_!RuDh?| zA4T&p*lOv7G+zJ@G%$rr0j&nx>#Yv;oi_Fuy;vI{rU&pvJalVOQo8{9*4>NHRytVHw}` zUii!4&w({ml{J)a03Zlpn~nxA#BFf8fnB-Gpk7k2W%>4z;pqe3g<`WAP^JJ z*bQZ?&;e5y$iHC$sgS_z;$)k}uTD2bgw<(JhGLCg4x#_UkJ#~LH82eWOYCvs4{$56 z05#^ibV**L|Do=0kS+Hf7>&XI!H3p>cC!VTgYTuTKQZisZ0*ev`#?yHZ}~Le0@T;n zx8WNT$ViDVjL^b!V*js_=!<5@9gwF2j>AWHMm|LWv8*0xOE{j;!R6jW|i1wQYkY40Qo;C6Wqg* zCFxK1EF)yAN{G!%X2t*BB$)7!N0FS@pZUIyUSTBWg1og}A(B~?!SUK1u&>3|;>$FJ zpNle3?-(-#T@NHudhV8oeBWf@U))?%bvzwqc$((Qwo33&qPG=g^vfJ9HOL^ zo7xnz;oGk-r;`US565Xms3r`mTT;8AP@abW;1v)sa)+;n8Lo#}CqQ7Q6#`pO7;9%b zWuV&Sy8Rb6TnO!K?6a$#uTuNU#_3Nz0)L}$Gfm3~2fRJ*KQjSH1@yuN+QFgbXO@w$ z&jZkpPy2(b_uOG@@5rH=jbsJXeBwKP@!?cxN57U82{%+1!DXPL?CP$A4+_2GCIATu z2m}CA9C$(eaX8n?_nadis>!S0L4znbA58?NR9szzsKtZfq?3E6Jc>o=YhvT-=PUC8 zkBlym>4UEjJGr0@$$)nC^kB*f2t7%_xf2DVYJk231_b^iMsjA01I2I~#*HfbSt5Hm zgs2SKe}mx5H*;uH0_$68Z*H_kF#7PHrc$Oa4{`?BBG7^y0ay`lQwAh)KKK}Bkb8oU zy9K3>f|_0XBf%U8#*%o`42h)qmCNuJ(EudI%8^FR%9HDkO$JpN`ET_Hk%Qy}=&8uX z>&(z-PDVU_bqlMDz!%roBS3m7rZtoX7-epQx()9d2igWs85w4LS_dfhKzaIWt-$FE z&udlA#4uCI$M1)d=TT@<8Dd z-VF2I$DQ1#L9c725V3`F|LUo`Z317Yf8#CNz>N+_9VQTPgbz#wvV$M}wQ;m>WW|?0 zQJkt_98XO`Rbhqwt;8W`4dQ&&&|sGery)Y)QX(!fg~;$_tyWEGv~oFvRYZlhh8q5D zFhdbxnM90R0r8Cga3Cq-M4|qb*@*4imf>7NId?l`j1BI22 z*Wb|8g=Cx|B9;Prq)OyN&zL=92}kF|Ajw!#V3OKaj3o|8&X$sn)R}wL(i2{P*Z#x3 zx|fHY4uFxISyomS&msX>8-nnqISBn>PXK%%sN7yn38?7tc)?2tpvmodgJdx*f{qQ% zzeL+KjIU^ebQAe-2b996%mWTp~n-9SJlb zQ_f%H^)E4p;PI|t4>5_}Bq%!NriONY#VHg@Ddb;$FQ4(CW@YpssqU3Sb{HpkneHBg zy%VtEV{1jkE(DDi-ZdyPl{6ZPst@?mhL(H7ro3R~hh}hGct9Tc)n7aFUgsFK^91I> zyg2egm-JU2*$^Jk;!g+Lxx4%4Z%p95R4_F)%@I`W{vdP>5xY*HBfv?+M2uDCc(Q!O zf1CjA$c|BpkT+57dpTj&7HshBA|0gpuZ7W+tvZ}?7%zc=+zPitOVaK%BKB30JE7fo zG@{9u-lY;Y3p2Iczv$x0pQK=5WGk_n`41LcU>&$P^MG*Uioafg4!tWZL$fJn1}<U)?YpG!)fOk=O0IaBuA^CBB=#>yR^^ z-wGVMr%TjR?dMT!PpKR&pA5w^q8_D_2j9#hzg5To)bmt-Gz*+yU0i&a%lOuS$ z`wC+O*bstsoZ3f_(z)cnb!CJEX(BEqhMP(nxcv3pE!g;$xMilX(jHJ|6~Ulmu%qLO z#5|jj3^6?t!&;HUUr&(L%;>egZ5|ODG>Im-l2C--*w=1+B*5L_ctOu~KzV@z%74hdjiL@YmorRY6H^n$VVcJdD{_IZE^N=vw! zcdoFX@%w(hYLmF8NOaC(q=m;qJBPFezw=+||C%`ga6lY^s4i`fV2N|0KW1m6-{&6Va>N7d&ClK{Au#Qi9eQR0qEd$J zN*(CN_)$ndllq8niDg5Z-$|0xboiEDEl5GjZ4)<6#IP%lOU;V(YK`AaD^LnW2sP6tM~X>-kh2Hb4nM@L7jxli^(!${C-idZenO*pOuV!n)k1m3xX zku)HJ0n2JNL=W@k-eXGlpf@XSg5kXEe;ub;{xNAe&rxBc>_^SsW*tsm<< zQ+e)7cF3Gjh%j6N3poqkYdJS46k{aDUQB=?H3c|nVaS$?S^qXOmxBI6IH(Ek8Ui;h zkWGXA?2@LI)-LEtGePSI4l?uLm2Hqo15&7!Kpb@1i-X5_N(5nRbYnx}c^t(lRKn8Q zWR#NOp(YA(nwVpoEGpwgDF3vY(fBar!1)}{O`$=VBtlI#aNrHxAv39QTW7)($or2| zdmxYu#R86>RuCx~3FJNm5WX}9X(#er{Bn!m8vXq6QMeBuJ_P2FHUNc01Ng%fRPzk# zocuA`YiTn2W+{W0_^r)e3i@o8JsmPtmA}O{?g{4~WHl`lt=yxPXORmo7|!fw{Sqdk z<(fzxIHWmmH@_vV&w_YvZ~=a50Zxm|PE5#sY;8yb*jn4x+n+3WLWx!{O;mKJL#{)#Po;hom z|8uJKbaP}w?wp}!GCl4R+0Y=t4Ahcx~qJBgR;&{Sb+YX^sIkHBv}?*0GiycGhR zQUv3BfWP8n3;l#-WiR40>_BUT9~JPim;p$vz{_3p`J0N${|OS;6O?^V5IH0DMqCVF z;%yHm(N$0h3CfMwK8tv&$_I#e;lhQHN7|KL&iAZM@eFu4ts>c(Ih z0=$u$Jp$xN1J5nM1|A9i-;JhY&4*q_WZvnrMkjZGUAV@(nmY3-T|E>7dH1;2r7EWh zh*&-6a`7N8ah}&di0F9kkAqbBZ>x{%n?=%$jHQdzFP1cu`gDAe8OqkKC-?_mF{JhX z%4hla!1LVZoxqz8vfJdEN`L=$2iO)K&&sj01d-VQJOvK4)EoJ$Vs9#Q3^VQN(5efg z=XI}ptoAmnVQ`@6J~_EN)LgmzZqkvKqzb9_R5T?`f)dNmtYisWnfz%!h+D*AVX9KC<;N= zksJ=Bbpx@%5%5L=Q+jVAut}Eag~MNgtbt2{g5&_4R0ewo0Rh1v$YTI;k@>0;*jvfz zG4zYuF~Yx&Ij0)FOmm07V!AsqtVLUF4ZPxdpi$+~!cZz` zqW0v^e~?t(UBDnHanXQ;?g&7uB^WpYuppoVTt`myh?l?WJmXUyIBv_~@?h-~QG&nA zK3>H-)&Ub{%N^&Ssw6xrO>UmlBaSur@WBEvItANKw!McVlGB9)1@GLAi$C+m&KTV$ z%qd2i#gQP#yEqKd%yWW^RZ-ujEQ##GL47EiNSgf*bj!8}oPoJKk2qF#uZ-y{V}v5f zjy3#N#b{^Dxz?ZVIcxG-W~|AZnh*5x>~I@$<^QxuZ=wir&kHONB^FH1DyK5&LRfI6 z!yLisQmhJ3?BuA{r=Pg|C2ne2xE?v3=D!X&vmczvTUwvUtD7MW7&?%kkcQRB1wjp+ z($nTf6i6{bjyIf%u5Quq?Ppm2)9b7N!XhUAEg_en9Hc$r0dq6os3zrXkQL93P?Hj; z0mlqxKx{s9*6>G+2_odbyP$^uGQgZQOF`%?4evw|WnC>bDgfG}3@ZW=#~Fn@#YoN^ z_GiHe$x`9REkxMqAo|j~OT$tuAx$GlDRH-m*kZ8^b9-hkw&5+~X0T-Ldc229(#SOh zjvx(Vc|E$ByYqFGy32h2?Y*ujrSMb z^p2isJ|r&Qiy4^hZQ1Ccn>KI(53BX|Ab0HZ=oC_+_|W4pcFTto1h|O2<7#I?4lbzh zZApaFn8lb~v@o1GbI)H-h?ek~cpfvYSjxVeC4bWAI#vkn45cjBr)n(~t>Mp{i!Z^` zh)yBVe0V5=)vWV(clbt4b7@1`xj~8rVl07S;PIKmZd`Gpvn}KCVVs6PT+ttuA#A+@ zT+QmO3h2pN@#N3|5mAY}krEvmn$M=8!P4A%yBE_#UR#vSg!%qwQRdaFhZWtEOOuw+ zg*-LU?sy8Sz2(4pdxMN81NNo0+Pe7fMTh?w7rGnox*%6095k5YA(N>PaYf}##i{C~ z5ryp87Dx8X$XsIMx9NwL4dm7SY;g3+K4DA1_?M{dAM1pc>c;gEPwr98$M`wb8Cv5% z@-@II27-x7<>{;5-FtRQuSxi)=MrO>=rrntc$~<{b8)MT-2KRTokf=VUR}OR&mW&@ z?pSJKBb?za7*ic~lK?k}K9H(O5$T1_yd;=(LYksyi(709Cxo}bx}^5Xe2m+l{?ex) zy4JvGLr14+`Z(@xkZwCK4pWx#FiCUAuq&gdTU75?<2hIwTmr)!*8^F=`Z2sOg4R~V zABI=r64>bjFHjm~8}`Yc-g;m3C=KjByVM|(a)sMW zEQGtAla~vZxA*YZ~ZR~McykgO}IG4sepm#9Dp()~(CxbnC_Cwq1T zYueqP`!mYw^tm{N(sK#23gskWq+;gR8_CY>mX~|XEtyvx_78r=#e)aV$a3s={#<+{ z&knC3NDC!EFEE0eLJ&nWCO&+0MqT&eyZK*zPRT(g;mnDG*=1T7W^X9vsoh@}b`^ea z!142;S5PkbWmb73M01ghP6&*;&-^?&~b zynqXgs&bwIcfr}IWO7asI27Q`N|b4$APHsU2t)O+(nOf$ybhkDKvHzw24-!c&`!yS>yuigDAO60g&^z z&ZoM5yBhzKWVr$SW}54hKgY$N-`K?{;GDFj<>6N-p0qi&jq*)%kR=lWuq`h+7#iQ2 zP4OD{V`5BZHy*R7Txh=_1PqYSw#$!`BK5{|Tz7Sz?Db{8zCwz}uSxRonx zr)!KTS$f)9aMM#a2LADzSb;av_gWX~j=AV#jOgPdS3`*JwJJ4z2HN;N%~O|LfZQr{sd83_H@9ogrO+3bI8 zOgc!Nzf;z|*~Ik`J>a!(e4i59y&-tSwRSn|`;L zkS%l1m0h4t%WrB{f{czb{>LWlcTD4GbnRiBUZ{_!NY1)ksjK9Ag~F65FODyyC$aal zB;wZqAX!3d2Xmcig!zgeos=~R!_isAK8%A6#*g}V@mpR|1{6D-lrZEUtI}GXn$=c; zlqw<@$J#@0zy@B@Nm%$ja@PJ{7ley-ypLg1C&@S!1CZZFRzD`B%YIqHM^j~&h}-O7 zHdv_*hB;!fBrSF+^od@*j_KnFGJHn}QF~0XY;e$4O z8%rxaNTsf4Eh007q23<=BJ_Auc|oiC@xWUFLdOI7^cT{293~IwLtn8IY#-YXecvYfQSgCrWqJ-ZvB-b2@U znN_D`WTW~CQ@;ZUvQPBKCkoNz!T>QWM2)%8SVDHMI+utA4RmC)$=~`F4%e+Qzu!(4 zkzLBZh<||9^S4rV&m9?aXx%@810$@EZLdiuLV2T1(7`M~%u{jHDigYt+74h zF$bxJ(t#b8r!ogD^{;wx&dUtqq|DF+(REpa4x;F`72O>2aUe)D!tHf{4p!aLLbV7f=`!d{uWk*zj=x zlG6h==3UF>sb;UEV~i*1H93IVBD=jSo3{623f3SwxccKmYL3W>7#Q+*V%8%yQMs*w zN4{kIWRl7jd$0^qc&GIR4_@cGFY)ZU8wmVbs{@pm?dUgW32h6lN+&FZiqGvmIZF+Cw%{Y_>JL0DS_I zH_i2pI&6&!r~jE{@vjwR5qZ*-ZIL^oWpV>k#UJs#5@iv(gTHOYsSB33oPkl0 z&G3gNVsIT;7m`E!pzuPx-tdzNXi{P>+z83e=KgFyK?__w&45bU?&CEwAQc1zKZSq{sL8y!IPLUTO^+XSoWq*!jj`#G=|Am6a9a&L5+UqGY|MN{ z4}3Q=M}vBv5JA|>{GchDE|`VnShOo~nno51pi~^){6j^=YMSuC8gPH(Gb*88rEkMK zf>iOK{4f^aXitG)JQT#Y1CIGPM--pX3sAHLz=E!VFu#^xV4Mj)ITKtAE4VfS$bD^? zlFd@dmsR8jwx6RELhSVL2)J`1JY+%Z*g1y%*HfDi0Dn>)ItheWSgToj|7nO#jNA zg1=kD&GX>W|#haP+V=sU8nYAwwe+hGuo&sF_!T1 zTJeyBVBR3@MgUJl14RUWSQ2oV;uDrY|Lp^!x@|z-1AfxLk7s}+heMK1-wmDX0rmo< z-nk_wN!lF1%)uP9l7Pb0Sz@$(EeWxnhri3G5dFq2m=k2iB5E$8GDEnt^6CCIf0b!P z)R^EPs|!ydw8lkOWSx8LMtNm}D`z(G2`I3voC1j1`k0cTDe z*19X=0)e3%_*@&lpqU7uK}5D>02Mu;a^gR|{v0l$bB%V2n=mZ9zinAvFh5E_?ph;- zVTtO)<*{MeYH~7XpIayBZ?n2TO4PLVTyz~XG*Nt6Wgd^*$gfjv57EnAtPNqWxM*E$ znf0Wa_U8m^gO9R0&4PIggq?lN&4+HX#ud$?D5-yuQ3QCiY%oS4us($}eKWsn+#You z?rc=B+n+6qo3Rnk_g%R_)#Myx(9bJjvQoJnTQ~gME3TBD_=R|q-O~QfLHgDIM~Uy? zfr4&=AbH?`egwilSpoQU8_YG7{rg)>Em<6&1z+S&t^3%l^sealo(n)zA5K2v!ZXEO zuqSWJ$R?z8hfwEON=mHFsGH`!>dUbR60N;+O3mZzLUi;?#DK^K^)kh6IH4u0G!}Oyrt+|o4*ZA`ul?PS?C@AKE7;}4tJBDKtu85@ zmtc$r#qgMhnKMvNe_yRyqW}el0^FVB%sVFnyV>F0b11%g5*X>YfSs|{GA0go3sPe!05$xKo49ip#!Z>Rp*ON+0#zoUy(0Ad7*kec zEw_97(rfU=KZbm627&^B?~`_q$4Ep{{(4D*{&kmu+l&SPB+ge11XoAzH|7U7*@O&P zEPLJDeSOdLH^1?M5{m9NFU{@ztCt0EHHZ7r4fy`#5@M|e7 zb2Z?3PUwnRrk%6@{Z$b5K%GW`GXN-ougD~TQI;U74I~&RJ~IZQLA<#f_&d*%fO`l4 zkYG+ox^2FQ3uWR@0lOh9-7qCCSv0wJLMbwmt{e;F1fBhtsXOsx_8V7%TgM5cOoHkA zvc?VjXPL@l9a8!P<3mj=_2S|`E{;KOdRjF)c>+`f4i$+rnyaSF*ZM7S}YvC^D!#UNH0OctRt8rV5qLYv-R#f$F-- zDgc>`0oyPhGyNw&uo>dQPh9|>q;_Do0+F@wCRz~?G{_CYms{m@_~=1~fB=3}1rUb9 zfatIdPc(o@aUjpOM*6M7hVyG_g42j9gZxhq4=Y;#8nIr!KD>KW1~my`#w1J+@ObHh z{r&Y1K+^f5VI8gRcY-z;hi<|dtuOEw%0%j=+gLsYKR~W zOw|AhX86g8AiWF*&OV?i>;Q7K$hkR8!@CjNVBmp%cd}7F5Mave1I&_O+?W#3HP|(;b04P~o!%A7|;Fbh6f6sXEKhw<{RdlL1FMfGILY zQUj&Q8hjazWx#Lf-+p*XclKyaAelJD;02-*J5o1w%~@j-jBLdc#Vg zOn(lJ&x|%vK8encw|+30EIRsA*4PHlq&06%Xcpm6a6Joh$nREi$Zc!!orP6O%5|H@ ziqv>hjqIy+IqQu@Wp9IF853xrGrzn4$qmN2t1hof#Hg-buzTY9gdy6m_(z7y?J?qM z(B#w~TU-%E-e`}xoTxfj`lYpSi$FNZ3f zSlWrP2Thg@4yEg$_4&OuCo{#8xxni`ZV5TSGKyp9n*=z(HQ0xU)^6k|0DzF^;(DCr z)h!V~A1y}N0igYSs?fL8mi zl+6SmNqGk^XQyLbjq<$esn=zy3;L2c08e(`yvm1$(Lm+F@}L9qKXa88-FBQjd21N}~dt0o*pMf4Omr?8=?OJ2o22x)UNS)lWRE zR*_b|sV!62HV6iE1OquJO_X|a(cG;e0D3|F#^zx9jR%_U7@Ri&oDic}Y$f&F>7;ND z!ioX3jU>+)bO)OEC#!5Y_3GUL)}`^MTlAAjkL$o|)m9Dui7#<(BhPbBEe2VD;Uc^f z;R|6S?WL>5nT3yhIJpQH+KvB6Hhdd(FQ|ldU0$H2nPh zHO8p%9l@vdP5@8kdctsi3j%AIkr#2;58(K{x1*$Sl zMj%hl4r!GlZU6joa*zC0;!6W^QC$b@kOfhd!MkF^?qZ@O)1IQ7nGqc5(SFCzH>KT& zK_5jdf3Y<0aGC^7Zapa@%084d>!8%};JW)0g(=4vp%J*$X)Z%`bNB<)H(hZ^=r=n$iEFLJbkvLr+I^m{OaN z`SJX>Kw7mpmURkg3OpJmuXT-x=w}7IUsMGcHAlPAv)$V=Hk{>j|T4eK}>%etK0@TUA|g zjcY;%c1h>8Ri0(uXa{MjfsG;-*S_30Rxt!@kF!2qH42m3Id*`zwA|gtUjWq-AAI$) z$i=cA2dBh1b0#RHbgIvz{z|0C6&j#5&X|KMLpIDCFJo~FD62c5roxQGoRjgsl>r8q z%d92){9jX!98Db^_q*Nc+ zU}&)W`!`N26RNPB1BpzqM<@eX>&0Au2VvX>!7aJu7T>B@d{Z*G-*x)g6P6GxBXEY` z8#n`~Ba{Ll(VOFmjlR+PR3W7%T`?8giUR7!wGo&3Sx&_&M)~yg-rhh!ahCvNRH?QY zV*jGLOMt!bdv6uL&^RMhYl`HoYVDixx#fE;d2R2w%^b38AlK_HIh0nJIDBOP?@j)1 z87Pete7bQ>DFCQ{z$US5OZi)wJe!t=%YLQ=U;&abW?l^Q1VTVFLJ|1lxPp*EIilsS zw}jio{>j(vvZj>pHtlS7=gorYXJKDD9Csz6?|H)3qZpSaMj0dR*bVo}pmCRg=XBl= zQ1INV`WVSa!aHSD>Rx}JwJaBKRYim~+4zs>U*{`QHvNlGF~<#_G`D?<5c?^`xooIXj_0@~GE4$}G64#@t}xDIp2~ zgT3dTPJ(hth-p`8lw~22O;-(3U&v3P$+WnNRCcRx78xA5>dT7%3O{=BvS!!sU8KFRqh%Q#m^Xw(A6C-D z-fQz2RkJjJ56J?)lUUE zAPpPxh0_<7Bsz@SAMFRuE+%UII`*@>2k;mRNH|#+dVTT7y59{WU2A6+bj>TH^oN(k z15KyXp8y{{r32BO(3cCm9?m}Z2(f@jlX2r=!@9vRD$~7lpt*29^ zj#E194d-A2tUf-kY0O(>4t$8~FoKYnPt@Ay4xdF^@&oLJtfRiOdvhP2ILlvPYUKyt zg;E1gEA4@cI(Xa|QMETz^GF8!b!Na)7~wEfUL3`9E)7yO?sB{8^*_=qIb0+_L!xZI zV~>yGJ;;F75ARvq)rh;qLx8RkrCzWiAO`rq%ynJ&x~l|#7qY0o#il-WKF~+)kx_K^ zS^{XcG~kMeRg;jl}P_?6G}gh6C@|w z4&M^(*{R8cocXzAq+z2G2|%z?yvSU>jsA7mj(Iax?O0Xf?w>%hC+P{p#J>7sm8Q%L zO%gw6()xAe=O%%@_D5VC>m}|6E4kqzs&q`*qm5vK{^V)T@p7(5gm+&Hq*nL@=s<;= z^i|d3AAv}~C<0+aQ`r31=&knJAHcKl$_|qKN?!{|B9SJkcb1^?vNyt!9+s?={upm3 zk*3t$V=1oeW6OB&5XRDghx9uyly}@J9!WWJ@dos0z(#kaJ$>j;swySADb@gi_(n9yZN@@GWH9sOTUH-r`iLP`~Q%ZOSV8bkwL zq(=V3Ie{xP2-pnMk-nbtCS4E^C$+HAU0?KX?Iyw=OtzMo!vW&9>~+4%%*M<2tu=w4 za9o9lYTpqNM*~#Mrm!vxDtS6})$t=GM-9Evl%(Q7CCyXMr)>2gsqK0d*_~|R zBViVt%{RbW1wwF0y3B*2J8Lm z0``x(Cwv8IGgPSyPd{t7=85xLU}9 zT{i<+l06Ih#QySs2_r#IMNh&wd8}b|-)=<(aMDB^?Vq!T94QFV?k*x>rifVc#-q=T z#YRjeSpb4&6usJ{z{N<~U!|+VgkrjhAm*c99Z29W0TzPmJ1dDu$bkiisssWSsPjJO zym-;Hr*rU)M0DLUgWmpzo?={$%u}fqMyJ=h42F=6sbXxGrD(+XSic&+!;$(|ome%t zgqTPnuz)fXwx8;N%`rLu;0@@^Hj|hBN&^ny>8~iuTT~?zjb((1G&mQS>0K;=|5@8Q zu`3LN1&5<}C=u)B#`>r(`}x2JjtAR?y$YauKJ#&OPy#xrtUkC9uP9M61eLZG)#;=> zvifUeDqSLYt~F>t)5a%K{&TThy&pPeGe26S~q$$>MPl(+1tvxm|q zkYN1RFDV3=kj^1O2{G&(#wl=Fbs(*lO4jKGK+~lt^kW@oNmH{0aQtgv(|J2Ss{WY@ z0y3FIe)_Hn=pFuw#O++r0tA=}OGK*WNZ$K%JHdj<1ssP4dWh85ooCM1%~;>$I!ScKDNJ+A8WQ(B5h-yctPkMh`vV4Xz4vvN3&V< z0D(QSv4vQp-oeNjO91L%|H}QduMXso1UGpL3FF5pf>NO7k-}03&7Qtq1sf0hc9Azd z(4-a2dg_(44&3_%e~WQer>(&!o{oxSoY(mzsaE&J>03J#evv6e_NA5(3%`L=yA9t9 z7yxYOfa7KtV^&I1&hY>j8MpcAud!wrGaYelr#T$+9o?VbIbkZh0yO@C?n%joy|*$o-@V?LvKp<&DEY8*2ljr@L3}( z8i;lK(~N|1{8Nw)PI{>xh{{AioZq6>HH(k{Knc*94Bk|2doak? z<;!*VnGp{CGa0A+I(XJkruBf`ivuN*`#B}d@Mc`AG@>Mw9GVcHK6sY=%pRK)t8#SV z5wI40zD`FbT*>p;U0;-ujvEtLi3~CN)1I`hkI;B;?M1umfAQ^b1O!DmuwUYUfKaBh zHmKnIMNpuDbhE9cs!0K*;MWv@)#O#2q|YASLS39TvODxjsMH9QCTh&OKmU-Pk-ple z;nC}e%+!d7pAXFsb)$!$;rDI4)M9?bh4iVp0+^u~4ZkbT~_~IB%E`-)b=)upQi^lZSA*wyl*lgU4c6 zWD6(m)qq_8DJU~>X7mUw7mLC6hXu9P9;bOQQ_1j)`^?XiVWK);we~LIbgupAb@}^` z^2*DHR)3&^Oip1|SJf8s)USKUo;wA$?_(dQykNsHGyek0IaPe6xF8LbSSGy_G;)9> z$qaxr+B`e$ z3JCjHUlNn_FDFczLfMuK3LWF!Z{0=L#*=Z+l|w?rgq>U1$@ql^IrQN74k&C6eMo-w zeXBk>6rr4Yo!;fiWLM~_$3mkZG%>hr6(Cl+Z`Qi;Wy*OHuafm6iUYXPN3ZVxNjit|>9OR9!@z|xy zYq4kO@IsQd5d&3kZ#oI_oAaCZC~02C6H3F(Co>A&DTD-XwwjkcVWc#K*k)u#V;~CS z%jg{E%qR-^@-t&8pk+-lxwGBT)CEYx;%k#eD*}U14*wBFup3o*^>-wF!W7f{nFbQG z2#13C3muCDQk9UX0e!f;Bq5ZYdQ7e(F(u`|&|_a5K@Y!vRsLxShuQUgbo3)4lQ^0u zet!+O)*tESl2`07O$8W7(ZTfAo+#e?l~L=K*~|6p{KD;_m|t-X=j&u5*4it3=c7{| zrO&xuSC$;j#vj3P1%k4(86LN=1?ug7hqr$%OC~v)6l{JPHUO;AJ20pe1TV0kRe5_& z1U?6*qwO8`K>q}N&g>6IqzV4~ezbSHX3*7WR3{1&njm7wq{BX2Qb!=InAH+M$p?g+ z+#uY1RH_ew-Pmunx}{c$jHGeRJzZLz_;RcA2^|dL%4b1@IK)hAuw%x*t<1onRr)4A zG6xE%B>Swzx{%rB=ZqE@_4)O79Sow^rNWx78DBL9dLRy`6XGkVl8J_Zjk;4e+84gT zD-jg-jQ5_np4Jke+xP7t=nU9SsGGqqir5tR}#(-(8iQZcvhG`d=Ey$z0mZPl&!8Y;$ zs1|_00R*1;vdKihKxlw^r08dUtOAOz#ioo)xA5hM&=f*V!O}70UYqh}i!a$AUh~g}PKoqHEE5w~; z1?R{&;cGTq^v@!u&TWjcC%^GNDDDj<1mn#7g=RkZ8v@;vMS5w?(_PfsTGvSh0*Ch_ z$wvCTtJCpnl3hnEAITR7b4Ez$mn{Pg9%(v$l7U_`Poy7S;=`&tVT)rjwOJhBJNmh` zjnVxK?@5Zd>5a^I0a@qu_KSur{1W^YY?!)bwQ`vmtv+g`IO1u!mIq*LfXtT-mx9z3 zFz5I{@$TK@h>?(x5H4BShPSrLBzd8Yuh$`hHWqZ@cVt!<*Ek~DJiYH%mF12MIAEJp ziNt@oSI6hU;GTz+Ct+Nm%9_9A+u19;P#05!%m8w${z}1@y|&oz2jt`2s5HgBjle^* zCS9fRLTQY9IxlWwb`E~I;5h|%L)#)b3~gmVpv=k~UDAHvKMSjweLK1-Vc8`XC`^8c znO67aQuB8!dA2)9mXSj&`3!$d^j+a0C@QFhz zU@WzQzCNpRB=bYORaP4p4+@_q__!km#QymJCt!{}%LyqbyQ9eBt&VY`qfpJ zhxW61=5MhS;~R&DKWed-qD{8z(dD5L+Z=OOIt=J;EEZn5VU96#TQZnPRUA&vu_ic4 z9yf7spx}Cs)P9fa-l|;4Dvgi7PB{rmuHTxKj(<+=o7_h0C6kKJDeR0{lOo+p(k7xe zvg+84KYl7Ja~;(2p?LV}riHQW_U~tr(G+C?N<_e^CHiFVcttyt%a^U8>`1&#Uae^} zC7L`rA~3~=xcJL0j(aXTkz&K&^@I9LsOFMX?J4*G>|i?Tt%Hc2(lywa-}b+n{|hz; z9G&=~*C1L&Th9}?nedaYfngGix*J$nP-68)-@U)*W_kgQ3M2-)t>?Z-lM>ZSd@YtR z5O8z(xg-%Z-EHC5B{P}nxPkTFG1Yv~DosfM9Zg2vAC_oaR=O^YEmsX1BheNZ6w)Nh zQ$$vTZn8eICc6?K$_sykk+}G&QyIiVYuuaok{BplnW3(E;7tGe-IpSj)XOsc_ho9# zNm~bMDzDKVPu~wWEO;)V(G$9HN<{92nUPaO%CQeEc2e`JjXk#3 zWmfl%FKn;QUB4dg_N6+bVIh}>U6EbBl>%m|MM6!Rz|H;hcZ1FszjteETiNBWg4NTu zS6W^cebRcg^84r3pXF-3`xe=pX|yOK6H~2_)|Y3Oj4?N_@+=#`g?@L zzWUvZb*^eH!KQLrW6jBEKK2lrhct_FuVrw`BuZ^@S%0*9Uc_-Tn-lVI$W$pl)C$5x zjug`oXk3U+ zPOiU@I+9B_H$IPtFG6A44HD2 zsgdK#O3T^*g*&jz_ z*u(&%vw}6DSKRN_$q3jpUe}jQAPfKaAMDS5{mzv~gn9g!#a@~X=#@t2-6SL=@Nqz3 z4)%7zAI@-E_UIy&K>?DRLX5ZADMgx%-t31_*X6C@ zcS*@=F^_f&9~Wmq-h8Ts)jy4#)<3i6o>mTTm8biwSJuj`ug^O_rR!fj`@1YtY0ZSH zc`o}~zlJ`I?^2testc{F%q)Rqe&Z~R;3&ucTulEqe@C_P9?e|ZpV)JQJ}0@BY%1_?8^9|8a6a!O=9kIpYCq$uc~3Q zp+oN;v#(_6^nsTk-e|E~usBG6<{(DHiy}n zhhhJI@BORI1|7a8Ttm4UKlSTFqz3sN14{jiW9S@77e zbce9=?rjxA$4Te!P{C0~9~m>!oTaAg&Iq#(B|lX4Ufxt+_MddW z_~qP(NJ}H^uYoZ0i!OTz7uHQzvU@RNJuS|{rF zEieim=e}Z!n}Fm8D+X(~&l3(9RRsO2>?a6fI==l}on2eA-2gsT5EQtvkVZo|6z|Ob zlC(v9x2|nb?8#G|^2LcK%F{AlGU5)xB6KeQS8s0_Rn-^${cakhLqr0#_`N4r7KS*OXXSahJg5dG7jf#kp!n=#q2x zp9h15Uu20yr3V*|S_Gqp*2GG3xhDFiDTQLs>h1Q(Sx_Dwbf0(a3wW8ZE6xWfb_feykqLV@6TJr&rwBe;!j7({(Vd_{t3_)$7s zDLIchrOXV$a?HjsZAzl}{1U5`Oe8{SjvUijtbcBlq&jy&EXvh$+* z1Sf)}=+6+PPnLlVa$aO##L@ib#-RX&!CO3Od~0{t6MfDtX}`Hjr15+$hmeg6KeHd~ z-@bmDXxq!{BS#hq)#m4R6{X?*C1n;hhVCDvVistK?#13M(8INz7tY)MzFVqT&gscm zkpH8NWF#3E;t&M|k1(=jB>rY6GbZIAm8OOzwPhF1X7Zl{WU5Oj8M6D@6G+6{ajfOs zOJ&EE$2|BBIyG6(DwzWrHZUn_sNEO?8M0HHTj+pKIzRC*h0B4sEfn#O2UHoIN zZ_^?{)DcpfHI>9-6f9~2og&`qPcIs!9STc&iFPaGcMwWGxln+vS0IyRFEaVBps&_> zUMP?U3Kum``zY)PwGpTxbXaK}{f#7_|A`b|>pj`8%tzz0{YE%4$uEo7dgrp;_r>SS ze&nosld0HW5yr$S+oVd@REvbQ_o8be2_>lT5z)v}C}ChNeM#lZN>*|%A;0Wcd221i z?w;w&9%Nu9q2a~Q4B0lLx#d6Fs$0h|DxxL~2pV}XXoyM*#pW~I`8sl1?S!K$&Oc~T z!?f6ChB`9dzM(aEV}%l5f92g7d1dH4Luy#beY7eW?DpKDy zivvL>wp&CSq}Y7st(MXmZmfGO?7`3=+2hLj#$c0#^ zrwrSe-~A?k1}in!ko|6k_tmy;tjIaLeS{EO3+5j~reFYSKn_H9?al+85ErYy7G&C{ z);C?9k=1n#5LUQ!9#$qZJ+_xFsZSEA5aW`9`q)0Z$_h`f4UN|H1@b%RXk!ye+8Vux zSZ;dk&tcl#PimtCB|8o!cV>!QKvXn57LSW#p z2%3~h^9d%v{ZzK*V+JV+**}>F&NX8NW*l3(k6gDN7fwAS56KQjGJGl5{GLqQ#0XKI z&_#?uT5HvXv2LfCc<3jU0G1->j6pu9fp(QEFAt*AdpjvZD1hM|pN{|oPcU|VjytD; zsH;ZhZw9kX5=gAO(*Tw_be117lw*AEUeOy!8F-Z(MWqukiL}0X@q*aRcd%Eb?>$u! z?Atr-nGX)9MjBswCt=P z6J+6&gxV~Frr3jQ_zHhymqx3eK6_{m>qy>D5 zmuqIbi z@11*3%OmXZUwxZPW=r`NBQrjND2ATu6nAH7wNt*-q(&vqm6LyC0@2ZTa8Z8r$BD(@ z8cVKmLIEQ^XQ9Hom>v-)(W4*m7N@8PUVpLBl|<*kH0Bq3d(i?%(N_OuaC59X#El9Q;`VwdtAHUaq#{6 zoZ!hB0&5+Bpl6S&SJ~5s2s86j-l_y(P$=hxGqX}zC`jfkOQrvKi=;_Qs0#|utKrt( zf$4PYs&{$?zK9j;#j*7o7ewkjp5I!a>@Z>B^dQ$(_6X ziKf`n?e4KVp{a7`_Am54M2^kZbCXri;+JY)i_);k(3a5#iZs4g==!_uo=ujwh-ZvA z{FCcm=F_UMMM?5420;L_I9_QEcY4YsLV>p7%V&nK-^gk$!<@w&hQO(P)s&fT?SMQF zEQlCN`luNJ4I$?&tObbURB?R$+VR4NMS<1=D8$G7kQN;Wl!;$Oeee|1AilNXp$cII z5`vqroJj~j>x>NJGmGPIY9wX8Br}-A;F#ai^G8N$91kEv8$Jj+z)(ggcxBQ52i0F8 zA)F1$9SL0FTh()9C8ekR&4+eY=1Y)7w*;#ch(WI$GqU=UZt!iQvr|@zsW!CxXXh>q zM&sd&KtAsd6$X*f98|2~hJ7^&QaRzzIh6Lr^Xy?BBj+!%EuQ(nCXiGlYsNv##z>2M znQ=b8g|WC{m;I!EUv)=_+e;zgASL|80gP7@?8VDHEA;@4%z zh~k;1zWfn)yObkAf`0WeOVNdk4z4A1#BE}9Alg@sErEE+SOTmLn z(##(Wz;pwnff0OE0$V}znHLHF88uISw=T8a^dZKP@A?Gc`Z2Szs8qfo60|9a<4^^y z)}S7wDjZB8ERLcfa{8r^mj{`Tk*E9Ll2<*tMR+_~mKe%jjUK8ta=wt-pX#AXCn#Y= zJlyv4)u+tNGFh9tcJ3t^5ZWJNy&VnvW%(}FG`^x!B6|qO(5nJJNawMuG0GFC;n)x7 zxyJjH7dVx@e+4?I9Y}@wgnbMa@x05@R4bpu!R#Uc6 zfVNYhW-90FMURJAledgmKW-%3JIBd;=2wg)FaAIK|{)xu_+j_{H^EF19tzNt1d z)Yn&xiJhS2@k2b*`Bv?W(;S`~J3B3m`UWN6c;)#_`sq)|UvI4deH`t4ajbP80Rks< zh<=ulAE)|9!?D5?A}XU+cN9ah z{!~TQu`ZJP{_R;`eZOxQ4Ue5>2(hbMt%McC8jLI%dOhMKX{jED7KV1{wD_G3A1|%A zI9NVUnM0uo;IB20ISyyhs^EJnE)>XP@|dL;S|EExUS;o>)XD@CC!3 zf8%Su+^)&zYnuNuGioV|3{*#1IQwUHu$A&5N*>c)njj7Or|Zjg(_B+bPUTfX0!^O7 zUe_p!&y*iL`AhXx?Ko(Ce`Y5f7l-HYq%S3>k&h~ulkcNTe$xv|uyuVHb2uUQJ@u*P zwe{LnXBA|t^ZAVu21(_KeuU8WE2eFs15UZ>aqHOU<>O@fosmCo$p&|35#*AF52o)W z7U_Vn2AjS@Zs}erucXwV%QaWI_{qP8ViTX>nl3j3^|diASvp3#Ld%i4xlnWC3pES_FViB@49Po^8a99 z`tq)Wo(IACPvCyedV9h9ssp(hV{;5QO4*BFoXEKLq1*Su=&rPc)1T}F7UiC&Bd$iJRvTe<|lEytR3#T zpm!q@SM)cITisS=B(|WTCDgG{(^kT;lNE6mAmrnJ8y#&i8MAsgn$AY?_sgJd7_B#= zltZtmXzP3ZsZOU`x!C-v+BE_TYe{FCV#R?#&xwJ>QY6CT~wnlg$nQ*BoaDOP|qicZhEVNP-7V;zTm}a<* zUZ-;Ty53H6fO)M}zD)k}?=5_#%JY!F2q!^*^ld!jkzGPEH1 zU<<=HIZ*9owSW(9w!pk~K9}n+M05xxkJV87icMi!+VPs(itRRLuwSKA3OT1cL5s*H zOd-&*%>?gx#x*C8d2IOG=Sw-}$Y;^P)D8qODF#(0Ra%lAjv-{zPM*E#2nGzSzK>G* zOk+P1P<)}M-<$c&Ns;-~n!j{obTQQ#&&x#qpuxK6{iNO>PDmLuCw{bw-ceiYc*z4{ z>VT`7gvSy!VDv+R=h`k`Fi{sHM{o79R3Jv17L`Owyf%&c1lHH|+5#$A_mlc*f;02| z%Zow}3{okEl5LhRpX;BtV>22ZOg^DIEJ$>|9Ej)%d0U6?$N zhid{IhK}b?^+}B4%>n~<>ANTk&cn&`oP)-u3f9h|k1s;73n+Z(A=t7YcH2PbH^Tvm z?#(Sef?u%7IT9=qywdAm#oSSOB+%Fd8wr}!tJ}*MB)kOL3M9_{PxuBhPz7=kO{gUL zc;y3H`?yF(@qQ8apys*V({O5e0yXXuMQ5hyOH0z zBGP-6^UAE@q3oLms7LZw)Mcv$`50k&M^b-y8%;i`Q>MKEh9D%*@pGunRxSALsP2$d z4V~O@z`zHYcQL7}UziptWyjqJLA2cshzwYU%!y}KW=_t&<+pKJ_dqCko=0cf<4Eo_ zN_dS{7s-j8kiGG_v(57Fsw+W4mO%uyR$P1IN7B63@<9)`^ImzbHtu_k=F{`%UvWso zJPFZg984*G*UKOy2~k5~G32XQ$Ge&LlGZ>tv{KYJITSZDv3tIFu-WAq)Ozltj5pXKZIa$|o^jNdx8i>ak6-@@8 zvBSm^+1{DJ5-{pn@Jnox`pX~A$)1a0E?ShK(-Vs>ilC0rY%pP=(eyI=)u1BLwO~6S z3AkUEzj49-7hy9ndnOp`ID?**t>EENLtJhox1xz08#IJk%T19G?dp*mSerukn?l;+ z#b=G@9o_tnPWO~j#1XX?OOGDs^6{c{-8V{2mxUuR3^RNpk&IS5$f!v6pd_j&#i+)f2H++bC$YbZ&0i@~z8%o&5_sx+@+HhzK798;_W!yZtlxgo{PzU4UL~AH?Hd2r9cjom zE6vxsEa#+60ZWW@P(fk;Izk#LQ2jn~G}mJoZE(c>0z-fEfeZ6Uo-$RVh(kAxCMAYO zB@q*X9Q4fPcCOw92?W^I)5*7}0SO-?m6Vqa7095t6}+qp^s}324#O!+uuadFpYKyV ziiuj3?ev7ljx$JK7E)aDG8K7mW62vzQxRV}Ruw(bf6X%YSM4S3MxbVGR#rs(1zsQG zu-`}NQvst8d5TjTbqt@(3nWOA;ke-i>R^uW%h+)xIRs=n1kbME+`1}9t6K%BYheZG zivFRC05%pe>lupnJg{x0h;s$To`D%8j*N}HyW)9mKlZTZ3;Q>w46fsc?c@M#4$i74Tv zvpsUpq0(bJ==0iQrvskG2o7|xssKUhFl3b$Po^4DH|*FzZRqrXCf-CY4D*%_q5Ym!>)JGvxTPi9P$50FZhKS)b)AN+hz2Je-ba9fMGPO zW)^Lu7@9lE&AC-%01F>t9CiKh*vF+crmOKJn2{@qu6H3JJ*OHBSlCYd^M;B{L${icw zHm|XBRNcS{zwG`23Ln4Xib9v+Lc01+=SV27@YBmm+jo?sNzbpp$!u`;Y?j8 ztWBrCeFm=R(GsNyIBZP)g{Q=b=X4^KeCWtIEuzNy(njP zM{oKbB}+SH(WdPzUnoPMt0~C)2VIkgu=(cj6Q1xHPHg-p`M)r^dW>JJR}soW{WefE z2}@^G_{peEA9&26dxgX8jzTY{e?3U&x3nZHU31ovBet)ac~9VO?=4K%lu0)9E9kXT zeoA}ski=crWalTw14P6KonOCSEf;^O36%kT0~zDX^B5;PeMvI$r}%WvU$6SCnyWoU zvpmc&{RHL%-ux);Bso>i`?R!(YKbSNw4BHbP}~b&>JXrt)BG=p6>@KPG1;@iOmbZc z@TCS)c?DjGe0Kgf`8O?IRxuIQQm~Fp;0vKV`Ye==p+3@S-_XkPttxyhmi{UxjV*Yu zDh2OIChN5Jg*5_uWaK+{NeUN{_2~Se1X??^II;C4QHiu!J=-hL9!@%`SR;BNI8Um> zA0ZV??Q;i}7>YC1?aWOmst)II^_v!5GqLmCu%)6*E`M?FbMP2q|B7Ls_tr7u^8OQn zYGT^M+Tk$M?B0qm#76v&Yo|J`Gct@Mr0?{gfWuW@^i1~sVUozW|BB!2M0MCrsgrTZ zi?;(PT+iAQ%duhqN-SB3{_3V`!R)qe-*6ELkDi*l+%8&Q>Fh)D^Kh4otj2VC zHf(ja*U`yb!S5`B_D-4+J#wYBmKQ0UnE75Ys37AqSWVZypdVGrAb6Wf{CfqZROgI) z4DBxsiWYmj6yk7!s-)nSzyNWbJ}l!W4jtfrh@~wuCer69?l@N~9+%(i3CPSJV!d9g zMH4R-|NHqq$ZEu&_Bd%fH{;G`jU=B-vR4>)BJmOHwaSB)vYsqbZN8`%7!C3 zOp~k^_Dp^RQrSdi4_; zwCG8h?C-1DUT392QD+s!sV0C;Y5OPZUDsdj7j94H!YA8$y@wqxWE~#=q&-OfuVW~s z(tgO|AV+rxiOEx-XQKK&SX;i)&J%W9C_G*lCwdlsy%~=_sh(>&XJ+9l9H%L+&R6$m z*EP5sX>B%TcVZ5=>i4fHcC2=$Eozn@cUUNSbWju0zb*#yh~)d2>Z$sfzh(a*X8wY0*p4RL3_rD;Pz5f5}sZ;9eoU~BQp zb+ZHkNX{yUgz1Jthg=x2J?gF0fct}RQP`SO*RD7CZokrMlmS@fd4RDvcA{3&?w3_Xq&xGjkL zJJVsuD&0917L8NoF9i_z_{7(*0H=%$I66re=~*^0`~9GL)_U>8zHy7;23$ER6hkFS z33l4@Pn2i}1OAn(Njzv)q6d+v7_Z$^E^i#W70A+5T2 zf}Ch%LmF((a=B*ZVS+^))|cRTSb2*#1D|6^|xJJ zU3{KLbUq>m-_H`^Z-!1SI1rJ0SvS9j8c6pY3X`C*ovX*^vKS0$8?KTp>{tRowm=QC z`$yqj#({KlmbSJbij$W~FIm(lH!Z~%SMK(she?vNTSY_yx~Cp(Zm%F+-H}{hU3g`D zB@O@>M_FHAVfk0y>>tckJ+dU_qbhg9C}75NS?%keLcyM`lhZ?l7u~I522@nT=RH%C zJ(Ie0wGbOCYw&=12k=JGv9qYu!b3-1JilHk3BUW+E*XZkgK{HY_HGeWO=SX29Rju z&EZ@RfYCGhK3fTPZfETY{F|dvDkq7_2H+J zYiBKQ4!XMXf`dr`uq>B_8cc0u@e{!$6$nr;0psO+*9yQI+ttOd@WF{*tQsYogJt;t zek=2st`rx1GY$XC2>(ln;J?>${r~^OCztefeoyK-tsH?=<@UUU_~NyxT(*{F%+EJq zK*daUFyZ6t>ksVMMY1*sz9nwI`KKPhBiXbVCgAw*&P4y^Jn(^1|3`k39G|724mdh; z5<2RoovO8Y1gMznL)r0yd-X`9vERNoO&Iam&qd&hTtL9HKe%HF1`9;01cX*S=K*f; z-6aI4wZcJ%1B39B2T{?{-i`ef=D?@x;?KU{r0qSx^EwU*pEJ$wHy`8Z`(2u43V9Mp zOH0=u4+=W|q)$6B)f9aF${87gJN^LNVw&2qma=jnx6OnhFyZHO*~9}jKp|CCRpz@h zTo#J?gaFUV1jqe=d9^i~zY>T>DQ0eN{sxyqml!*8T4` z{1Y-Vq5-6p3XUW@>VI~4y>L$nyqLO6zt0!d046N?9#}H#osW->yc(M<;7&$Z)zc-m z7G`G5<)Q$`dIPp7_W*;PAWZgph>XV;23{BiU#Ni>;#TOqTa4(<60Spop5ry#5zTg{ z21E6mg7O)CzYwK>3pyMIBoj}GtxN>UwQ8Q-Pu1{Y1eo^jBK6{&acek{8*X3-xMl2W zMf2Bkb~Z79Yt;!yAlj5Qpt{dFqTTq3>U*v|WYemEz^~bJjrdKzbvFWN!@jx^%bi5P zUH4wcvs|!vuD!tlK&pWHsWB%am*fDZzzVo(OScwhJDk~LpOZP&gyWv>-Rgql|)2tHH;Xq-MN!1>nL&GMKTczQOdsi-`G=h#2P$7iDxB!BF`uO50VpEc;Q zTLC!aZ(x&OvQu1A@;(fQg2isGzS3K8PJD(deD_j;;h3CZxwyPF>EC}%Rftc!mkkV0s}y@SCe^-)z+N)Y+^2SmSJTiq^Dr1n1)mN!=bQUU`x;ROo()X{q?8f>G;EY3SA@TL z?q@)O6TWOS@g5G!yf|1&e{S}G(qj=}YWFN_-;mLAn`=64or46+7%;8t0kJ!G+&Y8e zO&EWSugJ;dzR^2z%(a#nke|DZt?Cx6Yd=?3Rz3nZ=>uS)fFLcI`DU;)l8_Uek%E&; zzkNf6Lo8!?=Of^ddGOdE&I2g^)KlZX#R#tO&rXM=K z4Taxb3GX}wspn5OSnnLcR!ul)w0lKTA!&<8Tj`lg6qd7J?$?M%qcP(`0KJR<}wiyhj$4+Gr zhan6y^EWqV0ET*bwr@VZz<{IL03G(=Cu6oDARxC4&z_Aeh$I7ZGa5k5UO}u5SWWLp z&BB!$5`P72i2<@`bj!Bx2T08t#kF7rxcSI&;7blq2KJE9u|jd{Qvf4=^wtF*gvDgA z@Qw~Zl?FTT#UP>KFda|0=u_aJL3DqbQ&Upf%&U!NYOG4@UafoD?q_PVrJ9e7h#mPF zh=Ro(J|q{r@BE{S%NmOY&fsr|g(~@b!0z*qslf-y@3XN(BG>T13ZzE2s}oIIugKew z^uMM&^O~WGnse{eGo$ev!SNdyGRR z5(Dmh5iHvnYn_e4w>HtAnbsF9)(t`k7xUKX0h|s#4&KgrMm?hWAW=Z9<|Rs z%+H?n_}}#Sccuh20fm&@r$S@|LWIw8S*qP6+wY3GeTjq8@51N?oa(HY-5uUOB@uLE zvjU?#zUA_W_$i8xCxj`V@PMGS+L*1lH?{tO#M+_(YkNE!IDuLxXZqFCufmm?+ zJ7chPRtu_Hoj-+l z2m$hWLGt6rqMqPG;b{SWI1AKydDDlDjg6H1grL;81XaxlEgNV$3Irx2eAK2k6FnKpW)*ys}?e!en3z zy-k{dk)$LNSR#N3Ogr$<(fR0fYg67r3>`Er+~RYT?)ED#nKfD8ub(gAt4)Ox*VbKD5lK8L11S`~%}d~qFc&r!HI z0FJ}9QX z(5gunbhpniS@)!bAD3_*5)OaG-hd1U?J#f>h12)SCDZ@zQoadrxj-^PdLw%K0`vz6 zzx%Mu5di5C3Lip{Ab<>m0svCZ7|b567BAQ8wKZURd$ab;9|6(>6N%%40vJ4Ahtb!( zM&Rn~&N8SYTCtxZ5Q6)*eDN`)4ZCuAXw50NwUrWlbim|x7Z9B@%|OREKRiT(z`sEd zkf(K?dq9pAM2JEpJhmhX49QOReB^*uCCC9pt@I57yyOMfe8N+X3&BBm@Y5ea$gjYk zPWJtc1w1Fdw0Ujnoj45V9!u7{dAxw#17q+~&Pc{pf|yk}ND*!{jBLDW_6 z41(>dem5fhbxreYJgNI&Hn0uyghX3g8>7@S9ESpKFMl|$Y0w6`cofb8pX~H`1qB7| z1)rc&;K+s##b#3Re=jS;5y{2{y2*BeeWW!N=x}K(D`pTx0Q4Aqx;^;?9P;@-=Y-kU z*QXP!DB`w50@XB<@3{!9NI8tQJ|rRv*zg1uMs3p&_~^I)O_olbt0!>5rr%5gG90*9 zDRFUx8l9&=tc8o-ZoL^`(~ZPKrdhf>Gi?yaa25;(cEez$;{;By1>@Gu=kwlE)fTu6 zuRo40^Cz~>dGEJ8T}OLCf>k=tTV<;NR118K|D1LSXx)b2XAi+iY2)S2$i6%0E|rwj zRMb+9=%}bD^B#iTZFG1J1+NdEnbCosM%MtpUr0en-(x9sx6w^nI@C$uRF_B^K;1s6 zXJZ6ylKo2=Ef0pOvvakgb*b@S;2_h8?%OCyJY+raq6-k&%5UG6?v#UN798)H5n9+j zH3m-a81d3Khu}jYv7eDsE?Zt;t%D05kOC0D&;HpF0K+$M)^c~X?KoTBaz+XN+7EAufWv4&>in2ZH5BZC*X^JiiWp7&v>gN+b<`7m?_F-J z$Ip?${IERLE1!&pPi!#z(}7e62@=X5jf5XB1pR>ypdB3NJvRVw+>1CG1+0+4BbFg; zhA_|vPRP!Vm! zsKY*>M*${C9{i{h^xqiB$;q{W(mViId&Gm-;$|bpENecJhr6nSklo(hLog(+Pt*6p z0P2!2%j++*sfC5fT3-sBGnt{>2d*n#=m`CvFa^z?g3{7ZYIG(RmiCtO`Dh?EnC!-_ zxhcJOIE5}(lfi-DhZ4DX@JzP4v0(y4o5yzPcR2Q8rFK$3xMu}ka{>lkC<-R=i>7_} zLuWqBa0@kp+GN!Kh0v;zI&>mw$K5n1qghae16g zTY$)mbE7X}K>zG7Ap+l9ybIen?y{o-Wj3g+?6|$ssL_}8LQ~;yGSdcs(BtZwG=SH~ zsF(=@xaS;=Vn@(eICq-_jY;&;e$GC;M=(khH}@#}Z47}wa?*-YWfF!#{}*ERl92!a diff --git a/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/assets/image/uc_char_validation_loss.png b/toolbox/Megatron-DeepSpeed/examples_deepspeed/universal_checkpointing/assets/image/uc_char_validation_loss.png deleted file mode 100644 index 5a65f6bd12977042bdc3690d8fa51a69cbdf570a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 42352 zcmdqJbySx7*Czg;VjzkNg0!ee3Q9K$0@5iZ(%mT?ihxKd-6#h?>UR-dG7mDd+%#s``RCGDG8w)7z7w76zYcXtCun;)CD&b z3T^W0W%vto*XS(#i``m4(OT9_*VklmGkzotdQ`J+3=j2R!7O`70$W6befN`3LPgZ>j+b^=(Y}~JE~Wyb^?E2BioVs_DK$nsyi1A;{pct{lxo|lISl#fB5i;>9)Zghr5@) z?T?GPUFvqsn>&4ICLen16kd=w3#HV@rcuHfXHa?p(jXjA1*MBHvOGruG z%C0)&@WFlZfSkO3bn32%>-ni|!jr9^reh*7ZW@KGws$ek$9uPL-KsXLI^DbQur;d0 zY9Y`0_??xP#y9)zdGz_?SFM% z%S-usoeiCx0WBdkQJm2fpFe+|UM-rK|1*>iQ`%T7Tc4huuBr9H8me;TeJJEV3k!>h z&rqjdwzoRc^Gz7rnu8b?Rp)SP4u0ardv@SX!tL-o{>8=GQdnG#`L>9i#V+^ZLLXpa(F>CZG6%1iFrQc!aadAB-z_xt)~>Ec&tiqDx$ zhJ)LqSQ$-6*^;n=fB%;HA)hHCCgywi!BL{Qxp{DCsNy#Z@6qDJjyT@V45fU#!?`fO zSVAWKTd9(9^GCZYN!;Yd=cmKS$zr=_JCkNouWyg@iUHT8Aj9S(+v(m5`5 z$dBiE(}|NoqSfdx%igVi^7hsMQ}Wa?FfcG4%JaBM%)wQ4ko;Ej0xLyrpy+ymnsOvkGr>DysUg>fLVRr1Px28O^~ zt;X&g4SwXbH~JGR8`xC&-mpx#cJrOuyy9ZE$C9yPJt^X8@|grMkLlDn7rxXU{C@kL!IPtv>;mJV zz~#PltKH?-C|r6~F=T~c^oY5OhS6oHRgLF3D=OYobJ|4Dd{dMn=tt0-t`MM6>p6F5 z#_4+I6c`v7&Sjq`y&0Bfz0|$4KEd0%wK`JN1B>zFb-ExtZZKa@GSmik5uZ^P6IPxn zV}BveRac3gnK=ZOqpQ@$2w9w0udecY-JF9>A8N=PSULQY-}xPe{HDkh{fx823yV}2 zb_{uUeEfZ$vx6Dhbv?Ia(Xj5Ux19V%p4YG+-@?PwE9jUx?XWx=C*XEqg5je-T^|0j zw8`U^uiO3roE)4s28SK}5AN=1Z_8M;8hpvv*@q}of`f4$i$}h!aM=C+Ivp3h?qe=4 zu8HHj@BR$lhJ(}KN0`r5mj={_yahDqV18)2BXE#c+XF^8MiyA`~+*G*{v|M^T8)EIv-hgcX#{tb2=X} znN{qDPObkgx3_BW#n&q60LxhHPWG8{OoNrARW6{M&@7a*yqx2C-Q;_`eGb zWKny6hWMSFxJ*WV>TFKeEB(ahb>TAV&pNuJwiZS8^0J z1z&DqVKEiJd#2v~O}M~xED~(=Yh+|gy$>F{{nocBpS_)>9wJ_sXz;vKtE=K*Kj0!$ z;m{Zl=8z5OdE2fI3p59k`>8s1TJ9`LBf&0PwZh!oTpjiUlaR$1CN?!ce_0!+6XJ(Z zg4FLaTeH!zv7vlDtT>k=s%zJ-VcvU+LS@Rg#=mf9GoM7Q4i_fD^B+BaoM$${L-)3H zr!EVnoEZwmL|-y{m3VX^!HD{trMnD27xFm5+SP1_NCtm7nhcnU~S(|)p{|SjdRtmOh=hjYK#_J^nRCiUmGjCD7mf~7&Rbh zY)lJ3VgWyx2)G9c4{5Tg)9dSI2R9u^W*Ypo!1Ry}WgWM3mra#;2G7b;t!M%BbMpb35e4MG=__^uT_&sFd#>Ij<;mFJ-yb;N z9Z(ZI9?91W=-L82xR@@Nmgu@(vG3a-#cUKhJ~5$o-b@j9FElwNWw}r}v&#W2v0ouw zUYyr)&$u=ZdEx2l=`MJymOwb#&VPpin@;!ydGanYD4Ugj{Ltj&s*2`e~iDwZEPHa=&r}j&zyRX89X38?&K2a#2pnV@~abAW5FCE0-_p9y_V4 z$9Fwzc5;TZ{5-p739~g-EFv*mr6f<<-}z{#Yk#SyapM4R(7^HW@d~|axqh!8xYga$ zqg_U>SVld6o~KWxTu+Z|l6F%hVtyb$JcGS;#xa;<{AV!t2x8$>FaMMiphwS!9pKQv69OCY?I&c={!|Hm z)w5BrBf1M&L<-4RmX>F~Dla6ak? z9O%6#NKSFM@3eDz4)9}{kl9dsr}6{_gB&Tr#%bZz6gWfRlFfF;dsp&Z9&5KawU1Uh zzsVk0SVQ94>EbaRGxO_-@6r#E6^~2KY~((3-YIU|?*lYqSa9NcejJC)&~>sJtpf5$ zvyq?e8>eA0!a;E5rJP*wlp2^_Zc!1dSb_ySkcr?1M0SSZO0WW@Jp&UHKbeTlz~6?2FNK7(%*-T* z@^sj3*HrwsbAJ9zfe|echjf{sU780Wi_(|epqoc70st!AfI`f>6u)JFz zZEbDOHyaz$rpR{`%Fx|ImWG_Pau{OIA`%3mHGw*P*78U zF11;)Jl->bQ)o4+3h5eTps7|19c=9E_@S(Zy`M1dQfn)ncfHfn(((=nNQE#_;zF<8 zvT9U2K0f|^J+H8k8BVY3+5nom{>o;4wprz2O4oLTYbpfLpxD^6I%jrH&Yn(Ly&P z!dXd1MkddR4Ua}1^B|k@m52z#(S0yWW-uSr$-3(~Gnf6-_|O_!btf{W&Hj!`|Zbg z^r~%pg*$d?wRgxxw3QZOY{xau&rYpZ`u)0-MSH;hhQUgiI5^13Js{lTb1bZo;wMB# z(JP~bgoIFcu(7QQ$7Zr=n(Rz;&WORcee&_qs{ed91BT;$+;rkN3h8^EPV2lr&SJU! zHq2xi+(3$Ig~L#m0y6nV*mN8k`8KIVy-G<)we<>OVPnWeL!FkfK8STXn9aR@m)$iu zs7ORiT;jM-Cz8z2y0WktoxeT#>ta%)goK2kK<^Co>mMJhs(2y6_3Ubc|1T~n!M{5R z3#0?NS~Q!*1JStpg9E#y&M&FWTr83;3&OmK0IBK$ku@D|FZdnRb1$97MeSJ~4I@CI zJJiSLI*OtOV%8a zK7@1FEMv#Ho?wbgw8e0S0cedKP<8YJfHE^Z{Xr&8yCs;aBvNA9fV71@WNh(g8yk!1 zXn1mx`TP}y`74u<+4y4IA|)lQ4U2=(cXf3s?Ww^E6R;Q`jBe^RLV8?Kz_jDw9Ezn@ zp(<%;xQu4-@ZHZY(s!R^w&lb&7KVwpT;LCj&TK3rfB&Ere}ZWPI_14y_r1&$%#y!TWc1#{2> zSb1=8psu4+KjC`b4w?K>9jW)>RMXdo<;QDfy=k&ukTpkZ)_(>Pp&l@oZ--)Jx6i#7 zCmEXZ2~tn3=EWqU*?s{E7%a1;*J||FnX2)CLw(`Ol`F7Uie>MfK(2-5xc5h+GnP9; zwIUMG>L$RXC@!x=F^uX9DX-Jzoz4yy;P8Dw7>|)%kr3+o_ZP1KE#P4IE95Z!1QOR^ zTBYtBE)EWk*2Nmg?WM9+xl_aQ z7l27A;W-Y2d`=bC>@K7#Cuu=O3!Y_VoM6C<)}?x9+{gr*gD8Y0C0*yMeE`gFfmdhL z|9#&Ri?joXhB&+V4G2`8Ul$;G+6KJmnNJ+C9%M9-gYS+A?^gVTR zcUK^_=da5oy~}Dg-qy7jP&E-}t+Z=_i>ut;Pj+&d5rb4CSE{5)$IlDyr|en?aN>G2P$X6F(hpy2i`P3ltqZn@MHvy)T+P zWN1n}GZ!vi1TJdU9g9aDP@^va^JhOoR)Tqp0tofs0Fh7#2|k{h)348WY3u-9TRL4< z(k6FCUB3rMqw-+-`Zy4hspA~qAzjcfRCWPyMB8b=L{K{y4TwKpK3%EOsicb|qKAu1 zBw0r1u&~mg=P_^{exW-4{cY{!NkA-YAinKPpTN%(5<~=X#BkVHmeZr?>FL3o^>^rH zA$vo7i|fgz&`-w2g#jh+(Uy}3uGSc#ViApxct4_pHTprnD78_tfg)(slqxCGzcu$%$Hu>p>U#Z1HI z##AkB=R~Ou&48=`aEQu9rtQFC6hqPk4;Q`TDRXFq^bt2!ED)jr51Px zr<2V_Nv^U2$ncXYr68vjO6t0&iVfeWYiaomvm-1-5bzyR(m47gRx-2oO}%C=_x%GL z8c7FwuI8EbeG2i@Z8kMEH2^_F#TE};Pj~UiMS`CaFvlK7I+b(ssbV~XWuXSz7fzh7 zOYa{@uYd%;mzge;CPj&(v?f>p=1Y5ASX9*6Wnm|s?HAo-yt28krh7Z?(8<`?ICNiR zz8#%`-iP;WC{d33hTaruoQ&YCe>57cTVC1D&Uc9N-H_74(QSlSfh;>LCuEB5U}4)! zsc{g4vnQ99gsWUm59Y$u?s3^Yg)=h`G&mgkE3B-nNoq%sC29fQ_QhxP1pdxqshb?J z*&U-hPpwD6<|Zu6&8uOl!P{yptxIz|?5Km4A#^=pt?7k@Mp$gNcYp4|Oqh3;TIc~% zJVDY@eoss!wlEnFBKc))%yY;EjYD}J0WeGf9_(aq#B8+OzDKHfZK5g;Nm;-*NQ3P= zndAePZ0sd;^r@+-UyubT=4oGpgU(De0(^e1=@VPBcV>MId(qzP9 zI6|;^4&1=e5UgDagc<=PRS-kA8R>8HG=ooOx7)Z6uqzeDjHK-l?mJ?+TftkhSufr~ z!Rgn81MdWMIIy0dfvp5s)&`$00s|4sN6668BoV>jA<1)iNl2)XcK@-2q@-T88(M)u zkLSujcE0ftgW*7yIMfW>d>Mnmw`l@iFF)NI0c#%3QI%Nu+2#!)-~xUR7PwZf{kM!L zgo%V0uT-Fa``NQ+o(Afsqs3_9^BE7dRXWarX#Di)lR8+bV5g3YDa4=f0X5f{y1JM0 zSt{OMUP7TfUhJDwUZf1*{2X?d>y5H21jWQKMMG&5zZ*nMT^|5c_@H0M&CzPS8A`9_UszbEQ8U_B(qmcM(-Q zi6sQ86<)0edHT21)E*$O0VV0l5mf-i1qn$$koK^OG>~B~j#qG!x*TX!j-AecA3`R8 zFkCRo>lLc!qv-dm;gtHNrcw~H7(X5Ln}Kbe2~kjeHl&^oP-Gi^0+tz?d=-2t5u;v* zHc-T#gFxABhB0{({3hSn13O&IXf!sYzcI$+awxb_@XXHbGi#VQHBN%m_G^z76N@TrDBs z_p!i93PM~1dq@JGgm8dEdz^RKlYU37D2G2yZ&T9&B61S=9cEiLKE(pa{ulxEo}Fw< zLOH>7&0O{S_CjY50G>+l4NknGT6P#vs3)!9B;cw*dv+)Ezw(B6fB)t zI5~x)W5%Ai{0K`$qyk^{BI@Ikk_h|zSP9mJTpw?eQ zU#dRedLUV#{A6RoEAti~4o*azD0+oi9=k9I6zu5GPxG>G)oe8_bmt>~_(!pT6@RNH z(c|MKUaOc~E#6d6z=RiXP>$DJMrVHE)SoZH6jjLe6!m-2cnP72GjR)0hwZPLl zIiCWxgq>Z{PaGn$%QtN|`T>1gv3O%4HEU*dwRX6WFr4e~I*SPgGwuGy#{K*87!c$V zoDMNc4EHyt+&_9|K@FM_*sH0T8PSoS#Na9L3~kn`I7ppi()tXYoSY2$Gtf_uD!gN2 zh`($qvs+Bx`(kl9UGW*cVeeJ#jmZQ%$K%-_6)~Je`8F$pbgJdgot%gOh9w#ekQ5jV zJR8t!1Rhni$dvd-KxU?ixs46^{rEdTnolK%Dc@VQnrYY-i(vS*G1%4hwL7`BJ5|z^ zfYmfK{ioS@IRNQa_IH1-?yrs2kBzxI>=JD3jjT^EFGr?3U(-T1hjw_h0Wo=HYl zc4@@6`g#C*JinJjZ~DOR4K^n0C%_BOAFnSEgetvxgMT(LFc7dgLxJW19DO@1?mZ6c z`=X&I@4dYA{72pvnKGxPQI+rZ-~Wua4+;QATE(XUgsfSN@BS>q9Ta zQtR8c>z>avYH(^f`qBrqo4(v$v{r_+tp;2^Fb#T21&g-ECMH&+{h#lOZtm@AEdIU? zOp(RG#*4~BU54AYr^*lKtGViV^?%2O&Y3NC5 zule=sYk0W6e1>96__L&N78BgvW#%S}D6gBRt&oL5W_|rGwcI24H8{1ogUuvbm10aI zqZO{X2j5V%Uo^30hO9SmpBRS4ho{Gs)ZR{f1=ONLsE(@YxTH!f6*DtBVN|p;e%O;Y zbdnZ_#|kYy?>f^}O19*F$f3IdRbg?sCBR~O(rSzzAEsbYUjCVogp{k8G+8EMbrYb) zGcJWpxSC6dfHMvWz0$JT7D^a&uh`4O{Js7XFJ+IMZ$KXr3!@x1rpgJDmyC?{}VU(+va zK)6egO*IwNy9d8(hFDmCcIpUbAdt0v3;!FII-2!P_r;$imF?@CbWxl*WVGXOEcE++ zJX`39(fA$D3N?Q+Sg}i!jih>A*=kq->%~$fOb$-AbGe)je(N76@L$>5Y5Uc#c>DR;FTazP{tDa<+iTc2qCLRNdY)6-dEF~dQlK-X!t29K>B7+{qgQLS|HhO~vz zZsXXO_!!vwQQhgfPTzifu$X656;#v9b7JuIu2qUcJhZ&W(f$SWNZsFac>? z`6)L^n9_m018(ak2SoD(m(%+!rYdnOdqDHN$kkLG9O0ZL8~)RmCL0Hb7RYt_5gP~y zV85O-D12T^&ztS1bBpjmlvJtpkct`2*~2im>+yH$i8gkvC)a9U`l~uTpCLHh)_7Rw z6F;ybEXq3ZsIXtY#VPIrvL5D>js^Ogd0ksV#~%y|l_4E@vCv@xZt_#YE2qGn9qY?j zBqHvY$xy-6uOlJt+FGwVHW({4`8iTwU;oUgUox0Rfu!l)_h%47AeG{8k9@CE<>JTf z`0knM=%wMJbvMDJB-IK}K<&(pC5D9@Q1kHiz6N+P7Tm>FJzkJi`DxOfJJTTMNd~}; z_60JoU@GiSpRTg1^uJD%JLIW;lz;!`^|IRDG_?fzbdVoVeR)<4(xbVfJBz^49Bg+^ zfPiTr2l8%N6$j^r2^QGChe9`Vb=#?t`M@d$WF!F}0MjpV*u}7#Z>#I-@}8V572I6d zV|Lk^y#`rl%kPATTR6u(+9a(_O|O!EQFFxL`U&zwHU;V2eTJGRrhqT}9PIo-}P z64!H7?ZnGjKRA~c!&?d|P32bEDLLo^^duHDSiv->kM1NF)qw{PE;^!wG5AD(dF zetv|O?#i2$Kp+j=VVtxk`aBbkra&DNKcd?ENqXQ?z$LS2aT?Ce-QYcC=^yc&vK9|HpD zFJ7q(TNrdXvB2d$V{6?gmWXixVkK2@g@`SulkzN(T=bbvOHpx|-e`A*eCF_<5nDC| z+eX-{h3(EpsdV|LZ_6rU7Y_2nEYg2APFNCi{$x3+f*GA2R|z!w$J}UzC=#_2nj&%f z8A1(%?b?Nim32TfiqHEpWuloKv#rhqnJl~w*f^VBQAoeS`PHfY=FJGLo-;7Ut&3fx z-NhENK*F)uh6W&4Txm>FQc{;zA%_;JfdT$1 zx6{F`Lgg5p3L&2yJ;>QE=(Gfrb8rAD~yS@*e$t^iZs`CYo!+`>X*Yt~H6@=ZY-Dc#yyLsEub+W{KXHb6N? zr8>$f*|Wg44BMc-(l73`-Hyrsf__T^_}w4HQQSJt(Zw$Gv9)*)^rbo1d!(*zCIZ{t zuyd2zGR!TmpbD&QW^u6zJQt%=8c!xxuMf~f-hxSFWMpUx1LrAUy#4(KV!`zZE$4d z2jxr_Lo-+f`fsa$E?v_0wFAPYNFzcbRST+~_QP?I_n1{Me*##Pt0&pOQ&C?2N>Y-Q zpdF+F8nqP(K7mU;Rl4VA6_Df>s7O9CF$DsklZUVt(s4j9q0+pyCm^ISq*GxN3Zokn z&vTtTUvQ3Yh=0Ma-;+|&JeYcD7)=p(baFaD@EvM*g|)7u!#}$!whlP# zt2f@ZV(~f!6c&od?(sSultJNvN^)er{`7cXf4E?rah+}p>L)qxR;eN6zYhsvhRQn2 ziKv*^(~KhZUtD~AAFQ~=MOBG;U4jMumbtbISxl4#H;1z)2!XL7sE-@cK{2G@wJhPh zfFK4t-V@DbAD#Jz75q4b)A1@!B$GXrf)%aqa#s>|tyk4?uZ=%3XUlA}+ASP8uj=Yy zLHn04o0`8aNH5Z%k#gC60U=X2q_|L%xu~hBd2n{TZol2mss!Yvte!}~r%&-vH4A?7 zCX~MFwjxOb^(P-lh`lXiok!M7uw))z@h3x$C+BL^w z#sTu^X4Z+ec)%dKenxHKn@;;*gaXaORx28XGoA33#~|N( zT0N@-)X+t36tM(=3jRnpw7hqfM^);w3CA*%VAqw2d$v>{m2&)Qp{wOHO&5smZRL=DO z{=H5wJ6xzNz-x0P)F+sFtqIR8T(7B2IkT?4{r%!%or)h9jW1-Vc9l*&Jw2a%%e}n3 zu;k@Ew7vu?XBK>#!DuMCiNAGIBsowi-;#I#t%gRH*73NbnQWE{Iep#v&$yN9!DYhO~|FyWGWtYO9rCjEl{Ee;6IzsgVT%L7dqqPzkU zEPOR(-d>tB{+?*j+i7{b3;PeSH#eImu5o$X5@-cc)m5x2$@qu~S01@c$RP+UOele(TfLnv<$wAR(t zZPa4%0_b}V$2xTN{+G6!RZt?)-dXgmBmh>D0_2P!_x2#;_vErc3BL;#K{qu`sh}Mx zS2e&jm(RggI`(A!)b{NY z$3336k-_{1o{sl*M;}#Ax^^vGfmd&ZX3M`Ru5@Oodj9z+?6IPJmb%iRNLGogqDy=v ze*%rGv$Hc~3DI47paD%xO4{7sZhglICBl=_QxN!9b!~w^2QvA%?Lyb6Veo!+zp{Xw z3bn#oT_h(F(t?7c3HyrZqlSRB40(7FxCy>O_wR-J5Jzjzswaw91|BUA3uA>PH=eshFTv*YntyJHUS8N^1qPSb6C&s~P2$M|adK&7yQgnYq^)I}Y>XJ>D}3C^9-e6kuZMliNFT_*Ze zZ_R3!?^1=b{L5ro>)kS7rpPed_-rY@d}!#kyCamDB#C?H?|D@ARhQOiw%{7>oh12$`(Gpyeenyl>ZZ+ zYs}bIN;yZJy#F9uP#{z?%~@SFn;@TE&~kcE>4)5n#$jDv>w5l}jM zov~yb92lgeYLH7l-(RbcO1C=iJ6v25%ajL&u4umQlf{8`@jrhUQZxu45u}hHy7)Xn zN-859kJ<1NgWbk&6-kx~tA%$UuWf?b`k=oy>LD0#1#gJOyg3?A9Tyk!t@3kFv@R@()do{dOwZ0nFTC3}{4>}|_4)@UlD!S%v;aT=v0Jj) z1R6aTt{b#6NN20^*UtR@T?m~J5g?^AQq(mu(K&EcQ{!z2h`Y21_RMk}w++e0X_dCF zkdVZ(o7#?*BGToJ{x|NFs-schrE>tPKnbj4FgI?2Dy(hbwX!m`kUxk~_yyy83`*XuVt^L?AxN@xx)LX>nB&I&$8?wOcq#V{0;Z43g&(@~yRTtY zb&gk*fxw)Jj7&V7X(@`TQ4YuA{3t+npsjCKj{|K0trsXplH+z6F=M=p#s?P>FjiZ zL7De{d>)x+eEcvpRro{kcD59yQb>)HEYl2>$v0G+P^dFn`7If0oZieo=+$KSH)uaL zKLHvOIwq{vM)5D-aB!b*C(JYK6G0(~CKT}EU#B-%FZX(O@{$@AuWG0KUm>(ksyZXv z4oLS~XxbHX)Kx>vp)MrR8T%Gt-a!8RnD&oE){GqqAc)HhbXxNbK^Wzq?_LD)_e(*Z zw$c|Q+PK5}YaVmzF$)0`fb5{2A|n%MRhXR$Tm&72eGjPm@YrxWI$jY8rEMQ8D`~yh zB%4beD}MIFL#l!rzsgfm)&7-8WJ6omRq37N)Ko@|8a7pzk{hzA9p6mHd^s@$goL_^ z)(Meh5QHtw5El@#n4rC9A`CJbZ|p{B@VEjMu+LFZoHGs1V#da8vPC|hKhGYor*`M- z@jjL~f&ACN7f|~53UC-XuIH+rnD`x#WOn!w4k(pa+Q(XfT9c%*veJHdZ);8zYV_g1 z+$Z$F(h+OLrf|_M=s+aw!^~!`z{^PyFkMB9mrLu0bp5OA z`IzKNze)ZW9UYy&Qoc(f5Y5jUp8ElbX|?wU->@HZ_Gs4#8sdB7)F`8$8aO5B=i>yJI2pM|In+^rAOpt;s1U7=bZ-@Sx^Kn9XbI z*OgdLn`uAQ)~r3XSyq4!0w_gbXk%tU39f`E-THX%36unFRh@SWxjG5c2lP%%K%4~K zkGG)XB@wJ1)@-+l!vPpGhl<#Ah3uz#9b!bpF;{P98R58^s?C68TD{Rf6B zwn}h2AL=Q%ZonV$cWs>R^UfPg43)T?60}F2?icOv&^8=w22m=|Y2g63hvjvdV%_67;Jv4l`#-iKBTVEI(90l65P;zC7WaeB%2c zIo>1g=|_n_vHVl*(YP2{@Apc~|P@?;=a%c%BzWN`3z)%hFiKZ8R##`T{I zAAb^<4Tjh$2NsJyIkY1E=uzs7zTWO1zmIlkIOPY&a#4{`hYHQ-K|;{ZA{IhVLWe+O zUmvjt-!BmHz7lsn!j}SRSVo5LL_VYr-;%jcV`o8=EEv$Y?Cbn3S*HMMXZ*c}Hle#D zPikUyMvJdO))Tt8NQp~-76Oe~x9{D1d3tmsu3}RQ$Rb}7z4?<2t~jqaHScrM*Xh+g zJ=j=SoeM|L5H@@;m-IF;PNOAN_fm|TeN4w3pr8%!&e+`Cw4Y8MBw)UP&+>;KX#M)n z_Rz=^yRe4>Q87NV;XN=3vDgzo;iDd%Gjj`DRuUa5FVp$G<@8yB&_>=L2Z1Y{SN!Po#PshgMtB?lw{gT>B% zei@ew3+hQuQ4b|jd!XdSMyJZv8O^>;rFwFh^%F6Ywn2E;_LH;TUL2k?yGxL7wnY^^ zZN+jwv0VIJg}9!gqRoj^KJ;@gzffFyPs|HcbqrVBA=^uOFM#k*Qi?<2>uNlBu0z7& z;x97g2@8vCR#Nb~-@UtVfkChwN{%BrVyQj3W)tIFtDT^qDjFxK10On=PoWj3s;UZ} zhR|;w)Kuo7Z_IGfds1E7!M{OEq(MpStKm_wU`Qi9K~eM6WSBO|#q$s-^gaZofw(?O z`@F^)DrI%c@-24!YUOwNt;?6_(yo%_{g6#1`TF%MLTwPSTYduNkC`vgYvdLo$pA2K|nwNDX6I~#Jik&f;t<-qCsqS8>oZ?qr)k$ z_1vF7n|sc@uw27PxH;wwM#_sJ0*FWTJI;>@xE!zI0TvJ#$l}h)%R@LIPgj6@p1ZiT z;XAI#;R||9KWFG)FUIX)t$DCv8&@*EIU_VwXjC>uM+F3k?m*V6Cr|bR zv)Mc5Wb;dgLiW6RS)TP42Yi?42dE6zzI@ENcZ1+>Lv+4jA1UNKvCDl%TTtf#b(W9P z3<0Aq7j(zm$k%oD0%do0d3P*yF(Y3E&JYrRwJO$wqZMHy&UYW<}b{1P;gCzBZ{cx3hx^sda7IdF0XX3`JExA(ysfp(XB_2?BX(lb`c0`LxNeOlZ z2L^tCxA^-@DD7!rL~O3?9;Ig49+(TB0Elhc8f^)2#ZSotg%S(qQ~OX5bfWOXhTgs% z+t|mNY^6UJf)rtE=i)m*wa>wur-^p~3%CxOTb9HV>cpRq2D(@e22ijbm6jNg0_t zNQHa0c?JsaeF!}Q%@-;qmJypXRZO;&_Lk>o&d_AAI9d`_!hc)$J_MnKX?5|rZqw+p z+wBQfQw3&<4G`VQ+L}5#VxPQW$%71_yQc?9HD(%JL3G&O1_Y26sHgigU86XCxwiNA zbY~if=Wp_dd*iqXJRfVRXSaPA>AsMR!l~(oYPcJe_7!{QOvzrZDea+eJV<=^{F$5x zx(3#rlI{CbLxg`Xc}Rq+!zH#sW$6kihR{V||GoO5^8vd0?XT=qI|RDCJc73QR5h`Eyg9 z^1^kFWSPS@&gVFH|0OwA@}b{`fG4gl2xj?Fj%MY3gj^M@_Au&nH9J|1lVfk+iuswVTkM|4}h5r1vIe!?^4|n>CLL z#`Db@Ag_X3J~(r-6)Jm!w63{%_yI3(?7yiy^_EX?lYrRRIq3~2-~tFT%QX;%%GVIs zGO;-sQ#MY;pzy@0Kf9C&Sq>L+TzG|1{Xj5HPAWAJ4|S!p_1kZRm1q)yzfxMsEY5_H z1V&~tTQMPjrPK$9`Jv^QV`j=&)UY&ER^eegvHxG10dDBldsI*6hh7iI+U9Qg834j zVqX5*^;dCZCzn8rvi&poqR4a%X~u@)3MMH}G&Gkj7?OhajRp-34Uv$&l#W!J*O}aA zP4pHOWBhl#lo)T(>^3G}c-|xy@xZtn&Zu`2bT@O*vJWzhDCjeu)hC69UUiV}+S%EK zvng*=ubE03m{C|ea0}51hX2d`-11mctp-(H^}s+F(5cXNj54>iO$AmM{H^EYcB29!j78f$d*FuJ$J4!hJ4WXPdQ z3swN>{e~`oJ}If+vPFG{g-%e-#U$anM^B%Phm&hx!Z{+7`;7+WMc1T$3!cJMQbQHZb_I=Q9OZ;CV*d+NZ|2TVr&zM$R%8Dap1=|1SKv2Wda zZf^b*H2=>9UHPD(47$O+y}j>!CFfR))(*g0ZiTj8m&?rH1Og#X=iG*al`1UVO#P-k2%0nR+t1MnL}g0c;<{WMpvuAA1+Y zU)`c_FndF1$xH41X6jn--a7`{SBR}W4FdWXuB}*sIN4JfmG5w zccGgHX)7Q4X+rqrnbfQWv_TDyjL5Gfe;kR@i&w9v|t0yQLSTpotwWf&lHuVec+9ou9Nh&uPab zeiO9EGR}PcvLV@$nKtw@xe8KGQVw@` z{rRFfxI5(--ye72%d}r{+;CGw@L7pgGd_umEF2_z{p(;L62^N-rw23L*g;pX@1XgX z()t`9Y@?m9=-tm=99qGG%V{rOLS zKY9QkB^bWmK?YsG!7chzl`nfBit`a-Y~}Z1WR&0%v*pjgiAPuAw}Vh&{-!YE3h2Sl z;kLE*`M~>nH&lVS zyy4vnwT^qY@=g~`Og$Q-*dHKo3y;Kc`(UWUeKl*=Llw)XNcAPfMLziZrw8H~&(WTD z;@-{8mA7D~^>sqDZKdYhf06w``C?Y%&b`dpbwMC~;tJ(g~z{Or*JlCkJ`bOjT!|@{iMZN-# zn23kSZoG5}x*(<9m8pdO$E0@@LsESp_vydU$LTS8d{fQPugHPOkSpr#<3@FLsgeK8 zXqJKsRgcY_xg@GYN)aeqcv3l&2Rb(7c|3s^q`1Dyq{Vl(qEWgbwD)q$MToA*e4QS^ z6Xlgi^&-&^(Kcp&$m$m`+;AujmS82 zaVYTt3Ko1u!#MAsvDa-Q?~<)u9+!Kv@ItxNe{MwQ`6uKI4}$nJRyaXw!!(TGdoYH- znoCnN4c5P>HP#)+?SYWUFcS!sGNS|A|DHJxv-sF;J=Hj z8lLgj9|8bo;c_|HAOmVo3$P*NE+0Wl90#5$@`t^~gTjlSr8A>B2{L2JN9cfl_;;k1 z&G1gR^FV(vCt+$6dM}{&RRc=!K*o4ML(@hpeN{N}a}19UvzWdFI%^iL>^{RT{cq-!8N>bbeqrgvU7Kkqh3lpNZa&?2 z@xmEsMa;9?)P(#KX)OWhhP0)}R$0zr@bK{15AQ>b9I3h}WT`;Bdk?c>gsT{U5sX+x zG&;yNH%K25@T^m_vmh9-Ie-EkQWKIT3T+z}5df2IQ?w5>@ml}=Jraw&oT!~R6tPXq zYHd9f@Z!QCCV;FadkF5pfI?qx!gEnkQ7yPr2c$vw2adU6OmN`}(h&f6A9;a*-g2r2 zeH<>!K>B{i)?jUsdyAlv^lx1dE}(&g^d$hCPoF=xSjC2BfIBff#^X;N$hTV&z`ykX z4mQr6$D9`zmzJ8Lp+N%fYJsb*5Wz?+$gP1bx@u;&uCi_m3@Y5G#GuoHgM)$o#iFF- z|AQzLrV5}Qgjar4K2>XZ@!|y%Goiz0Yd*?so&y(+`;Gi;+b$*DgtyLl#4*#4*rNQ> zzfQ+OjOTViP*9Nc<`s45wt(FQ`EUE~8D!UgaSVO_jmb2<_~@7zzt6lV=+}g6NxVOO zYO*1E0KW7IF@FncqcMx-QOdU{*87O7;tFcUsJ?jR=2N&VC`rfl?2rPQT9V)bA&5}u znJoK7*en0Ds&ZdtmRgmBw)P{?N_N^1P4i}c5S?CGdhR5{k$xWfbqsM3FZA+6tD&c+ zW?*1o;{aM*xE{j%U4Q1 zPgxk+bX6wBiX<}~t*P zmIG=74MN`3CRAO`t*y18Tn-e$GiaHB&Wx~?{~A284ubRUlbTyUgXqeu0Y515wb@KPjc9Qfq$A zMgQMgd+%_d`~H1cyHuzsRHR{6vNEzlM6$98B_ktyWVS?<%E%~$lo7J`2-zc9nPrRY zk(uB5ab5R!9MA8M=Qy6{KJM$d?)$)Jyg%>pdY!NHJYW7)_fVVQ6Zp$}rn0BN8OY7& z`m_C_@P-RRST*cagbHUAJZzv)>-cmUMnV6ozntbqrfXdh<Yw3DEcXnrZpsAuh z5x4#)*fL1n{0YmIc%`W+%Ovfb2R=SNioNy=zcs+!@*i7UUS77I=;DtyfZ7Etdl4J4 zVOW{WK(m2rAuqsvzp>0OX$8tP&;8ZttcRQ^MG{3SJhU3;%ziz8rc%FRM8xnV>9Y2J zK@R>5x0n|zex=@Yo0@E3zorqxg^pvmTH^pL%-k zfYO}fV_T8CJGjUOjxI3h!mY)CUR}BRBi?TmS*f0msQUXptA0ze@Fu_F z-49>OKR2Rs#1CL?l&y2M(}-qh*6pBoDaJo92Km+0)R27yeOOx25?C@E}$GMx$+xQLPGU5L;1W?CQkWloE86OncSJC zWt&Grmv8k&mCm{Hi!yp2#GOe=JpOL$Dq;iK0k6M>W^Mk+5%2m~jYL%`W#vZUycktB zkb{p22ynEmox6FUY4PrwMt!%}_O~~9ESB?sKTpi4a&s088m`JXw;^bWs_~l%)SmUs z%X_$IGsj>(RJ32Rez>6wQStQ`Ct0{s+&c*03Azp{m)!w%KAVfbd=4dd-kRE5e6@9= zp!_e_9|B=<&`p%w#a>}FDN00UOZ7MpF|&VlX0Zb{Msc{sGV2-20I4We5H42OP2v=z zAIFJNY`BuD?b1}&BMvtyh&Y_U9prWIZ4s7nUZBrb%i4nw+A=_{(=-!S!)S`sv z)2lPl>M1uR8^Dcx0q+c@D2ZJM&uSZc59w0pY<+XVO8WNcok9)2Oka`^^?bL)EuYd@ zsDmDz`87kjL8AC)PgxQ_yXmzh{+4*{*9m?4-;_&ZdtY5-w&1 zRYxl`^EsYwX#*`EN2-9U^4Ag$Sl8ICJ`d;U@CZ=%qSOs>Qc2NFU z1HF-uB1}=d%-5kY{nxam-F|?K&Ea})<(C~@+4K#0VeyKs742QUL3^`(YKbn>YGy+I0iu7_~>kiIDJC{tHEF5O3t=K#fMM_piUx8f9NOv`ThRo+DY~&%IrYO`(Zq^+Q-T2$ebJ~hO!7tTpKKuED>%(d5 zv=f|X1CxX3^qW4v9=R3p3BNVnh1hYMDgX*$dIitiz=E%%b053x0z=%xj|;QZbwv`x zpl<1PqsyNPX~{oi zC2dWg|GlkD;lX>!1q+(K!)DNQaTdL<VjqGRs&kbkDc z*UGldQ-tM1L-(#}3H$RV!XrBV>t}?@yFH)CuB>g1UTH4>^10*RtBc|45Ej>0azlAkOpDoSXMUR-lo;dv`!sG*crm9Vx~5)oI#y5C+U0%u=Zc8@ zOW;@z>iFam_nEf6Wp4J)Yv)RpI))n~=o$3nG5Tt3H$--YNENPW zyKZ&Q#hsxoGbcDsg{ipf_CLR_X8B2_Df>PAV2PxP^r_1k61Z@_7&8Y;J0gN%=O)q4 z&$xBFY;xA($=H%#yAio69vd*-6x%kIl93Suz+3sFI){kE8Hps!w_F}J z`J2`P&hp***u-dhu3l~7oERdzig7rXhkJnUU50S`d#b^1m`l6`F*4oa2OR|ct7ej3 zM--VU=iFoVSH;o#YP%HwR{22F!u4XN@a(@ANN_p2b%tO=flCSQn%7jtYO#gEc z96+Vex?Y7tB+g-g;QsH8RlS%}mDp&$#Rv?#hG7)WnSa?uS9bFCh10l3iJ7dx-lsA# zQFwPktd2BeKP6=uVI6|~=xX~%3Js)IG(CxoZeNagUA?S7I$7~V`tm>1<&j2kI*IH* zqGzCM!LVHdiiHBb_zulXv)k2%9*)QVnUL>-6V3pn4leB5`h?}zKXb{kcgBulC*&A} zkqAsJQcc^=B6KDk1fX+zf63<{BAfvSf3bn!gy_8|bd$fRUGcRZzIdEVHT;(HbItGS z3AGzL#B>Qh;bOZ{M2g**DwIPz$;b?W27yyeYh*M`e2n3~L6eF~y8A8#bJS|3eZd7o z+@(4R_X9QWJO$KS!im4n#+C_v+HG*jK!6&?S*D%uI33sd^f2$)wsBnUA;JHH>*SFm zM`GbP6)pKJ>Q!S^1qrg4Uq6aOz~*M)<6C^v30FN%NZpAfZu=f;5>d?LyMuvhAhB%H zb(!|4(m%9Kz9n^H&!~DJeOuU#hN?V1eszw;3{e};Q`k_01KO*9Ai6|i`f6=uu1 zha{3{7dY`?P1MiZU(dDeMTkrc6r>U$&DX)O5hG5DmD7F)BpV)*4fmydc#;954hjee z(5^V{?TbfG6oL*gu6{J{TbkVQNgklZdUU`7DniVydy(7o_%2 zs&lC0eGXEckPBP=o%62QMeSE=cbGYwrW$6r?8WHlOu5!?x~t;6|G@uis2GJDSxrq^ zxZg+`ZGsJx0yAZHcJ=@ZU6`D_wzg7u=9$yV{|yNr99J}A4fNZW+}79gaBEWcvwoil zvZ?la0^d6i=4Q$Gs&}LWYWpw}nN}S=aO8vThACfN6RsCH1rqSY8C1~9>*}5Xbuh@g zZup}J43EQC)ok!>TwGlv#%gaU(tAfpC$wltB!&Fh``6|3#S%J3PUm@+#}0}2Wi)I> zPM;q9{$M~e;9nqoNs|N%f~R6kC&&7fyg4%e|Cy!q4-W^8+vD?tN{|W<8u3lRASxi= zybX`i@SFQC1-ww4kL@)3HOc zRu0ZwQlTQJ*S;TA`A@0i?+dNZAy&tY8Ax{{s61@WrdZz=QbxI#>#yLYn=$~Wh(mcs z7Pj=(#WIO~YiC=m95NXMS)b6izPWgp1s2gZ8_69G^Q^=-`Ec z@Xy474!qN&(nIM>81?rQQc?mWwLemzj{-cxF>N5IUfM7*q`v+dKIxMs*vyva$X8SwRx?I)Mr5^0YcxuWzvGKtc7JSW8$o8PStmC|xE*7a* z`GVm+LJWV*`8ns9{rfnoP;#oN6_MviazE#5ulmmk?#PH-0Y=}3U|&H7Dbji2A#D0d zx!5xN7_P^2-EQ1>{H9+>+A{YEU2+NYUYjd8vY8hF?8yb6yAK;>uo=Wrh(Sf+y@|}u zG^@-kL-jUVSq+x}pDcaA6vx+e;{rZ(9w9PhcJC@V6`V&e@kKqM;J7l)*QKGl0v>#y4YaH9i*k(m_R*0cz^q=1Zic=Y87*d=@NcHmz3Q!Z1 z9K%t7?OrdQ;*nz{Ff~`51*YQyPj+YB(^T#e>CR4wDLD=>7^-9b$_=DViLBTkSb0!j%9pOI#m1e zv$Ywqf&5dFcF)ec@HTad=CG{&c1x&K-*;qCYzvJJqbkj7)%#{4l>^n&rPu6Js(IC% zoa#hedn0x8*M^1$<-CJY#k13vKap=NPCWPjEn-72BEO@fncaNi6Gmodahb#BO@vt= ze4k)bxQWK&20&2Iy?E`K#DZ0Dj?I~Ayu`AZ9-c^`bjV_=EDHW-hRbS^z#bqNpumfO zfLq+Bo<4lI1$ypHAYXA9%!&nH_vtv5eK@2tN&m)b{@)||%Uc!Wv;8;z_C_zJ1t8&9 z$XgFkQ(s2;2%MTDBrxvOhjoi=xF8J#Nkr`oqs>g=l=1Phqw@aYI}%(rEiA%gho1UG zKl|rGpJmT)HH2XsVe>{fe#6`h>7WFlN#oAUGqB;oksa)@`w$XJ2GTu&N>xZ7d&lp{ zgVCR+8o|hNs(T}}{G`W-LY{6>JzH|VpTUF(BpqMyKTD95UTw*z*yjDvyu4CxhZaxU zL^;QF+lemH=e!#U#OZ)4Qcvk*t#8??Mk7Z2Rrkpy?7*gf) zJoV4O^^GmHWHV+g^f)YdJYb!BHNrb#cZFK+AUxMNRc6H~FUJ&wQH#7K2rtD8gQIS0IUH4s=ll2uyC=XB3t zwg~Z*gx@2J7HI!hxcEbMa$5ZpNrvWCOrP+aXV}OZ zqb@B`J)rP-r5W=M2N>&O&L@oB*)H#19}iE*KP@JULAE>&j#%g3!7f5uVw=_+yj7Fi zL}q_PkL_l7duOok=_LKreP#{oeOx z)g4;;qs>Q>vF0-^*;)O)b7->r@NStxb{xtZ)oIheFNE(g{@&sUn`?y0sjk8>GKHPZ zYhof2+3=)otXM?Qnj#C;k%GBvYx_FAZn`x(m(czM4|B$)PeG6fQ|K#bE)|kQ(b_V=s&Wu0#v+JN3HQVG^$?%p@ z1(ddL(;nS@{*6)N(C&Ap0}~9y`NXJufsz1jpn@swhclnhG^Yhh<3;O#FMy7mb0-?6 zLPyZx;7P;P2;nfmXF}6Cs67)X3*mtaSvFijh}3NkI@UH0C}R2BwYZ|RuC`7Veb!yq z9-chlkYgnh&VU8C^k9+>m5ZpUnzqfBFiQvR;4s)P_+nT$u)yT4adIl4!?p z$nUq~SJs8&4)Y07X^X=1(YeBkSFVvmQ37BLMeM7~0t0fB*hX zM7H7KIqToz0UW(DfQsxL$Xe|^niG)s9D%43MpoV^(U1`BJqQynqzVK=Ad_KD=OX-; zOyr|b>AbU*%la@&2k+^tA6{64+3@eZ0djmo%pJh4OLvdB4E8sLD2XSlX2*RH^Aiq; zl255$jtkH}+fG6Vp$9=xg%B@9^#c#}&H)b6 z8>Ffxr(AW&^z=Mg)wNjo5r^#BO>T4Qbn6D(!u)f_LAAXC8WDKA9s~0G6|REv`Lk{63m@; zZirT?DTuSMzIJ(*^3QSqMf)Wyz?9O@X6PEaGS3aYR{LL7lRdk46Atz`pL!9dqvQpq zEJTJNuLCA_Hf&rtjiqt|3k=}3h@B9=EIh8Wt@^hEUnF4G-|)l(TjwXp9^;k0ZvZiX zxHoWjrKGIA-CFYX!A$M$t|P(IH@M!P6{_yF>8)MtKdHaHIr0wGp*UoG`8&9hHGi^0l_k0%u;( zPlw=7if$7mV91T`qoJu<|NEzCn3cqCqi&K%A1M=qNAk_izLnk)RRw;xjm5u?smbyo0(DVK((ESPIR; z8jxt=fzMTfTsq*x+J|@V2oy#D9ut9qE!Wx#P%LE)B!!iG^dQX44B-w3@{hhz&>vmu zOKWaVL((`K`XqV>5wpEyOqB|9IpE`iR)QE_q{dNBVzMP-b{fo!SOH-7)?2 zM6t+Y!oodMF7O9}%GNmiPZ6|ma9tbu8d=KKHa|Nnp3q`F-FGQ^5G8~!rKSBj-XaX7 zv^b}W$yA^^I2@+akY7}9{qwi$#?cu?f-3V!f4{h+BOeKb#Xv0xq`P#MDDRe+Gd8SX znh0V4YDVBzV*{Yz0|YXUD`02#8H_EQwuEyb*%7{CqC5cc;|O$g;rbL&Z0 zoIn2x`S&-}WGKBq9{4e8E0Er?KH)zKTL66XYWT)iAV?vqC73d(rfHIjD&ArVBwfym zdn6Nc>(BH#obhzeMK-|sl3D~wc+?koME_T>9)s)(h>P*~@dxnvd#lV!*b4)B zscI^`uC2`n8{5nV_|Nmzq{7ab3qKIdV3N2Iqs%OAkH*mj;m1k2ub#$e-c z4}N0s{QkibF#y256*`saN}G}50NXO|GD;+)9ok%Y%C+>usEFYcU1SDpuF zuU1E@vS|Eiu>0HPN`>!x_-MlG4H%d`SbBGH?Iv1aL^ak0I_&5Q#vUvd`LpFkYOD&W z>YD4b7I)p!YMHB@PNz1Ya<4O88|NlmZ-IUg0bw7FPMn&za3a@)FA-It1aPXR-3f`yC8DB@6FUg} zFQXL~fLO41ycZXGNNk`O<%j7WQy!A<_5#w@+bYBZj0>H+h)#3|s33FTCV~5l zZjzy*^6p*GiI~6ZMGKiUK`!jwzBhn8^E>_8L`>GPGpvPg%{(YdQU|m)@xg!_38+a% z$4Clds(8)vuzlSoGi$h=^NG+&weWX}b`k5Og;S3EuW`$FwW4yOFcOFx+_GXqfhSxb zmY`-?WF(YeYfYV>G8epVcAMsOR@?zGuhsWoYns;^E+w?=qY5;hI43LfzhTwj07UEj z4S#0Bs0&y6PqN5oKT1$$o|;bQJ-d+q#?tCjy^Z!^vM#-rA8iUGDENGM<(?uq$>x9R zsMi0=WjT9)W|oL9BiIj*ySYnEvXHK_ukKJvn&P!o$Dvc&OfEv>HeI~uK7jXAhg#iV zVXL6K?qP%eJr@J!t?Jv(1H|?l)Y_DdxKz-*4O}B)f4Lzx=z4vzuP&GU-5Id$5M4ic zdZo8EgSk-muAltMwrjO56CUYU>zA$jg4aL|HTgoYH_wZH{fm;0t#6u&t<)ztFq}#wXtr_g7<4I)KUSbs}Sbw0dL19dKFu9F`b;j_sJ?@1BSR+MWCYJbxFA5A1Jd-N!Sx@@EVJ)V5MKEp zIzU^=``$!nr=kQ4C25bb))Sc+ZRx8`#ANpJzQRG|*ov{ZLh~5fXbgPQVTi(9%}pgwudIJ{0<>0)}HC4u?@uVh3rSkC_p~x~ufqX_zcrhh;Q9 zZ+mXzOa5e@WBko=^zh*tjKt2pdk5h=K@xEGdMIik-%vy&XnA9<1Oo-{hH&QUSIg@nFwU3y4-%4tSZN}%+e`zBq0acZMK1j-VhJ>Dqj;)($gu zND9$li)jDn2IpGs@nIBBpr4~Y1BbtT1b7eMb2#&gSmi-Mnd>kq2ip9vmV`q{+_Vc_ zXpn+|!4x{24~>H+OkbFDm&%!9I1Hv0k-W2vkC%<;T;%(%MOCF8@h+CG?e_**?C*=s zQ`>O<0b8p?*^A&r;BY~;MdXh#ZhH{``Vy?jQ?Cz4KNs%9SfS8E39ryDb3?z)pN#QV zk;cP^>B7nee)xjMD+d;aq-`?1s`X#i&Rbnt$`*b?5X*`gdtV@c4{ zD;hQrL_0Xx_i-SChA0ymze^Hd^JtVSE&nWc&Z#$NvoKFMXA*VmUXMIdeq~Wq%Rbhy zpT=JTj539Ty>W4;d$WbB3I5R~Hv(Nij)7$PCanB%cO(p3fY3?Bao2r@N`qceQL!OE z=8fD3t}VAyKF5P}7eo3&bmZi*Mo$bx&pZ%vUQ`oN!Z5IOLYo0Qd?&;6SK0)eKKr>^>dVA>=@vL=+*f{4=zlXp1v_5m`0+0_ptbm(DcHUT{~ zN|ohTOLat6hsya{D!yM?C%Sx2cO1yu&9F7N7s&0iu3hhlHI1FhRWqq=@(YLa{Ozdz zYbhD3({i4y&GG7vE88@`S#203*U`hQMa*eCR^iMw@ZQVGV;{Opt0{d;uhp~*wOiK? za=VX%dZX_$iT06?H1kf!&OG2;-j=j7EMNmToL^;m8%B_h=&{_l9;YJXRP?(61ui6P zd~O;WKV#hZ83*!Y&Rhp`Fl(TJC?jZvebwR{Df-ezKW4`V)`j>A^s-;}pM{_yWPX^X zb<&@>eH63-IF26sMDL}a|M=y~MYoar@Ex2j4dwdAu7G%9!%0~lxBRtYO|qrq*4%DE z`(K;<57D0zj5q$VMSG`W`o^#De|`6!Jz|cuq@^)5kNiML_Q*~sYC;K?N96(HDUW+r zujHI_6FmwX_{E#}-jzV6UES_mLgaNp+280eo%3_lxU{0cr17JwWKnU8bJBEa2iHla zQ_>7b8MvE=$iM?6Jq(FMQR}9FvlEqmi9vj>(`T*7hh7Z*yeHnos}K}xI6;BIiUW2{ zAvU1cBq}2Oi|SF@!@X9ThXbCxJZbSp=37!kOQHbdxxYTYXkn)p)Nt5LCjNv%w=4rB zl%+rJl(u_MeiDcyi*w@f-VyXfcZnvhqBXj$YJA`k#n49k?qILPjL`7J&BouN*?pRY z_=ARgl#&_7D_Q$g-R>`ZwzKhMe)LI!@OXaoByir!aPeM{SG>>*gQ{oJU)$e4EfDVu zeR6VA*H_2igc#6&x_Ucq!r5X@8xUzz6#mhEqN1^xllf~jf11bl9~Du%E*Ne6rUWsc z9Pj6M{l1%QByOfdTWb-Q=+)X0E#rOiZYl4{-*27^qk}pF+@FK-Q;rU24Mgy21yh z7pbmfR<8@Cf~d`j+h2k?0q_n(@Rhix64m&RPSI)ko4##fQ|RHp__@DQrauc>mkb$+cgy5}qm-^?Xv zE!aN(q{vHC(0hRE!q>6%A+l>b6}>j5)wTeT@hxo0?+=|&k2C+gCGkLO=vMBPL$i9* zOz*UXjz@cVP~3%f2H=DqLXV%-Uw*B~S=?zm`i0Sa%YNJQ-(!SNx>lG?oLzs=oEKDg z^gTt?3&qzLNRVwe7{;i6OSixM&9t!eet(-ZUv9XQcM~5|*U2-soPWCFdz+$o&rjgw zR@tezvyuiMj#=VkvrK3+i+nL&n{{S*{a zW$sXURyGQ|t<9^!0gW0Zh$qoh=vrSYlX!A5#xUUStZ7ovn9RqXO@}TWAH4H%XsF~l zf3E4FU$i;VUKGd0BUhwhdmZ8b;)Pzi846QG_bjMr1Z`G8d4iG15jZ-_QbmCe^eiJV z^sSn}O^ePa57`d&TbUl}@n+6vXYc#{W&EA;q{O&WMvBs*+#l-%wlG3kywUFNJbJVP zmwJ?!x5AaR4v3c^a-kZ-a|z`w#be*Uw<{KiTln4Vu(nwt6)atk$>-eD9m2oroO`^w zSgv_*(rJ&k6bIwK7R^3CdZ8S2%6}kPA?tyi2vm#)Dqpf6;{@+d#8m1Kt39okYZi+{ zu8LILJ<&k})`&OdzDr!+ofk=I6SZj%mVBek<+}HL_ujo<;LHGmi=dqlcm`m{5iMq> zXJ(u(HtqtcQLlgEYRH`v(ghA4o_Vi+zOj5L>3!`aT|sfjiHnY{=Hdk=Zr|G}my@?K zNRUyEI^_PZpzT9a9vEkS{{A%Q^h66QBG2{(DJZkXG`jn6{s&ojnEOs-!IkNS%Q!WM zJ9{a^;HJc-ph|J0@7YCyEW#I+<1{=%sRUCpl?q0_&i-5WgeBZr5_DPx@thKgjr*t* z3?FLW@cRR_r8ON}4LfipP8qEC7s)H9V-XkC9xd6e|8KSQugD+9#qB)Dm%rQ7{nqx~ zF`fTWV3#6OPF`JLi&k~yX|cYjJBRC9x~ve&XWdXcKbk|ix%=BU@N_Lm? zoPxjGtSEMicqR*M*Y*7zrr%ZATQ9$C9b4TTDXDjA^-4~ZGP8~XBmaqtOk+-ir2p`e z4&Fnn6*HKtA`81|U@+foM^wU5@CB?Q_i7l;T1w$*FAJ{E%n_e*v7y#)CU$ z@^3$hsYEhNP|-%oUp+a#tk~BuUNPExNhUs;S(@_E(cbc_@_!Wqq^>8OcyYKwyB`_oloyt z+^+LjDOa8_t##b!t3O)II7({JOwLj&o_+U)shW3+%=LYp-ssaEz1c1NRU?(8S5(yh zCEg=AZ)l0MNdale=`nQABHC`D37srm0(Lw}87(HWcC@lf){O8Ql1q^ z@+a&F=--rEvy$Uf$hRYLi{(eQvG)X2uCL-9XGVLhm9$h1Zeh` z{J-^V%NsqDlF+mN`c3YtAaXxTH<5+utXN|zxl68mxi-!@F4eDH?E@|2lp@}m7+6}R zb2yBMc`+#fZz;%m$XvTYANe0`I*{WYTymrSmy(oldu0D{>HJ0ZCx^&>zHqGgq@AE< zDcxV{`XSy?IZ-}C=9HCH+z<5_#nZY5?g4gs5_swx4j?nc;*!Et&GQ{BU<*+aLmcvs+S{6!k?3pMu+Fj&hF!3PEVQ?WR)K^3w< zHU6DuJ~|&R9+4;ODQfiho{o{49voyP3ZrKQy&K>qpdRCoxoORNhqIq8d<6iT!jHfrJ zChn&1Q%jL~_W1(Ob_+Qras|gj;=@#;PxUn>nAiCBy{dn?n_RtcjT4&OyU|w-%svjz ze4>!wh}66`i$`Wkj%)48usHRW%jqg>$4XFRPr<#RiMzJsNzu>U?;ZcU>CWoI!wO9; zoEE-nOLfQtsj!nv1`VF}l|x_i2Q?Nrp;I9#KOm14~i1V7G|;4vd{)#Svr_5?k( zBe#h3$&s24-s_^0xyPnt%yMm$thVhV2LWXSlpk`gfqn%sEoMz*+y`EN<5ps zeqfBsvwAe(uBJ!j`mopj#&0X*;?SvYwX3s~M-K-^>R;9fAX0=AoU`u=q8{2BK7OX zuA&B4Qvwb@{qtSaHP7@-?(z$EccGGTudOzoEE4w_h~UTT_{B(to7WRhktJW1_@zP@ zc-|x2etjybQ^)USbnt5DzQ00<*KUw19@~h(qb@7vI_dwgI^Algvn9q`#{2M78}b;x zzU_npY-T*Dk&qNz+nM&yd%uiUi2oGVvT3D0BHEVyewc7b)*Jh#&R+Vz{7!*!MG?o& zbJXG5H_mTN0_Id@x|mOstNg9^$3-LGCe`ss=B;k~Q_pdumjMnvp@`RO2^bq&O{kcG z$JOpT-yNuZO4@g;8SEE-lhq`a74+*0dFqNIwNpV4E;}u@@%2iM$L6#3O?3pVxAhcA zUAjI*e#E`u=yvL@63SaIUHhgmwLMZHuGvDRdHx#*Z{;vSLDD0O$w(mwZbFGWJJB9xV<`bk_YKQIWk1PuR7+$I$ zo(}0^tqn0jjW(K}*5No@=>n2^P zYjawxpvzQS0B7eITSr(=wQj3>>6e%-b?RQ9UTZ&Yt_8bM3?g_C#4>m@iG_s15LH9c;qT6GcX0h$;%ofJvOIUGpN$URr;1i$vz5kwPdzf;??>b@mUm5} zdCS*Bp7s8{t!%pI(FiGx#vJ*#R@dT&q5uJ;dlP(#XV_uCS6e8{-0*l#A^)>f2W327Ji*b9OB(|(Fp5}13#jEO< zs~X4EeOK9qGVacuu6wBT^GJZk{F|)M%Jfj?wv`+6Q^`4NZ`_X|f{A|k{DK=JZ#r#G zC8Oq9X<{q&8jDpn$=~tUH-2J`c~OnEdN86NIARc`2fXPKJuW% zi2;p#zB}{pWrrVrDW$i|lXFa;lyDX~aMY!1+G5!E;s+~ha*u?6hBJFpn7q+uDUsWq zs_{vBXIS3mE*e#n*4qr;q?3^rUiQ{Kr(Qj+s`fK6>E~He6I$hF!H;(p%;tYSXz@94 z@IXw5uD|%C@1i~FEn`OtNnZX_K0_yx&tnpoTx1Q~^h}q6w&>U%q2GU4)M`?6>MN91 zxduJ%Y8NbWwfectQF)AfRGVrE=uKcbq3jU({xj8E64u#_-~9W%Ib2SPM=if?s*cfb zoZI-K3=<@QJznJ^Tc5e_Tn;Y^Wb=NYq~SepEnGdgTUlqd{k%;q+xj-%c(2Cr66+af zvha~76~&Inx6cQX2YWG|+L&XFk3cnIbvR(+aKQHW6SApgyND?DxaD74i&n=X90S7{ z#O8Gu@9>9Ih)o?4d>p=S%Nv?px5_ifBXiZfnnACkJb49=?P|E(o2zlgq_9V+V&Ccdr=#MJhd+-Fya zPvW~E+gL44EG@Bg(W3$m$2o5>4xR|IR;Rxj*Roe1a)%-vsoswE0QXt>0GYIF`pO>< zQeIK1lm9Z>tX`IQ+s>5az<~q&Fyb#*A#|a@?#BWSQBqDp)C23`O++}r}Rey&oGdP&@nf!L6lmux5F5r(mJ#A0`>$y98ln!n4&?gFA zXU%=T5^F;eG^H*ZHL2yta{Z32d<$+FftSY|Y#Vi=mIuh<<;Eo()VP^O%v!%5_dz>)yMF}vV7cgNhs8AZ!a_P@?Nyz+#5>!koL~)iikCMCs>rTSzE!v4 zueHM76FLPJW@h7(5lknq3-TXSW)+mxZkAl|w?F4?{^RJNTlcQzQMV|8;S2FLh2l*q zL1*@*Ygeb8a%$B*AQY3D{`X-5cWkF!>M3WFA7^7bwkaje4q1C_v$>>F7ai-JI&0WG z%DKmB?;ZSCU_80V)T+t!+z59z4L9{1E%&I_m28aC9br1~c(-)e-<;f&D7;Db>eVmC z+?+SdytGUlhH7b0K_R+D(QZyw~$Yaq@cl^@rn80uO3BhUc96_&Hk_B6S4t72sNgoY; zq@e3hk=yRphSM{*q5G;-kQ`&~4#%z`E}!M}(XF3yb3>72aA37Om5eSOr@V9nN`G_? zkL)3T*2QXmxX40(oI8h2HYiU?_jATFo2z-Ac3ctfsOb*f6Vm#V47|B)5fh`IGwI#? z+_ENVMi+Fulv@Jg1z8kaDicg9}G**Cw_0^p0kJmz^W+yd#aB zWA-lvN{{M14Zit=(bP&E(i^{v>) zSXknUZ{F@4JEt9&{8Mu+urTv_Qu7b3UkOpnZz=&G^pWO2707?;B-o8{vvbTH`!yHbtMoIfJWU;fX5U#LT~OX_^OpRF*p= zu{L9p&4bwH+ZqcOevNLq?m9Z5pxKa96IhOCm&IqQA-HhM3fzfuDo!i_?TvaQ!eB6I+H?kwsVAzQZgVlcWoXD46B85g>J?!L zl9vmnkr9fOSLxD-_(bALBkKWCFapr<;cn>~DkNE*;pc8_KENnzJh~whV9@kX%_9m6 zSW4W!8==V~inEgYiI`Jt=b;lDMs=MWis*B^(X&zYikEkHzGXLP%;ykKSEQ1&PqCW2dQAq7#AOZ9OWj^ zqZ>h@7#fv(NIRO`ZZD5$Iy zbacKgCYKZg1niC+t-v8fD9s4RpFP{Ru&I7L#=`OhMa<}y%%>v$d*^vEx)9V7hIY71 zDcG!NCve7}N!}1KiJjjPR^2`-Dud32sVU=EZhR##Q?^y;WSwy}J3MNNdhPdYtLWIxc=U@Nw$$-|FEle`ZMvY-RymSe%G#j{0&h3@;NV@MjbqzjUJj?8X6jCOIxI0 zb{ArT`n`T#18()3qt)iu5-k1>*ECvn=-4cE4Jt~0MQrd9Z&Ey>qa+v zL5Ola#M5&JX!ewpl-vf!4%*_bcFLyIo#k^pJUmqJNPxY1&(?iBVUO#%%*X$kc6Ub5 zEgZlo8fo2TyWp~T4sK0zV1}TrLgCWTVJ1mOXNbSyoXY!et6zK!k{4lNdqpo__PV}? zwz~={Dqrw6p+c<}PT@Aujp{~;3R<*nn!djJ_V(vd54Sgc@pkBulJnEn_;>HhGLwEg z;q$)7!($Gnft}#(wkoW= zfy4LzqMMsAd*R~oAa>pD#kV-yGZMO!x_@*itEr)X`+1CyZ=-Q~U{;`2L3nh(x{}h( ztb^&VpwfZ`xmh3c>sHGcZd+IHM&kjv2MmKf%bODbbFGDlG0|q&L17%WqboK9HQbon#qYz6QcW#IYUNWlxR&14{MI8Pr~O6 zCPFV^&mW4e^}OmJ(@$0yb7zinBv0CPww7nYK^$TeE|gxlX|Fz<*BZ_484>KSp42dU zu~p%_va0G#^EG)zO-;>VHYIK8ai_SkHosdyB%6Mr36iy>dR9zoD$&x>?N8p1jLIt) zuYPW{`i)-Rke0{I&L1ZJ}VzNcg1e>LVNxWZ^? zjs&S(P1jZBgPg4`H^3li#q&Kl%$V8P9Y)F;Rm)o%t@5E|$aY!c#KO-Mz#|cTz=t9G zF>MXDXaj$t@|wKRmx4b%HSmM`ZJCDwxrR=%@fG=nZRT3ETCpKt$jQhU1qHhzm(Q4M z9~NlH?sIMpxwZE#$xnPzL$qH)e_D#P_F6d&7YdaWNd1Xq5iAqF}?MRf`K?TY90 zMv&Ci?{}DLV`qs5 zWsgBhJbrw$e^nXYUKuIQ<~PTwSM~bPu+VzzqI;Tkl0Mjv!o*=7+vhrpT))g=dcZn#B{!Qk8td}EGL(_Qq5#!5}S-=Y`bXJ`K_Y0v27YHiYn2x z@{XfVdF#2PyXK|Acay&mi^WbXlB|%($Lm5;(r#jtF#g zK&M#Dt8>WsrrmgAqIMmoI}sD$Y(+fYgdL`&q$HTnDr$WD!qU9_OnGzH-ilUJTtI1i@8~46&0WYP6dBob*ieV>8Ge;8+_*W zw_G#La4$q&tQa!{UT|mQwYIjdtgQSwkvD0G$eZL1pVb4HuufOAC7-)d^2qS{wVk~J zw-91mb>i?a$_fh!L1)4pqXD|fQBj*692`6&q&_t9unLnTs`{vQuDZ{}CnVfm8PB4@ zvp~O>n6{1$wT{l8LadTjltCixvaIY$m;)pBnHh%^)Zxx z_;QY8iCLf2Puo{yyY1DxbB72QaW*|Bd{GV^GRpht1WI0B=6vF&J$v^O;Rc-c575)g z!@~;UfFTHwc6}nYBNRv#-a;{0_T|eoF3*>qGzA$-aGfC}(+ZPrkOOs2GhGecS`0q3 zJa~Gz3wr5%QRU_0^2eg$Ktr^W(o$)Ns&MMhFL1H3`Lw4Q!GD+p;z=}PWP!jZ=k`aj zoox(UXccdGOG5GsWJBq=A}M2E#U;orj}G@c0Nt-^MmcrUx$Xu zXlT%2C?sQ)Ui$f!;0-Yl7M873xE)dY6j|*hhwt+zw>2k}nA;)Nf^3ci9f|)SZaWHP z6m0K2T1*BLvwGkpnDzNPJa{c_Z1j=Ufe`u?bINIKex&gs7{#hd+EwtOp@C<`y)#y| zm6b$@im`@qJ=o1R;bTGg&O8@h_JXZm`%w!xPNM_rOohnbm!Mmw;0a`8WJa}L9TLlC zBvPfi_n$t!Ae?5T=F1*OLXKngns* zt>#$8y$dj7gG>&M_`H*noz?A|Cfo@wgezI@hVH-q-` zTI^b(S4Bld3)1cqrU+lWsUQ!5A0i?6bK2EBUKaq4qh z#la&-o@ypC^76K!Ne_j25cW?mLz{6~-9$e>cmVB3pWKsp&!0c_kQgCN?Xz%S(FzMm zTsGzob*eeK^eUnAb|-ascsOweAXtxT`nC8wtpR!FNlwn0$L1#b@RR(zw}>9WLeJb? zQ})B)AQD-;WhxFNthZVW1;OVqA3S65zPh1-{NZO&bZ!rGoe%8x#{3pu`OSqx`~G33 z+proDy*fkaR6Kpd{QdP!)$+_UGy#Csn18`XK8#?1q9m?)G*m>qtv;J@MZt2Y)L>{w zf08{koR2_9@r#JCU_;xi8&In5%}kDs{Rzf*f`jh3Q$Dh1l1_)?I_|#&;D?PDM#qdR z*v`ZHDXFMhSXO|ljA!zieJ9b&8}p(l$*2M~uOtq9;OAHw$XZWwaO{KD7%JnQTS0n< ziFEW|f#^A`sgJ5`VE&-5Y4|EG#A~jlsQCHBb3ri)iLIDT=u7hwc{p-#wB)jQ;^PyI zc9x=b0zx?YnIc>sAkC4Ceu;$*GrThFfYLHD$FT`RRz^WVA&M9uNoENw`p}!IQ^*3_ zE%CvT%46kEe>tQJIRJYE$w4s^0xWq`TT$VOUz`mpTJ@k}Q`O@Pp}t^$8sIhpqPq6p zO*~wdh{NfKW5yzXdOLg@j$;Nu#k^f4#*8zB*fw?D*LVO$M1J~20qb+PWFhiFkI6<= zH8fKJpN0VPacsmTB@G3t=`b)edm+#|f2XSwLRLcCjKHws$BziS5Xaq_0fC0Rn_ozK z9Q*g`IFIw73pzHC+6a+#-}O!uxtMYe!I_Tx8@dD45pg28>0w8`>);>h2$e@wWu@ha zPrL_3lJv%jzlD{JZ9EfbMGcmv#qzJ~43CWC2KM6$e8