From a0c7d4adf9687f4debf11d45aa3fefb375baeea0 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 13:49:28 +0800 Subject: [PATCH 01/28] extract BaseMegatronMapper --- .../mappers/base_megatron_mapper.py | 203 ++++++++++++++++++ chatlearn/synchronizer/mappers/mapper.py | 152 +------------ 2 files changed, 209 insertions(+), 146 deletions(-) create mode 100644 chatlearn/synchronizer/mappers/base_megatron_mapper.py diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py new file mode 100644 index 00000000..0cea6cbd --- /dev/null +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -0,0 +1,203 @@ +# Copyright 2025 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Mapper for Megatron to vLLM""" +from collections import defaultdict +from typing import List, Dict, TYPE_CHECKING, Union + +from megatron.training.utils import unwrap_model + +from chatlearn.configs import PolicyConfig +from chatlearn.configs.megatron_config import MegatronPolicyTrainerConfig +from chatlearn.utils.mappings import ShardedTensorInfo + +from .mapping_helpers import ( + process_normal_tensor, + process_gate_up_tensor, + process_qkv_tensor, + VLLM_HELPERS, + HF_HELPERS +) + +if TYPE_CHECKING: + from megatron.core.models.gpt import GPTModel + from chatlearn.models.megatron_module import MegatronModule + +class BaseMegatronMapper: + """MegatronMapper""" + def __init__( + self, + dst_model_config: PolicyConfig, + model: 'MegatronModule', + *, + mapper_config: Union[VLLM_HELPERS, HF_HELPERS] = VLLM_HELPERS, + ): + """The Mapper for Megatron sync. In each remote Megatron Actor, + the method of this class is called to generate the parameter mapping + between src and dst. Currently, the mapper supports mapping + MCore Model to vLLM or HF Model. + + WARNING: The mapper assumes that the weights name of same + submodules in different vLLM models are still same. + + Args: + dst_model_config (PolicyConfig): The config of target model to + be sychronized + model (MegatronModule): The source Megatron Module + mapper_config (Union[VLLM_HELPERS, HF_HELPERS]): The mapping mode. + """ + self.model: List['GPTModel'] = unwrap_model(model.model) + self._src_model_config: MegatronPolicyTrainerConfig = model.module_args + self._dst_model_config = dst_model_config + self._mapper_config = mapper_config + self._dst_tp_size = 1 if mapper_config.force_full_model else self._dst_model_config.tensor_model_parallel_size + self._src_name_to_metadata: Dict[str, ShardedTensorInfo] = model.get_parameter_metadata(key_type='local_name') + self._dst_name_to_metadata: Dict[str, ShardedTensorInfo] = None + self._mapping = None + + def generate_sync_mapping( + self, + dst_name_to_metadata: Dict[str, ShardedTensorInfo] + ) -> Dict[ShardedTensorInfo, List[ShardedTensorInfo]]: + """ Generate the synchronization mapping of this local rank. + + Args: + dst_name_to_metadata (Dict[str, ShardedTensorInfo]): mapping a global + parameter name to the corresponding ShardedTensorInfo. + + Returns: + Dict[ShardedTensorInfo, List[ShardedTensorInfo]]: The return + dict is the plan including all local parameters to be synchronized. The + mapper will ensure that the key of mapping for each mapping type is + non-overlapping and can merge into the full state dict of this rank. + For most cases, the length of dst shards list is 1, except for GQA with + large vLLM TP. + """ + self._dst_name_to_metadata = dst_name_to_metadata + return self._map_model() + + def dump_sync_mapping(self, folder_path: str, sync_mapping: Dict): + """dump the generayed sync mapping to the given folder path in JSON format. + + Args: + folder_path (str): The folder path to dump the sync mapping. + sync_mapping (Dict): The sync mapping to be saved. + """ + raise NotImplementedError() + + # NOTE: the following function implements the module-wise sync mapping + def _map_model(self): + """Mapping the local name of src model to global name of + dst model + """ + raise NotImplementedError() + + # NOTE: the following function implements the tensor-wise sync mapping + def _inner_map_for_tensor_parallel( + self, + src_key: str, + dst_key: str, + *, + global_expert_id: int=None, + num_experts: int=None, + mapping_type: str='column' + ): + AXES = {'column': 0, 'row': 1} + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + mapping = {} + for src_meta, dst_meta in process_normal_tensor( + src_info, + self._dst_tp_size, + axis=AXES[mapping_type] + ): + src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id + src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype + if global_expert_id is not None: + dst_meta = ( + dst_meta + .unsqueeze(offset=global_expert_id, length=num_experts, axis=0) + .refragment(1, axis=0) # 1 is dst EP + ) + mapping[src_meta] = [dst_meta] + return mapping + + def _inner_map_for_full_shape( + self, + src_key: str, + dst_key: str + ): + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + return { + src_info.copy(): [dst_info.copy()] + } + + def _inner_map_for_gate_up_proj(self, src_key: str, dst_key: str, proj_type: str, *, global_expert_id: int=None, num_experts: int=None): + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + mapping = {} + for src_meta, dst_meta in process_gate_up_tensor( + src_info, + self._dst_tp_size, + proj_type=proj_type + ): + src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id + src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype + if global_expert_id is not None: + dst_meta = ( + dst_meta + .unsqueeze(offset=global_expert_id, length=num_experts, axis=0) + .refragment(1, axis=0) # 1 is dst EP + ) + mapping[src_meta] = [dst_meta] + return mapping + + def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, num_attention_heads: int, num_query_groups: int): + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + mapping = defaultdict(list) + for src_meta, dst_meta in process_qkv_tensor( + src_info, + num_attention_heads, + num_query_groups, + self._dst_tp_size, + proj_type=proj_type + ): + src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id + src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype + mapping[src_meta].append(dst_meta) + return mapping + + def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + dst_meta = src_info.refragment(1) + dst_meta.param_id = dst_info.param_id + dst_meta.dtype = dst_info.dtype + return { + src_info.copy(): [dst_meta] + } + + @property + def _src_arch(self): + return self._src_model_config.megatron_model_cfg + + def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]): + if self._mapping is None: + self._mapping = defaultdict(list) + for src_meta, dst_metas in results.items(): + self._mapping[src_meta] += dst_metas + return self._mapping diff --git a/chatlearn/synchronizer/mappers/mapper.py b/chatlearn/synchronizer/mappers/mapper.py index 0ab95c40..f7714f14 100644 --- a/chatlearn/synchronizer/mappers/mapper.py +++ b/chatlearn/synchronizer/mappers/mapper.py @@ -14,11 +14,9 @@ # ============================================================================== """Mapper for Megatron to vLLM""" -from collections import defaultdict -from typing import List, Dict, Tuple, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Union import inspect -import torch from torch import nn from transformers import AutoConfig @@ -26,22 +24,17 @@ from megatron.core.transformer.transformer_layer import get_transformer_layer_offset from megatron.core.transformer.moe.moe_layer import MoELayer from megatron.core.transformer.moe.experts import TEGroupedMLP -from megatron.training.utils import unwrap_model from chatlearn.configs import PolicyConfig -from chatlearn.configs.megatron_config import MegatronPolicyTrainerConfig -from chatlearn.utils.mappings import ShardedTensorInfo from .mapping_helpers import ( - process_normal_tensor, - process_gate_up_tensor, - process_qkv_tensor, VLLM_HELPERS, HF_HELPERS ) +from .base_megatron_mapper import BaseMegatronMapper + if TYPE_CHECKING: - from megatron.core.models.gpt import GPTModel from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding from megatron.core.transformer.transformer_layer import TransformerLayer from megatron.core.tensor_parallel import ColumnParallelLinear @@ -50,7 +43,7 @@ from megatron.core.transformer.attention import SelfAttention from chatlearn.models.megatron_module import MegatronModule -class MegatronMapper: +class MegatronMapper(BaseMegatronMapper): """MegatronMapper""" def __init__( self, @@ -73,43 +66,7 @@ def __init__( model (MegatronModule): The source Megatron Module mapper_config (Union[VLLM_HELPERS, HF_HELPERS]): The mapping mode. """ - self.model: List['GPTModel'] = unwrap_model(model.model) - self._src_model_config: MegatronPolicyTrainerConfig = model.module_args - self._dst_model_config = dst_model_config - self._mapper_config = mapper_config - self._dst_tp_size = 1 if mapper_config.force_full_model else self._dst_model_config.tensor_model_parallel_size - self._src_name_to_metadata = model.get_parameter_metadata(key_type='local_name') - self._mapping = None - - def generate_sync_mapping( - self, - dst_name_to_metadata: Dict[str, Tuple[int, torch.dtype]] - ) -> Dict[ShardedTensorInfo, List[ShardedTensorInfo]]: - """ Generate the synchronization mapping of this local rank. - - Args: - dst_name_to_metadata (Dict[str, Tuple[int, torch.dtype]]): mapping a global - parameter name to its param_id and datatype - - Returns: - Dict[ShardedTensorInfo, List[ShardedTensorInfo]]: The return - dict is the plan including all local parameters to be synchronized. The - mapper will ensure that the key of mapping for each mapping type is - non-overlapping and can merge into the full state dict of this rank. - For most cases, the length of dst shards list is 1, except for GQA with - large vLLM TP. - """ - self._dst_name_to_metadata = dst_name_to_metadata - return self._map_model() - - def dump_sync_mapping(self, folder_path: str, sync_mapping: Dict): - """dump the generayed sync mapping to the given folder path in JSON format. - - Args: - folder_path (str): The folder path to dump the sync mapping. - sync_mapping (Dict): The sync mapping to be saved. - """ - raise NotImplementedError() + super().__init__(dst_model_config=dst_model_config, model=model, mapper_config=mapper_config) def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): dst_language_prefix = self._mapper_config.dst_language_prefix @@ -232,6 +189,7 @@ def _map_llm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): src_prefix=f"{vp_stage}-output_layer.", dst_prefix="", )) + # NOTE: the following function implements the module-wise sync mapping def _map_model(self): """Mapping the local name of src model to global name of @@ -602,101 +560,3 @@ def _map_postprocess_layer(self, module: 'ColumnParallelLinear', src_prefix='', f"{dst_prefix}lm_head.weight", mapping_type='column' ) - - # NOTE: the following function implements the tensor-wise sync mapping - def _inner_map_for_tensor_parallel( - self, - src_key: str, - dst_key: str, - *, - global_expert_id: int=None, - num_experts: int=None, - mapping_type: str='column' - ): - AXES = {'column': 0, 'row': 1} - src_info = self._src_name_to_metadata[src_key] - dst_info = self._dst_name_to_metadata[dst_key] - mapping = {} - for src_meta, dst_meta in process_normal_tensor( - src_info, - self._dst_tp_size, - axis=AXES[mapping_type] - ): - src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id - src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype - if global_expert_id is not None: - dst_meta = ( - dst_meta - .unsqueeze(offset=global_expert_id, length=num_experts, axis=0) - .refragment(1, axis=0) # 1 is dst EP - ) - mapping[src_meta] = [dst_meta] - return mapping - - def _inner_map_for_full_shape( - self, - src_key: str, - dst_key: str - ): - src_info = self._src_name_to_metadata[src_key] - dst_info = self._dst_name_to_metadata[dst_key] - return { - src_info.copy(): [dst_info.copy()] - } - - def _inner_map_for_gate_up_proj(self, src_key: str, dst_key: str, proj_type: str, *, global_expert_id: int=None, num_experts: int=None): - src_info = self._src_name_to_metadata[src_key] - dst_info = self._dst_name_to_metadata[dst_key] - mapping = {} - for src_meta, dst_meta in process_gate_up_tensor( - src_info, - self._dst_tp_size, - proj_type=proj_type - ): - src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id - src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype - if global_expert_id is not None: - dst_meta = ( - dst_meta - .unsqueeze(offset=global_expert_id, length=num_experts, axis=0) - .refragment(1, axis=0) # 1 is dst EP - ) - mapping[src_meta] = [dst_meta] - return mapping - - def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, num_attention_heads: int, num_query_groups: int): - src_info = self._src_name_to_metadata[src_key] - dst_info = self._dst_name_to_metadata[dst_key] - mapping = defaultdict(list) - for src_meta, dst_meta in process_qkv_tensor( - src_info, - num_attention_heads, - num_query_groups, - self._dst_tp_size, - proj_type=proj_type - ): - src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id - src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype - mapping[src_meta].append(dst_meta) - return mapping - - def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): - src_info = self._src_name_to_metadata[src_key] - dst_info = self._dst_name_to_metadata[dst_key] - dst_meta = src_info.refragment(1) - dst_meta.param_id = dst_info.param_id - dst_meta.dtype = dst_info.dtype - return { - src_info.copy(): [dst_meta] - } - - @property - def _src_arch(self): - return self._src_model_config.megatron_model_cfg - - def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]): - if self._mapping is None: - self._mapping = defaultdict(list) - for src_meta, dst_metas in results.items(): - self._mapping[src_meta] += dst_metas - return self._mapping From f305c7f915751821239938ea213358045a3aea47 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 13:50:52 +0800 Subject: [PATCH 02/28] rename MegatronMapper --- chatlearn/synchronizer/mappers/__init__.py | 4 ++-- .../mappers/{mapper.py => megatron_llm_mapper.py} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename chatlearn/synchronizer/mappers/{mapper.py => megatron_llm_mapper.py} (99%) diff --git a/chatlearn/synchronizer/mappers/__init__.py b/chatlearn/synchronizer/mappers/__init__.py index d041c331..fb2c175a 100644 --- a/chatlearn/synchronizer/mappers/__init__.py +++ b/chatlearn/synchronizer/mappers/__init__.py @@ -34,8 +34,8 @@ def name_to_mapper_cls(mapper_name: str): # pylint: disable=import-outside-toplevel from .mapping_helpers import VLLM_HELPERS, HF_HELPERS if mapper_name in ["MegatronVLLMMapper", "MegatronSGLangMapper"]: - from .mapper import MegatronMapper + from .megatron_llm_mapper import MegatronLLMMapper helper_mappings = {"MegatronVLLMMapper": VLLM_HELPERS, "MegatronSGLangMapper": HF_HELPERS} - return partial(MegatronMapper, mapper_config=helper_mappings[mapper_name]) + return partial(MegatronLLMMapper, mapper_config=helper_mappings[mapper_name]) else: raise ValueError(f"Unrecognized Mapper {mapper_name}") diff --git a/chatlearn/synchronizer/mappers/mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py similarity index 99% rename from chatlearn/synchronizer/mappers/mapper.py rename to chatlearn/synchronizer/mappers/megatron_llm_mapper.py index f7714f14..354c74e9 100644 --- a/chatlearn/synchronizer/mappers/mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -43,8 +43,8 @@ from megatron.core.transformer.attention import SelfAttention from chatlearn.models.megatron_module import MegatronModule -class MegatronMapper(BaseMegatronMapper): - """MegatronMapper""" +class MegatronLLMMapper(BaseMegatronMapper): + """MegatronLLMMapper""" def __init__( self, dst_model_config: PolicyConfig, From 63c74acf5f5134ea8a059681e4ccbcb332266479 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 14:03:25 +0800 Subject: [PATCH 03/28] make update_mapping only called in '_inner_map_*' --- .../mappers/base_megatron_mapper.py | 36 +-- .../mappers/megatron_llm_mapper.py | 250 ++++++++---------- 2 files changed, 133 insertions(+), 153 deletions(-) diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index 0cea6cbd..bd67930a 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -32,11 +32,11 @@ ) if TYPE_CHECKING: - from megatron.core.models.gpt import GPTModel + from megatron.core.transformer.module import MegatronModule as MCoreModule from chatlearn.models.megatron_module import MegatronModule class BaseMegatronMapper: - """MegatronMapper""" + """BaseMegatronMapper""" def __init__( self, dst_model_config: PolicyConfig, @@ -44,7 +44,7 @@ def __init__( *, mapper_config: Union[VLLM_HELPERS, HF_HELPERS] = VLLM_HELPERS, ): - """The Mapper for Megatron sync. In each remote Megatron Actor, + """The Base Mapper for Megatron sync. In each remote Megatron Actor, the method of this class is called to generate the parameter mapping between src and dst. Currently, the mapper supports mapping MCore Model to vLLM or HF Model. @@ -58,7 +58,7 @@ def __init__( model (MegatronModule): The source Megatron Module mapper_config (Union[VLLM_HELPERS, HF_HELPERS]): The mapping mode. """ - self.model: List['GPTModel'] = unwrap_model(model.model) + self.model: List['MCoreModule'] = unwrap_model(model.model) self._src_model_config: MegatronPolicyTrainerConfig = model.module_args self._dst_model_config = dst_model_config self._mapper_config = mapper_config @@ -97,13 +97,11 @@ def dump_sync_mapping(self, folder_path: str, sync_mapping: Dict): """ raise NotImplementedError() - # NOTE: the following function implements the module-wise sync mapping def _map_model(self): - """Mapping the local name of src model to global name of - dst model + """Mapping the local name of src model to global name of dst model """ raise NotImplementedError() - + # NOTE: the following function implements the tensor-wise sync mapping def _inner_map_for_tensor_parallel( self, @@ -116,6 +114,10 @@ def _inner_map_for_tensor_parallel( ): AXES = {'column': 0, 'row': 1} src_info = self._src_name_to_metadata[src_key] + # NOTE: we should do nothing to bias of RowParallel, call full shape mapping. + if src_info.ndim == 1 and mapping_type == 'row': + return self._inner_map_for_full_shape(src_key, dst_key) + dst_info = self._dst_name_to_metadata[dst_key] mapping = {} for src_meta, dst_meta in process_normal_tensor( @@ -132,6 +134,7 @@ def _inner_map_for_tensor_parallel( .refragment(1, axis=0) # 1 is dst EP ) mapping[src_meta] = [dst_meta] + self._update_mapping(mapping) return mapping def _inner_map_for_full_shape( @@ -141,9 +144,9 @@ def _inner_map_for_full_shape( ): src_info = self._src_name_to_metadata[src_key] dst_info = self._dst_name_to_metadata[dst_key] - return { - src_info.copy(): [dst_info.copy()] - } + results = {src_info.copy(): [dst_info.copy()]} + self._update_mapping(results) + return results def _inner_map_for_gate_up_proj(self, src_key: str, dst_key: str, proj_type: str, *, global_expert_id: int=None, num_experts: int=None): src_info = self._src_name_to_metadata[src_key] @@ -163,6 +166,7 @@ def _inner_map_for_gate_up_proj(self, src_key: str, dst_key: str, proj_type: str .refragment(1, axis=0) # 1 is dst EP ) mapping[src_meta] = [dst_meta] + self._update_mapping(mapping) return mapping def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, num_attention_heads: int, num_query_groups: int): @@ -179,6 +183,7 @@ def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, nu src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype mapping[src_meta].append(dst_meta) + self._update_mapping(mapping) return mapping def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): @@ -187,17 +192,16 @@ def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): dst_meta = src_info.refragment(1) dst_meta.param_id = dst_info.param_id dst_meta.dtype = dst_info.dtype - return { - src_info.copy(): [dst_meta] - } + results = {src_info.copy(): [dst_meta]} + self._update_mapping(results) + return results @property def _src_arch(self): return self._src_model_config.megatron_model_cfg - def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]): + def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]) -> None: if self._mapping is None: self._mapping = defaultdict(list) for src_meta, dst_metas in results.items(): self._mapping[src_meta] += dst_metas - return self._mapping diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index 354c74e9..f4c24d3c 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -74,121 +74,121 @@ def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): dst_lm_head_prefix = self._mapper_config.dst_lm_head_prefix if model.pre_process: - self._update_mapping(self._map_preprocess_layer( + self._map_preprocess_layer( model.language_model.embedding, src_prefix=f"{vp_stage}-language_model.embedding.", dst_prefix=f"{dst_language_prefix}", - )) + ) - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{vp_stage}-vision_model.patch_embed.proj.weight", f"{dst_vision_prefix}patch_embed.proj.weight" - )) + ) # vision model decoder for layer_idx in range(model.vision_config.num_layers): global_layer_id = layer_offset + layer_idx - self._update_mapping(self._map_vision_layer( + self._map_vision_layer( model.vision_model.decoder.layers[layer_idx], src_prefix=f"{vp_stage}-vision_model.decoder.layers.{layer_idx}.", dst_prefix=f"{dst_vision_prefix}blocks.{global_layer_id}.", num_attention_heads=model.vision_config.num_attention_heads, num_query_groups=model.vision_config.num_query_groups - )) + ) # vision model projection - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{vp_stage}-vision_model.decoder.final_layernorm.weight", f"{dst_vision_prefix}merger.ln_q.weight" - )) + ) - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{vp_stage}-vision_model.projection.encoder.linear_fc1.weight", f"{dst_vision_prefix}merger.mlp.0.weight", mapping_type='column' - )) + ) - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{vp_stage}-vision_model.projection.encoder.linear_fc1.bias", f"{dst_vision_prefix}merger.mlp.0.bias", mapping_type='column' - )) + ) - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{vp_stage}-vision_model.projection.encoder.linear_fc2.weight", f"{dst_vision_prefix}merger.mlp.2.weight", mapping_type='row' - )) + ) # bias for row is not slice, so we need to map it to full shape - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{vp_stage}-vision_model.projection.encoder.linear_fc2.bias", f"{dst_vision_prefix}merger.mlp.2.bias" - )) + ) for layer_idx in range(model.language_model.decoder.num_layers_per_pipeline_rank): global_layer_id = layer_offset + layer_idx - self._update_mapping(self._map_decoder_layer( + self._map_decoder_layer( model.language_model.decoder.layers[layer_idx], src_prefix=f"{vp_stage}-language_model.decoder.layers.{layer_idx}.", dst_prefix=f"{dst_language_prefix}layers.{global_layer_id}.", - )) + ) if model.post_process: - self._update_mapping(self._map_norm_layer( + self._map_norm_layer( model.language_model.decoder.final_layernorm, src_prefix=f"{vp_stage}-language_model.decoder.final_layernorm.", dst_prefix=f"{dst_language_prefix}norm.", - )) + ) if model.share_embeddings_and_output_weights and model.pre_process: - self._update_mapping(self._map_postprocess_layer( + self._map_postprocess_layer( model.language_model.embedding, src_prefix=f"{vp_stage}-language_model.embedding.word_embeddings.", dst_prefix=f"{dst_lm_head_prefix}", - )) + ) else: - self._update_mapping(self._map_postprocess_layer( + self._map_postprocess_layer( model.language_model.output_layer, src_prefix=f"{vp_stage}-language_model.output_layer.", dst_prefix=f"{dst_lm_head_prefix}", - )) + ) def _map_llm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): if model.pre_process: - self._update_mapping(self._map_preprocess_layer( + self._map_preprocess_layer( model.embedding, src_prefix=f"{vp_stage}-embedding.", dst_prefix="model.", - )) + ) for layer_idx in range(model.decoder.num_layers_per_pipeline_rank): global_layer_id = layer_offset + layer_idx - self._update_mapping(self._map_decoder_layer( + self._map_decoder_layer( model.decoder.layers[layer_idx], src_prefix=f"{vp_stage}-decoder.layers.{layer_idx}.", dst_prefix=f"model.layers.{global_layer_id}.", - )) + ) if model.post_process: - self._update_mapping(self._map_norm_layer( + self._map_norm_layer( model.decoder.final_layernorm, src_prefix=f"{vp_stage}-decoder.final_layernorm.", dst_prefix="model.norm.", - )) + ) if model.share_embeddings_and_output_weights and model.pre_process: - self._update_mapping(self._map_postprocess_layer( + self._map_postprocess_layer( model.embedding, src_prefix=f"{vp_stage}-embedding.word_embeddings.", dst_prefix="", - )) + ) else: - self._update_mapping(self._map_postprocess_layer( + self._map_postprocess_layer( model.output_layer, src_prefix=f"{vp_stage}-output_layer.", dst_prefix="", - )) + ) # NOTE: the following function implements the module-wise sync mapping def _map_model(self): @@ -205,10 +205,8 @@ def _map_model(self): if len(self.model) > 1: mpu.set_virtual_pipeline_model_parallel_rank(None) - if hasattr(model, 'vision_model'): - model.mtp_process = False - - if model.mtp_process: + # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch + if getattr(model, 'mtp_process', None): raise NotImplementedError("Currently, the mapper does not support MTP") if hasattr(model, 'vision_model'): @@ -219,14 +217,12 @@ def _map_model(self): mapping = self._mapping self._mapping = None - return mapping def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str='', *, is_norm_layer: bool=True): """If is_norm_layer is True, try to map on all possible keys, otherwise only map on `layer_norm_weight` and `layer_norm_bias` """ - mapping = {} _keynames = { 'weight': 'weight', 'bias': 'bias', @@ -239,14 +235,12 @@ def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str for item in possible_keys: if getattr(module, item, None) is None or getattr(module, item).numel() == 0: continue - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{src_prefix}{item}", f"{dst_prefix}{_keynames[item]}" - )) - return mapping + ) def _map_decoder_layer(self, module: 'TransformerLayer', src_prefix: str='', dst_prefix: str=''): - mapping = {} if self._src_arch.multi_latent_attention: map_attn_func = self._map_mla_selfattn norm_layer = module.input_layernorm @@ -259,8 +253,8 @@ def _map_decoder_layer(self, module: 'TransformerLayer', src_prefix: str='', dst norm_src_key = f"{src_prefix}self_attention.linear_qkv." norm_dst_key = f"{dst_prefix}input_layernorm." is_norm_layer = False - self._update_mapping(map_attn_func(module.self_attention, src_prefix=f"{src_prefix}self_attention.", dst_prefix=f"{dst_prefix}self_attn.")) - self._update_mapping(self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer)) + map_attn_func(module.self_attention, src_prefix=f"{src_prefix}self_attention.", dst_prefix=f"{dst_prefix}self_attn.") + self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer) if isinstance(module.mlp, MoELayer): map_mlp_func = self._map_moe_layer @@ -274,9 +268,8 @@ def _map_decoder_layer(self, module: 'TransformerLayer', src_prefix: str='', dst norm_src_key = f"{src_prefix}mlp.linear_fc1." norm_dst_key = f"{dst_prefix}post_attention_layernorm." is_norm_layer = False - self._update_mapping(map_mlp_func(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.")) - self._update_mapping(self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer)) - return mapping + map_mlp_func(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.") + self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer) def _map_vision_layer( self, @@ -286,61 +279,58 @@ def _map_vision_layer( num_attention_heads: int = None, num_query_groups: int = None ): - mapping = {} - # module.self_attention # linear_proj - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{src_prefix}self_attention.linear_proj.weight", f"{dst_prefix}attn.proj.weight", mapping_type='row' - )) + ) # bias for row is not slice, so we need to map it to full shape - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{src_prefix}self_attention.linear_proj.bias", f"{dst_prefix}attn.proj.bias" - )) + ) # linear_qkv - self._update_mapping(self._inner_map_for_qkv_proj( + self._inner_map_for_qkv_proj( f"{src_prefix}self_attention.linear_qkv.weight", f"{dst_prefix}attn.qkv.weight", proj_type='qkv_proj', num_attention_heads=num_attention_heads, num_query_groups=num_query_groups - )) + ) if self._src_arch.add_qkv_bias: - self._update_mapping(self._inner_map_for_qkv_proj( + self._inner_map_for_qkv_proj( f"{src_prefix}self_attention.linear_qkv.bias", f"{dst_prefix}attn.qkv.bias", proj_type='qkv_proj', num_attention_heads=num_attention_heads, num_query_groups=num_query_groups - )) + ) # linear_qkv_norm - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{src_prefix}self_attention.linear_qkv.layer_norm_weight", f"{dst_prefix}norm1.weight" - )) + ) # module.mlp - self._update_mapping(self._map_mlp(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.", is_vision_block=True)) + self._map_mlp(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.", is_vision_block=True) # mlp norm - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{src_prefix}mlp.linear_fc1.layer_norm_weight", f"{dst_prefix}norm2.weight" - )) - return mapping + ) def _map_moe_layer(self, module: 'MoELayer', src_prefix='', dst_prefix=''): mapping = {} # router - self._update_mapping(self._inner_map_for_full_shape(f"{src_prefix}router.weight", f"{dst_prefix}gate.weight")) + self._inner_map_for_full_shape(f"{src_prefix}router.weight", f"{dst_prefix}gate.weight") if module.router.enable_expert_bias: - self._update_mapping(self._inner_map_for_full_shape(f"{src_prefix}router.expert_bias", f"{dst_prefix}gate.e_score_correction_bias")) + self._inner_map_for_full_shape(f"{src_prefix}router.expert_bias", f"{dst_prefix}gate.e_score_correction_bias") if not module.config.moe_grouped_gemm: raise NotImplementedError("Parameter Sync w/ MoE SequentialMLP is not supported") @@ -348,35 +338,32 @@ def _map_moe_layer(self, module: 'MoELayer', src_prefix='', dst_prefix=''): raise NotImplementedError("Parameter Sync w/ Legacy GroupedMLP is not supported") # experts - self._update_mapping(self._map_group_mlp( + self._map_group_mlp( module.experts, src_prefix=f"{src_prefix}experts.", dst_prefix=f"{dst_prefix}experts." - )) + ) # shared experts if module.shared_experts is not None: if module.shared_experts.use_shared_expert_gate: - self._update_mapping( - self._inner_map_for_full_shape( - f"{src_prefix}shared_experts.gate_weight", - f"{dst_prefix}shared_expert_gate.weight" - ) + self._inner_map_for_full_shape( + f"{src_prefix}shared_experts.gate_weight", + f"{dst_prefix}shared_expert_gate.weight" ) # NOTE: if transformer.config have n_shared_experts, mapping to `shared_experts`, otherwise `shared_expert` # `shared_experts`: DeepSeek-V2, DeepSeek-V3, etc. # `shared_expert`: Qwen2-MoE, LLaMA-4, etc. hf_config = AutoConfig.from_pretrained(self._dst_model_config.load, trust_remote_code=self._dst_model_config.trust_remote_code) shared_expert_key = 'shared_experts' if hasattr(hf_config, 'n_shared_experts') else 'shared_expert' - self._update_mapping(self._map_mlp( + self._map_mlp( module.shared_experts, src_prefix=f"{src_prefix}shared_experts.", dst_prefix=f"{dst_prefix}{shared_expert_key}." - )) + ) return mapping def _map_mlp(self, module: 'MLP', src_prefix: str='', dst_prefix: str='', is_vision_block=False): - mapping = {} if not module.config.gated_linear_unit: raise NotImplementedError("Parameter Sync w/o GatedLinear is not supported") @@ -385,31 +372,30 @@ def _map_mlp(self, module: 'MLP', src_prefix: str='', dst_prefix: str='', is_vis dst_names = ['gate_up_proj'] for dst_name in dst_names: - self._update_mapping(self._inner_map_for_gate_up_proj( + self._inner_map_for_gate_up_proj( f"{src_prefix}linear_fc1.weight", f"{dst_prefix}{dst_name}.weight", proj_type=dst_name - )) + ) if module.config.add_bias_linear: - self._update_mapping(self._inner_map_for_gate_up_proj( + self._inner_map_for_gate_up_proj( f"{src_prefix}linear_fc1.bias", f"{dst_prefix}{dst_name}.bias", proj_type=dst_name - )) + ) - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_fc2.weight", f"{dst_prefix}down_proj.weight", mapping_type='row' - )) + ) if module.config.add_bias_linear: - self._update_mapping(self._inner_map_for_full_shape( + self._inner_map_for_full_shape( f"{src_prefix}linear_fc2.bias", f"{dst_prefix}down_proj.bias" - )) - return mapping + ) def _map_group_mlp(self, module: 'TEGroupedMLP', src_prefix: str='', dst_prefix: str=''): # pylint: disable=unused-argument @@ -418,131 +404,121 @@ def _map_group_mlp(self, module: 'TEGroupedMLP', src_prefix: str='', dst_prefix: num_experts = self._src_arch.num_experts global_expert_id_start = num_experts // src_ep_size * src_ep_rank global_expert_id_end = num_experts // src_ep_size * (src_ep_rank + 1) - mapping = {} for local_expert_id, global_expert_id in enumerate(range(global_expert_id_start, global_expert_id_end)): if self._mapper_config.merge_expert: if not self._mapper_config.merge_gate_up: raise NotImplementedError("merge_expert w/o merge_gate_up is not implemented.") - self._update_mapping(self._inner_map_for_gate_up_proj( + self._inner_map_for_gate_up_proj( f"{src_prefix}linear_fc1.weight{local_expert_id}", f"{dst_prefix}w13_weight", proj_type='gate_up_proj', global_expert_id=global_expert_id, num_experts=num_experts - )) - self._update_mapping(self._inner_map_for_tensor_parallel( + ) + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_fc2.weight{local_expert_id}", f"{dst_prefix}w2_weight", global_expert_id=global_expert_id, num_experts=num_experts, mapping_type='row' - )) + ) else: if self._mapper_config.merge_gate_up: raise NotImplementedError("no merge_expert w/ merge_gate_up is not implemented.") for dst_name in ['gate_proj', 'up_proj']: - self._update_mapping(self._inner_map_for_gate_up_proj( + self._inner_map_for_gate_up_proj( f"{src_prefix}linear_fc1.weight{local_expert_id}", f"{dst_prefix}{global_expert_id}.{dst_name}.weight", proj_type=dst_name, - )) - self._update_mapping(self._inner_map_for_tensor_parallel( + ) + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_fc2.weight{local_expert_id}", f"{dst_prefix}{global_expert_id}.down_proj.weight", mapping_type='row' - )) - return mapping + ) def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_prefix: str=''): - mapping = {} if self._src_arch.q_lora_rank is None: - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_q_proj.weight", f"{dst_prefix}q_proj.weight", mapping_type='column' - )) + ) else: - self._update_mapping(self._inner_map_for_mla_down_proj( + self._inner_map_for_mla_down_proj( f"{src_prefix}linear_q_down_proj.weight", f"{dst_prefix}q_a_proj.weight", - )) - self._update_mapping(self._inner_map_for_tensor_parallel( + ) + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_q_up_proj.weight", f"{dst_prefix}q_b_proj.weight", mapping_type='column' - )) + ) if self._src_arch.qk_layernorm: - self._update_mapping( - self._map_norm_layer( - module.linear_q_up_proj, - f"{src_prefix}linear_q_up_proj.", - f"{dst_prefix}q_a_layernorm.", - is_norm_layer=False - ) + self._map_norm_layer( + module.linear_q_up_proj, + f"{src_prefix}linear_q_up_proj.", + f"{dst_prefix}q_a_layernorm.", + is_norm_layer=False ) - self._update_mapping(self._inner_map_for_mla_down_proj( + self._inner_map_for_mla_down_proj( f"{src_prefix}linear_kv_down_proj.weight", f"{dst_prefix}kv_a_proj_with_mqa.weight", - )) - self._update_mapping(self._inner_map_for_tensor_parallel( + ) + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_kv_up_proj.weight", f"{dst_prefix}kv_b_proj.weight", mapping_type='column' - )) + ) if self._src_arch.qk_layernorm: - self._update_mapping( - self._map_norm_layer( - module.linear_kv_up_proj, - f"{src_prefix}linear_kv_up_proj.", - f"{dst_prefix}kv_a_layernorm.", - is_norm_layer=False - ) + self._map_norm_layer( + module.linear_kv_up_proj, + f"{src_prefix}linear_kv_up_proj.", + f"{dst_prefix}kv_a_layernorm.", + is_norm_layer=False ) - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_proj.weight", f"{dst_prefix}o_proj.weight", mapping_type='row' - )) - return mapping + ) def _map_selfattn(self, module: 'SelfAttention', src_prefix: str='', dst_prefix: str=''): - mapping = {} if self._src_arch.qk_layernorm: - self._update_mapping(self._map_norm_layer(module.q_layernorm, f"{src_prefix}q_layernorm.", f"{dst_prefix}q_norm.")) - self._update_mapping(self._map_norm_layer(module.k_layernorm, f"{src_prefix}k_layernorm.", f"{dst_prefix}k_norm.")) + self._map_norm_layer(module.q_layernorm, f"{src_prefix}q_layernorm.", f"{dst_prefix}q_norm.") + self._map_norm_layer(module.k_layernorm, f"{src_prefix}k_layernorm.", f"{dst_prefix}k_norm.") dst_names = ['q_proj', 'k_proj', 'v_proj'] if self._mapper_config.merge_qkv: dst_names = ['qkv_proj'] for dst_name in dst_names: - self._update_mapping(self._inner_map_for_qkv_proj( + self._inner_map_for_qkv_proj( f"{src_prefix}linear_qkv.weight", f"{dst_prefix}{dst_name}.weight", proj_type=dst_name, num_attention_heads = self._src_arch.num_attention_heads, num_query_groups = self._src_arch.num_query_groups - )) + ) if self._src_arch.add_qkv_bias: - self._update_mapping(self._inner_map_for_qkv_proj( + self._inner_map_for_qkv_proj( f"{src_prefix}linear_qkv.bias", f"{dst_prefix}{dst_name}.bias", proj_type=dst_name, num_attention_heads = self._src_arch.num_attention_heads, num_query_groups = self._src_arch.num_query_groups - )) + ) - self._update_mapping(self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{src_prefix}linear_proj.weight", f"{dst_prefix}o_proj.weight", mapping_type='row' - )) - return mapping + ) def _map_preprocess_layer(self, module: 'LanguageModelEmbedding', src_prefix='', dst_prefix=''): if module.add_position_embedding: raise NotImplementedError("learned_absolute embedding is not supported") - return self._inner_map_for_tensor_parallel( + self._inner_map_for_tensor_parallel( f"{src_prefix}word_embeddings.weight", f"{dst_prefix}embed_tokens.weight", mapping_type='column' @@ -554,8 +530,8 @@ def _map_postprocess_layer(self, module: 'ColumnParallelLinear', src_prefix='', not self._src_arch.untie_embeddings_and_output_weights and f"{dst_prefix}lm_head.weight" not in self._dst_name_to_metadata ): - return {} - return self._inner_map_for_tensor_parallel( + return + self._inner_map_for_tensor_parallel( f"{src_prefix}weight", f"{dst_prefix}lm_head.weight", mapping_type='column' From 3983fa171515caa4d8d76aa8674a447a6ca6c0a4 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 14:15:14 +0800 Subject: [PATCH 04/28] extract Megatron Mapper for VLM --- chatlearn/synchronizer/mappers/__init__.py | 28 ++- .../mappers/megatron_llm_mapper.py | 146 +---------- .../mappers/megatron_vlm_mapper.py | 230 ++++++++++++++++++ 3 files changed, 250 insertions(+), 154 deletions(-) create mode 100644 chatlearn/synchronizer/mappers/megatron_vlm_mapper.py diff --git a/chatlearn/synchronizer/mappers/__init__.py b/chatlearn/synchronizer/mappers/__init__.py index fb2c175a..58a469af 100644 --- a/chatlearn/synchronizer/mappers/__init__.py +++ b/chatlearn/synchronizer/mappers/__init__.py @@ -22,20 +22,30 @@ def get_mapper_name(src_model: 'DistModel', dst_model: 'DistModel'): src_type = src_model.runtime_args.train_backend dst_type = dst_model.runtime_args.rollout_backend - if src_type == 'megatron' and dst_type == 'vllm': - return "MegatronVLLMMapper" - elif src_type == 'megatron' and dst_type == 'sglang': - return "MegatronSGLangMapper" - else: - raise NotImplementedError(f"Unsupported src/dst model combination: {src_type}-{dst_type}") - + model_type = src_model.runtime_args.model_type # llm or vlm + + mapping = { + 'llm-megatron-vllm': "MegatronVLLMMapper-LLM", + 'llm-megatron-sglang': "MegatronSGLangMapper-LLM", + 'vlm-megatron-vllm': "MegatronVLLMMapper-VLM", + 'vlm-megatron-sglang': "MegatronSGLangMapper-VLM", + } + key = f'{model_type}-{src_type}-{dst_type}' + if key not in mapping: + raise NotImplementedError(f"Unsupported src/dst model combination: {key}") + return mapping[key] + def name_to_mapper_cls(mapper_name: str): # pylint: disable=import-outside-toplevel from .mapping_helpers import VLLM_HELPERS, HF_HELPERS - if mapper_name in ["MegatronVLLMMapper", "MegatronSGLangMapper"]: + if mapper_name in ["MegatronVLLMMapper-LLM", "MegatronSGLangMapper-LLM"]: from .megatron_llm_mapper import MegatronLLMMapper - helper_mappings = {"MegatronVLLMMapper": VLLM_HELPERS, "MegatronSGLangMapper": HF_HELPERS} + helper_mappings = {"MegatronVLLMMapper-LLM": VLLM_HELPERS, "MegatronSGLangMapper-LLM": HF_HELPERS} return partial(MegatronLLMMapper, mapper_config=helper_mappings[mapper_name]) + elif mapper_name in ["MegatronVLLMMapper-VLM", "MegatronSGLangMapper-VLM"]: + from .megatron_vlm_mapper import MegatronVLMMapper + helper_mappings = {"MegatronVLLMMapper-VLM": VLLM_HELPERS, "MegatronSGLangMapper-VLM": HF_HELPERS} + return partial(MegatronVLMMapper, mapper_config=helper_mappings[mapper_name]) else: raise ValueError(f"Unrecognized Mapper {mapper_name}") diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index f4c24d3c..f40dc3a4 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -68,92 +68,6 @@ def __init__( """ super().__init__(dst_model_config=dst_model_config, model=model, mapper_config=mapper_config) - def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): - dst_language_prefix = self._mapper_config.dst_language_prefix - dst_vision_prefix = self._mapper_config.dst_vision_prefix - dst_lm_head_prefix = self._mapper_config.dst_lm_head_prefix - - if model.pre_process: - self._map_preprocess_layer( - model.language_model.embedding, - src_prefix=f"{vp_stage}-language_model.embedding.", - dst_prefix=f"{dst_language_prefix}", - ) - - self._inner_map_for_full_shape( - f"{vp_stage}-vision_model.patch_embed.proj.weight", - f"{dst_vision_prefix}patch_embed.proj.weight" - ) - - # vision model decoder - for layer_idx in range(model.vision_config.num_layers): - global_layer_id = layer_offset + layer_idx - self._map_vision_layer( - model.vision_model.decoder.layers[layer_idx], - src_prefix=f"{vp_stage}-vision_model.decoder.layers.{layer_idx}.", - dst_prefix=f"{dst_vision_prefix}blocks.{global_layer_id}.", - num_attention_heads=model.vision_config.num_attention_heads, - num_query_groups=model.vision_config.num_query_groups - ) - - # vision model projection - self._inner_map_for_full_shape( - f"{vp_stage}-vision_model.decoder.final_layernorm.weight", - f"{dst_vision_prefix}merger.ln_q.weight" - ) - - self._inner_map_for_tensor_parallel( - f"{vp_stage}-vision_model.projection.encoder.linear_fc1.weight", - f"{dst_vision_prefix}merger.mlp.0.weight", - mapping_type='column' - ) - - self._inner_map_for_tensor_parallel( - f"{vp_stage}-vision_model.projection.encoder.linear_fc1.bias", - f"{dst_vision_prefix}merger.mlp.0.bias", - mapping_type='column' - ) - - self._inner_map_for_tensor_parallel( - f"{vp_stage}-vision_model.projection.encoder.linear_fc2.weight", - f"{dst_vision_prefix}merger.mlp.2.weight", - mapping_type='row' - ) - - # bias for row is not slice, so we need to map it to full shape - self._inner_map_for_full_shape( - f"{vp_stage}-vision_model.projection.encoder.linear_fc2.bias", - f"{dst_vision_prefix}merger.mlp.2.bias" - ) - - for layer_idx in range(model.language_model.decoder.num_layers_per_pipeline_rank): - global_layer_id = layer_offset + layer_idx - self._map_decoder_layer( - model.language_model.decoder.layers[layer_idx], - src_prefix=f"{vp_stage}-language_model.decoder.layers.{layer_idx}.", - dst_prefix=f"{dst_language_prefix}layers.{global_layer_id}.", - ) - - if model.post_process: - self._map_norm_layer( - model.language_model.decoder.final_layernorm, - src_prefix=f"{vp_stage}-language_model.decoder.final_layernorm.", - dst_prefix=f"{dst_language_prefix}norm.", - ) - - if model.share_embeddings_and_output_weights and model.pre_process: - self._map_postprocess_layer( - model.language_model.embedding, - src_prefix=f"{vp_stage}-language_model.embedding.word_embeddings.", - dst_prefix=f"{dst_lm_head_prefix}", - ) - else: - self._map_postprocess_layer( - model.language_model.output_layer, - src_prefix=f"{vp_stage}-language_model.output_layer.", - dst_prefix=f"{dst_lm_head_prefix}", - ) - def _map_llm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): if model.pre_process: self._map_preprocess_layer( @@ -209,11 +123,7 @@ def _map_model(self): if getattr(model, 'mtp_process', None): raise NotImplementedError("Currently, the mapper does not support MTP") - if hasattr(model, 'vision_model'): - self._map_vlm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) - else: - # llm model - self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) mapping = self._mapping self._mapping = None @@ -271,60 +181,6 @@ def _map_decoder_layer(self, module: 'TransformerLayer', src_prefix: str='', dst map_mlp_func(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.") self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer) - def _map_vision_layer( - self, - module: 'TransformerLayer', - src_prefix: str = '', - dst_prefix: str = '', - num_attention_heads: int = None, - num_query_groups: int = None - ): - # module.self_attention - # linear_proj - self._inner_map_for_tensor_parallel( - f"{src_prefix}self_attention.linear_proj.weight", - f"{dst_prefix}attn.proj.weight", - mapping_type='row' - ) - - # bias for row is not slice, so we need to map it to full shape - self._inner_map_for_full_shape( - f"{src_prefix}self_attention.linear_proj.bias", - f"{dst_prefix}attn.proj.bias" - ) - - # linear_qkv - self._inner_map_for_qkv_proj( - f"{src_prefix}self_attention.linear_qkv.weight", - f"{dst_prefix}attn.qkv.weight", - proj_type='qkv_proj', - num_attention_heads=num_attention_heads, - num_query_groups=num_query_groups - ) - if self._src_arch.add_qkv_bias: - self._inner_map_for_qkv_proj( - f"{src_prefix}self_attention.linear_qkv.bias", - f"{dst_prefix}attn.qkv.bias", - proj_type='qkv_proj', - num_attention_heads=num_attention_heads, - num_query_groups=num_query_groups - ) - - # linear_qkv_norm - self._inner_map_for_full_shape( - f"{src_prefix}self_attention.linear_qkv.layer_norm_weight", - f"{dst_prefix}norm1.weight" - ) - - # module.mlp - self._map_mlp(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.", is_vision_block=True) - - # mlp norm - self._inner_map_for_full_shape( - f"{src_prefix}mlp.linear_fc1.layer_norm_weight", - f"{dst_prefix}norm2.weight" - ) - def _map_moe_layer(self, module: 'MoELayer', src_prefix='', dst_prefix=''): mapping = {} # router diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py new file mode 100644 index 00000000..e7e1529c --- /dev/null +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -0,0 +1,230 @@ +# Copyright 2025 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Mapper for Megatron to vLLM""" +from typing import TYPE_CHECKING, Union + +import inspect +from torch import nn + +from megatron.core import mpu +from megatron.core.transformer.transformer_layer import get_transformer_layer_offset + +from chatlearn.configs import PolicyConfig + +from .mapping_helpers import ( + VLLM_HELPERS, + HF_HELPERS +) + +from .megatron_llm_mapper import MegatronLLMMapper + +if TYPE_CHECKING: + from chatlearn.models.megatron_module import MegatronModule + +class MegatronVLMMapper(MegatronLLMMapper): + """MegatronVLMMapper""" + def __init__( + self, + dst_model_config: PolicyConfig, + model: 'MegatronModule', + *, + mapper_config: Union[VLLM_HELPERS, HF_HELPERS] = VLLM_HELPERS, + ): + """The VLM Mapper for Megatron sync. In each remote Megatron Actor, + the method of this class is called to generate the parameter mapping + between src and dst. Currently, the mapper supports mapping + MCore Model to vLLM or HF Model. + + WARNING: The mapper assumes that the weights name of same + submodules in different vLLM models are still same. + + Args: + dst_model_config (PolicyConfig): The config of target model to + be sychronized + model (MegatronModule): The source Megatron Module + mapper_config (Union[VLLM_HELPERS, HF_HELPERS]): The mapping mode. + """ + super().__init__(dst_model_config=dst_model_config, model=model, mapper_config=mapper_config) + + def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): + dst_language_prefix = self._mapper_config.dst_language_prefix + dst_vision_prefix = self._mapper_config.dst_vision_prefix + dst_lm_head_prefix = self._mapper_config.dst_lm_head_prefix + + if model.pre_process: + self._map_preprocess_layer( + model.language_model.embedding, + src_prefix=f"{vp_stage}-language_model.embedding.", + dst_prefix=f"{dst_language_prefix}", + ) + + self._inner_map_for_full_shape( + f"{vp_stage}-vision_model.patch_embed.proj.weight", + f"{dst_vision_prefix}patch_embed.proj.weight" + ) + + # vision model decoder + for layer_idx in range(model.vision_config.num_layers): + global_layer_id = layer_offset + layer_idx + self._map_vision_layer( + model.vision_model.decoder.layers[layer_idx], + src_prefix=f"{vp_stage}-vision_model.decoder.layers.{layer_idx}.", + dst_prefix=f"{dst_vision_prefix}blocks.{global_layer_id}.", + num_attention_heads=model.vision_config.num_attention_heads, + num_query_groups=model.vision_config.num_query_groups + ) + + # vision model projection + self._inner_map_for_full_shape( + f"{vp_stage}-vision_model.decoder.final_layernorm.weight", + f"{dst_vision_prefix}merger.ln_q.weight" + ) + + self._inner_map_for_tensor_parallel( + f"{vp_stage}-vision_model.projection.encoder.linear_fc1.weight", + f"{dst_vision_prefix}merger.mlp.0.weight", + mapping_type='column' + ) + + self._inner_map_for_tensor_parallel( + f"{vp_stage}-vision_model.projection.encoder.linear_fc1.bias", + f"{dst_vision_prefix}merger.mlp.0.bias", + mapping_type='column' + ) + + self._inner_map_for_tensor_parallel( + f"{vp_stage}-vision_model.projection.encoder.linear_fc2.weight", + f"{dst_vision_prefix}merger.mlp.2.weight", + mapping_type='row' + ) + + # bias for row is not slice, so we need to map it to full shape + self._inner_map_for_full_shape( + f"{vp_stage}-vision_model.projection.encoder.linear_fc2.bias", + f"{dst_vision_prefix}merger.mlp.2.bias" + ) + + for layer_idx in range(model.language_model.decoder.num_layers_per_pipeline_rank): + global_layer_id = layer_offset + layer_idx + self._map_decoder_layer( + model.language_model.decoder.layers[layer_idx], + src_prefix=f"{vp_stage}-language_model.decoder.layers.{layer_idx}.", + dst_prefix=f"{dst_language_prefix}layers.{global_layer_id}.", + ) + + if model.post_process: + self._map_norm_layer( + model.language_model.decoder.final_layernorm, + src_prefix=f"{vp_stage}-language_model.decoder.final_layernorm.", + dst_prefix=f"{dst_language_prefix}norm.", + ) + + if model.share_embeddings_and_output_weights and model.pre_process: + self._map_postprocess_layer( + model.language_model.embedding, + src_prefix=f"{vp_stage}-language_model.embedding.word_embeddings.", + dst_prefix=f"{dst_lm_head_prefix}", + ) + else: + self._map_postprocess_layer( + model.language_model.output_layer, + src_prefix=f"{vp_stage}-language_model.output_layer.", + dst_prefix=f"{dst_lm_head_prefix}", + ) + + # NOTE: the following function implements the module-wise sync mapping + def _map_model(self): + """Mapping the local name of src model to global name of + dst model + """ + for vp_stage, model in enumerate(self.model): + if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: + layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) + else: + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) + layer_offset = get_transformer_layer_offset(model.config) + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(None) + + # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch + if getattr(model, 'mtp_process', None): + raise NotImplementedError("Currently, the mapper does not support MTP") + + if hasattr(model, 'vision_model'): + self._map_vlm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + else: + # llm model + self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + + mapping = self._mapping + self._mapping = None + return mapping + + def _map_vision_layer( + self, + module: 'TransformerLayer', + src_prefix: str = '', + dst_prefix: str = '', + num_attention_heads: int = None, + num_query_groups: int = None + ): + # module.self_attention + # linear_proj + self._inner_map_for_tensor_parallel( + f"{src_prefix}self_attention.linear_proj.weight", + f"{dst_prefix}attn.proj.weight", + mapping_type='row' + ) + + # bias for row is not slice, so we need to map it to full shape + self._inner_map_for_full_shape( + f"{src_prefix}self_attention.linear_proj.bias", + f"{dst_prefix}attn.proj.bias" + ) + + # linear_qkv + self._inner_map_for_qkv_proj( + f"{src_prefix}self_attention.linear_qkv.weight", + f"{dst_prefix}attn.qkv.weight", + proj_type='qkv_proj', + num_attention_heads=num_attention_heads, + num_query_groups=num_query_groups + ) + if self._src_arch.add_qkv_bias: + self._inner_map_for_qkv_proj( + f"{src_prefix}self_attention.linear_qkv.bias", + f"{dst_prefix}attn.qkv.bias", + proj_type='qkv_proj', + num_attention_heads=num_attention_heads, + num_query_groups=num_query_groups + ) + + # linear_qkv_norm + self._inner_map_for_full_shape( + f"{src_prefix}self_attention.linear_qkv.layer_norm_weight", + f"{dst_prefix}norm1.weight" + ) + + # module.mlp + self._map_mlp(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.", is_vision_block=True) + + # mlp norm + self._inner_map_for_full_shape( + f"{src_prefix}mlp.linear_fc1.layer_norm_weight", + f"{dst_prefix}norm2.weight" + ) + From 852df210408018c9fae6e5de2de548aa08c60c80 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 14:19:25 +0800 Subject: [PATCH 05/28] clean docstring --- .../mappers/base_megatron_mapper.py | 10 +-- .../mappers/megatron_llm_mapper.py | 58 ++++++++-------- .../mappers/megatron_vlm_mapper.py | 66 +++++++++---------- 3 files changed, 59 insertions(+), 75 deletions(-) diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index bd67930a..3c1be044 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== -"""Mapper for Megatron to vLLM""" +"""Basic Mapper for Megatron to rollout framework""" from collections import defaultdict from typing import List, Dict, TYPE_CHECKING, Union @@ -46,11 +46,7 @@ def __init__( ): """The Base Mapper for Megatron sync. In each remote Megatron Actor, the method of this class is called to generate the parameter mapping - between src and dst. Currently, the mapper supports mapping - MCore Model to vLLM or HF Model. - - WARNING: The mapper assumes that the weights name of same - submodules in different vLLM models are still same. + between src and dst. Args: dst_model_config (PolicyConfig): The config of target model to @@ -83,7 +79,7 @@ def generate_sync_mapping( mapper will ensure that the key of mapping for each mapping type is non-overlapping and can merge into the full state dict of this rank. For most cases, the length of dst shards list is 1, except for GQA with - large vLLM TP. + large TP. """ self._dst_name_to_metadata = dst_name_to_metadata return self._map_model() diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index f40dc3a4..11034d6b 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -52,13 +52,7 @@ def __init__( *, mapper_config: Union[VLLM_HELPERS, HF_HELPERS] = VLLM_HELPERS, ): - """The Mapper for Megatron sync. In each remote Megatron Actor, - the method of this class is called to generate the parameter mapping - between src and dst. Currently, the mapper supports mapping - MCore Model to vLLM or HF Model. - - WARNING: The mapper assumes that the weights name of same - submodules in different vLLM models are still same. + """The Mapper for Megatron LLM sync. Args: dst_model_config (PolicyConfig): The config of target model to @@ -68,6 +62,31 @@ def __init__( """ super().__init__(dst_model_config=dst_model_config, model=model, mapper_config=mapper_config) + # NOTE: the following function implements the module-wise sync mapping + def _map_model(self): + """Mapping the local name of src model to global name of + dst model + """ + for vp_stage, model in enumerate(self.model): + if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: + layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) + else: + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) + layer_offset = get_transformer_layer_offset(model.config) + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(None) + + # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch + if getattr(model, 'mtp_process', None): + raise NotImplementedError("Currently, the mapper does not support MTP") + + self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + + mapping = self._mapping + self._mapping = None + return mapping + def _map_llm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): if model.pre_process: self._map_preprocess_layer( @@ -103,31 +122,6 @@ def _map_llm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): src_prefix=f"{vp_stage}-output_layer.", dst_prefix="", ) - - # NOTE: the following function implements the module-wise sync mapping - def _map_model(self): - """Mapping the local name of src model to global name of - dst model - """ - for vp_stage, model in enumerate(self.model): - if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: - layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) - else: - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) - layer_offset = get_transformer_layer_offset(model.config) - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(None) - - # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch - if getattr(model, 'mtp_process', None): - raise NotImplementedError("Currently, the mapper does not support MTP") - - self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) - - mapping = self._mapping - self._mapping = None - return mapping def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str='', *, is_norm_layer: bool=True): """If is_norm_layer is True, try to map on all possible keys, diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py index e7e1529c..7e8cc7b6 100644 --- a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -43,13 +43,7 @@ def __init__( *, mapper_config: Union[VLLM_HELPERS, HF_HELPERS] = VLLM_HELPERS, ): - """The VLM Mapper for Megatron sync. In each remote Megatron Actor, - the method of this class is called to generate the parameter mapping - between src and dst. Currently, the mapper supports mapping - MCore Model to vLLM or HF Model. - - WARNING: The mapper assumes that the weights name of same - submodules in different vLLM models are still same. + """The Mapper for Megatron VLM sync. Args: dst_model_config (PolicyConfig): The config of target model to @@ -59,6 +53,35 @@ def __init__( """ super().__init__(dst_model_config=dst_model_config, model=model, mapper_config=mapper_config) + # NOTE: the following function implements the module-wise sync mapping + def _map_model(self): + """Mapping the local name of src model to global name of + dst model + """ + for vp_stage, model in enumerate(self.model): + if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: + layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) + else: + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) + layer_offset = get_transformer_layer_offset(model.config) + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(None) + + # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch + if getattr(model, 'mtp_process', None): + raise NotImplementedError("Currently, the mapper does not support MTP") + + if hasattr(model, 'vision_model'): + self._map_vlm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + else: + # llm model + self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + + mapping = self._mapping + self._mapping = None + return mapping + def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): dst_language_prefix = self._mapper_config.dst_language_prefix dst_vision_prefix = self._mapper_config.dst_vision_prefix @@ -145,35 +168,6 @@ def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): dst_prefix=f"{dst_lm_head_prefix}", ) - # NOTE: the following function implements the module-wise sync mapping - def _map_model(self): - """Mapping the local name of src model to global name of - dst model - """ - for vp_stage, model in enumerate(self.model): - if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: - layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) - else: - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) - layer_offset = get_transformer_layer_offset(model.config) - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(None) - - # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch - if getattr(model, 'mtp_process', None): - raise NotImplementedError("Currently, the mapper does not support MTP") - - if hasattr(model, 'vision_model'): - self._map_vlm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) - else: - # llm model - self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) - - mapping = self._mapping - self._mapping = None - return mapping - def _map_vision_layer( self, module: 'TransformerLayer', From 508d5c7f8bcd0690983a91fe64657fbfe41d41a0 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 14:26:24 +0800 Subject: [PATCH 06/28] fix pylint --- chatlearn/synchronizer/mappers/__init__.py | 4 ++-- chatlearn/synchronizer/mappers/base_megatron_mapper.py | 3 ++- chatlearn/synchronizer/mappers/megatron_vlm_mapper.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/chatlearn/synchronizer/mappers/__init__.py b/chatlearn/synchronizer/mappers/__init__.py index 58a469af..ff438970 100644 --- a/chatlearn/synchronizer/mappers/__init__.py +++ b/chatlearn/synchronizer/mappers/__init__.py @@ -23,7 +23,7 @@ def get_mapper_name(src_model: 'DistModel', dst_model: 'DistModel'): src_type = src_model.runtime_args.train_backend dst_type = dst_model.runtime_args.rollout_backend model_type = src_model.runtime_args.model_type # llm or vlm - + mapping = { 'llm-megatron-vllm': "MegatronVLLMMapper-LLM", 'llm-megatron-sglang': "MegatronSGLangMapper-LLM", @@ -34,7 +34,7 @@ def get_mapper_name(src_model: 'DistModel', dst_model: 'DistModel'): if key not in mapping: raise NotImplementedError(f"Unsupported src/dst model combination: {key}") return mapping[key] - + def name_to_mapper_cls(mapper_name: str): # pylint: disable=import-outside-toplevel diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index 3c1be044..6bb7f4c6 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -91,13 +91,14 @@ def dump_sync_mapping(self, folder_path: str, sync_mapping: Dict): folder_path (str): The folder path to dump the sync mapping. sync_mapping (Dict): The sync mapping to be saved. """ + # pylint: disable=abstract-method raise NotImplementedError() def _map_model(self): """Mapping the local name of src model to global name of dst model """ raise NotImplementedError() - + # NOTE: the following function implements the tensor-wise sync mapping def _inner_map_for_tensor_parallel( self, diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py index 7e8cc7b6..bd47e212 100644 --- a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: from chatlearn.models.megatron_module import MegatronModule + from megatron.core.transformer.transformer_layer import TransformerLayer class MegatronVLMMapper(MegatronLLMMapper): """MegatronVLMMapper""" @@ -221,4 +222,3 @@ def _map_vision_layer( f"{src_prefix}mlp.linear_fc1.layer_norm_weight", f"{dst_prefix}norm2.weight" ) - From ff21e815ededc80587aeb80e9ff47c760bdd2be3 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 14:35:45 +0800 Subject: [PATCH 07/28] fix pylint --- chatlearn/synchronizer/mappers/base_megatron_mapper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index 6bb7f4c6..ee092bac 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -86,13 +86,12 @@ def generate_sync_mapping( def dump_sync_mapping(self, folder_path: str, sync_mapping: Dict): """dump the generayed sync mapping to the given folder path in JSON format. + Currently do nothing. Args: folder_path (str): The folder path to dump the sync mapping. sync_mapping (Dict): The sync mapping to be saved. """ - # pylint: disable=abstract-method - raise NotImplementedError() def _map_model(self): """Mapping the local name of src model to global name of dst model From a62294062e83960b69750607a94e1a9789f1deef Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 14:43:08 +0800 Subject: [PATCH 08/28] remove src_arch --- .../mappers/base_megatron_mapper.py | 4 ---- .../mappers/megatron_llm_mapper.py | 24 +++++++++---------- .../mappers/megatron_vlm_mapper.py | 2 +- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index ee092bac..082c10ee 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -192,10 +192,6 @@ def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): self._update_mapping(results) return results - @property - def _src_arch(self): - return self._src_model_config.megatron_model_cfg - def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]) -> None: if self._mapping is None: self._mapping = defaultdict(list) diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index 11034d6b..c46bccda 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -145,7 +145,7 @@ def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str ) def _map_decoder_layer(self, module: 'TransformerLayer', src_prefix: str='', dst_prefix: str=''): - if self._src_arch.multi_latent_attention: + if module.config.multi_latent_attention: map_attn_func = self._map_mla_selfattn norm_layer = module.input_layernorm norm_src_key = f"{src_prefix}input_layernorm." @@ -251,7 +251,7 @@ def _map_group_mlp(self, module: 'TEGroupedMLP', src_prefix: str='', dst_prefix: # pylint: disable=unused-argument src_ep_rank = mpu.get_expert_model_parallel_rank() src_ep_size = mpu.get_expert_model_parallel_world_size() - num_experts = self._src_arch.num_experts + num_experts = module.config.num_moe_experts global_expert_id_start = num_experts // src_ep_size * src_ep_rank global_expert_id_end = num_experts // src_ep_size * (src_ep_rank + 1) for local_expert_id, global_expert_id in enumerate(range(global_expert_id_start, global_expert_id_end)): @@ -288,7 +288,7 @@ def _map_group_mlp(self, module: 'TEGroupedMLP', src_prefix: str='', dst_prefix: ) def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_prefix: str=''): - if self._src_arch.q_lora_rank is None: + if module.config.q_lora_rank is None: self._inner_map_for_tensor_parallel( f"{src_prefix}linear_q_proj.weight", f"{dst_prefix}q_proj.weight", @@ -304,7 +304,7 @@ def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_ f"{dst_prefix}q_b_proj.weight", mapping_type='column' ) - if self._src_arch.qk_layernorm: + if module.config.qk_layernorm: self._map_norm_layer( module.linear_q_up_proj, f"{src_prefix}linear_q_up_proj.", @@ -320,7 +320,7 @@ def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_ f"{dst_prefix}kv_b_proj.weight", mapping_type='column' ) - if self._src_arch.qk_layernorm: + if module.config.qk_layernorm: self._map_norm_layer( module.linear_kv_up_proj, f"{src_prefix}linear_kv_up_proj.", @@ -334,7 +334,7 @@ def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_ ) def _map_selfattn(self, module: 'SelfAttention', src_prefix: str='', dst_prefix: str=''): - if self._src_arch.qk_layernorm: + if module.config.qk_layernorm: self._map_norm_layer(module.q_layernorm, f"{src_prefix}q_layernorm.", f"{dst_prefix}q_norm.") self._map_norm_layer(module.k_layernorm, f"{src_prefix}k_layernorm.", f"{dst_prefix}k_norm.") @@ -347,16 +347,16 @@ def _map_selfattn(self, module: 'SelfAttention', src_prefix: str='', dst_prefix: f"{src_prefix}linear_qkv.weight", f"{dst_prefix}{dst_name}.weight", proj_type=dst_name, - num_attention_heads = self._src_arch.num_attention_heads, - num_query_groups = self._src_arch.num_query_groups + num_attention_heads = module.config.num_attention_heads, + num_query_groups = module.config.num_query_groups ) - if self._src_arch.add_qkv_bias: + if module.config.add_qkv_bias: self._inner_map_for_qkv_proj( f"{src_prefix}linear_qkv.bias", f"{dst_prefix}{dst_name}.bias", proj_type=dst_name, - num_attention_heads = self._src_arch.num_attention_heads, - num_query_groups = self._src_arch.num_query_groups + num_attention_heads = module.config.num_attention_heads, + num_query_groups = module.config.num_query_groups ) self._inner_map_for_tensor_parallel( @@ -377,7 +377,7 @@ def _map_preprocess_layer(self, module: 'LanguageModelEmbedding', src_prefix='', def _map_postprocess_layer(self, module: 'ColumnParallelLinear', src_prefix='', dst_prefix=''): # pylint: disable=unused-argument if ( - not self._src_arch.untie_embeddings_and_output_weights and + not self._src_model_config.megatron_model_cfg.untie_embeddings_and_output_weights and f"{dst_prefix}lm_head.weight" not in self._dst_name_to_metadata ): return diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py index bd47e212..6930acd9 100644 --- a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -199,7 +199,7 @@ def _map_vision_layer( num_attention_heads=num_attention_heads, num_query_groups=num_query_groups ) - if self._src_arch.add_qkv_bias: + if module.config.add_qkv_bias: self._inner_map_for_qkv_proj( f"{src_prefix}self_attention.linear_qkv.bias", f"{dst_prefix}attn.qkv.bias", From 46fe4cf034f4118868b0c6af8873b5e197a88113 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 29 Sep 2025 17:44:59 +0800 Subject: [PATCH 09/28] make some mapping functions be fully configurable --- .../synchronizer/mappers/mapping_helpers.py | 1 + .../mappers/megatron_llm_mapper.py | 214 +++++++++++------- .../mappers/megatron_vlm_mapper.py | 196 ++++++---------- chatlearn/synchronizer/mappers/metadata.py | 52 +++++ 4 files changed, 250 insertions(+), 213 deletions(-) create mode 100644 chatlearn/synchronizer/mappers/metadata.py diff --git a/chatlearn/synchronizer/mappers/mapping_helpers.py b/chatlearn/synchronizer/mappers/mapping_helpers.py index c8244e1c..b1c65377 100644 --- a/chatlearn/synchronizer/mappers/mapping_helpers.py +++ b/chatlearn/synchronizer/mappers/mapping_helpers.py @@ -241,6 +241,7 @@ def __maybe_merge(mappings: List[Tuple[ShardedTensorInfo, ShardedTensorInfo]], a )) return results +# TODO: deprecate these config classes @dataclass(frozen=True) class VLLM_HELPERS: """The mapper configs for vllm""" diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index c46bccda..f1e1ba1f 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -31,7 +31,14 @@ VLLM_HELPERS, HF_HELPERS ) - +from .metadata import ( + SelfAttnKeyMapping, + MLPKeyMapping, + DecoderLayerKeyMapping, + LanguageModelKeyMapping, + MoELayerKeyMapping, + MLASelfAttnKeyMapping +) from .base_megatron_mapper import BaseMegatronMapper if TYPE_CHECKING: @@ -52,7 +59,7 @@ def __init__( *, mapper_config: Union[VLLM_HELPERS, HF_HELPERS] = VLLM_HELPERS, ): - """The Mapper for Megatron LLM sync. + """The Mapper for Megatron LLM sync. Args: dst_model_config (PolicyConfig): The config of target model to @@ -67,6 +74,12 @@ def _map_model(self): """Mapping the local name of src model to global name of dst model """ + cfg = LanguageModelKeyMapping( + decoder_layer_cfg=DecoderLayerKeyMapping( + self_attn_cfg=SelfAttnKeyMapping(use_merged_qkv=self._mapper_config.merge_qkv), + mlp_cfg=MLPKeyMapping(use_merged_gate_up=self._mapper_config.merge_gate_up) + ) + ) for vp_stage, model in enumerate(self.model): if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) @@ -78,49 +91,63 @@ def _map_model(self): mpu.set_virtual_pipeline_model_parallel_rank(None) # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch - if getattr(model, 'mtp_process', None): + if getattr(model, 'mtp_process', False): raise NotImplementedError("Currently, the mapper does not support MTP") - self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) + self._map_llm_model( + model, + cfg, + layer_offset=layer_offset, + src_prefix=f"{vp_stage}-", + dst_prefix="" + ) mapping = self._mapping self._mapping = None return mapping - def _map_llm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): + def _map_llm_model( + self, + model: nn.Module, + cfg: LanguageModelKeyMapping, + layer_offset: int, + src_prefix: str='', + dst_prefix: str='' + ): if model.pre_process: self._map_preprocess_layer( model.embedding, - src_prefix=f"{vp_stage}-embedding.", - dst_prefix="model.", + src_prefix=f"{src_prefix}embedding.", + dst_prefix=f"{dst_prefix}{cfg.word_embeddings}", ) for layer_idx in range(model.decoder.num_layers_per_pipeline_rank): global_layer_id = layer_offset + layer_idx self._map_decoder_layer( model.decoder.layers[layer_idx], - src_prefix=f"{vp_stage}-decoder.layers.{layer_idx}.", - dst_prefix=f"model.layers.{global_layer_id}.", + cfg=cfg.decoder_layer_cfg, + src_prefix=f"{src_prefix}decoder.layers.{layer_idx}.", + dst_prefix=f"{dst_prefix}{cfg.decoder_layer}{global_layer_id}.", ) if model.post_process: self._map_norm_layer( model.decoder.final_layernorm, - src_prefix=f"{vp_stage}-decoder.final_layernorm.", - dst_prefix="model.norm.", + src_prefix=f"{src_prefix}decoder.final_layernorm.", + dst_prefix=f"{dst_prefix}{cfg.final_layernorm}", ) if model.share_embeddings_and_output_weights and model.pre_process: self._map_postprocess_layer( model.embedding, - src_prefix=f"{vp_stage}-embedding.word_embeddings.", - dst_prefix="", + src_prefix=f"{src_prefix}embedding.word_embeddings.", + dst_prefix=f"{dst_prefix}{cfg.output_layer}", ) else: self._map_postprocess_layer( model.output_layer, - src_prefix=f"{vp_stage}-output_layer.", - dst_prefix="", + src_prefix=f"{src_prefix}output_layer.", + dst_prefix=f"{dst_prefix}{cfg.output_layer}", ) def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str='', *, is_norm_layer: bool=True): @@ -144,38 +171,55 @@ def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str f"{dst_prefix}{_keynames[item]}" ) - def _map_decoder_layer(self, module: 'TransformerLayer', src_prefix: str='', dst_prefix: str=''): + def _map_decoder_layer(self, module: 'TransformerLayer', cfg: DecoderLayerKeyMapping, src_prefix: str='', dst_prefix: str=''): if module.config.multi_latent_attention: map_attn_func = self._map_mla_selfattn norm_layer = module.input_layernorm norm_src_key = f"{src_prefix}input_layernorm." - norm_dst_key = f"{dst_prefix}input_layernorm." is_norm_layer = True else: map_attn_func = self._map_selfattn norm_layer = module.self_attention.linear_qkv norm_src_key = f"{src_prefix}self_attention.linear_qkv." - norm_dst_key = f"{dst_prefix}input_layernorm." is_norm_layer = False - map_attn_func(module.self_attention, src_prefix=f"{src_prefix}self_attention.", dst_prefix=f"{dst_prefix}self_attn.") - self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer) + map_attn_func( + module.self_attention, + cfg=cfg.self_attn_cfg, + src_prefix=f"{src_prefix}self_attention.", + dst_prefix=f"{dst_prefix}{cfg.self_attn}", + ) + self._map_norm_layer( + norm_layer, + norm_src_key, + dst_prefix=f"{dst_prefix}{cfg.input_layernorm}", + is_norm_layer=is_norm_layer + ) if isinstance(module.mlp, MoELayer): map_mlp_func = self._map_moe_layer norm_layer = module.pre_mlp_layernorm norm_src_key = f"{src_prefix}pre_mlp_layernorm." - norm_dst_key = f"{dst_prefix}post_attention_layernorm." is_norm_layer = True else: map_mlp_func = self._map_mlp norm_layer = module.mlp.linear_fc1 norm_src_key = f"{src_prefix}mlp.linear_fc1." - norm_dst_key = f"{dst_prefix}post_attention_layernorm." is_norm_layer = False - map_mlp_func(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.") - self._map_norm_layer(norm_layer, norm_src_key, norm_dst_key, is_norm_layer=is_norm_layer) + map_mlp_func( + module.mlp, + cfg=cfg.mlp_cfg, + src_prefix=f"{src_prefix}mlp.", + dst_prefix=f"{dst_prefix}{cfg.mlp}", + ) + self._map_norm_layer( + norm_layer, + norm_src_key, + dst_prefix=f"{dst_prefix}{cfg.pre_mlp_layernorm}", + is_norm_layer=is_norm_layer + ) - def _map_moe_layer(self, module: 'MoELayer', src_prefix='', dst_prefix=''): + def _map_moe_layer(self, module: 'MoELayer', cfg: MoELayerKeyMapping, src_prefix='', dst_prefix=''): + # pylint: disable=unused-argument mapping = {} # router self._inner_map_for_full_shape(f"{src_prefix}router.weight", f"{dst_prefix}gate.weight") @@ -208,43 +252,45 @@ def _map_moe_layer(self, module: 'MoELayer', src_prefix='', dst_prefix=''): shared_expert_key = 'shared_experts' if hasattr(hf_config, 'n_shared_experts') else 'shared_expert' self._map_mlp( module.shared_experts, + cfg=MLPKeyMapping(use_merged_gate_up=self._mapper_config.merge_gate_up), src_prefix=f"{src_prefix}shared_experts.", dst_prefix=f"{dst_prefix}{shared_expert_key}." ) return mapping - def _map_mlp(self, module: 'MLP', src_prefix: str='', dst_prefix: str='', is_vision_block=False): - if not module.config.gated_linear_unit: - raise NotImplementedError("Parameter Sync w/o GatedLinear is not supported") - - dst_names = ['gate_proj', 'up_proj'] - if self._mapper_config.merge_gate_up and not is_vision_block: - dst_names = ['gate_up_proj'] - - for dst_name in dst_names: - self._inner_map_for_gate_up_proj( - f"{src_prefix}linear_fc1.weight", - f"{dst_prefix}{dst_name}.weight", - proj_type=dst_name - ) + def _map_mlp( + self, + module: 'MLP', + cfg: MLPKeyMapping, + src_prefix: str='', + dst_prefix: str='', + ): + param_types = ['weight'] + if module.config.add_bias_linear: + param_types = ['weight', 'bias'] - if module.config.add_bias_linear: - self._inner_map_for_gate_up_proj( - f"{src_prefix}linear_fc1.bias", - f"{dst_prefix}{dst_name}.bias", - proj_type=dst_name + for param_type in param_types: + if not module.config.gated_linear_unit: + self._inner_map_for_tensor_parallel( + f"{src_prefix}linear_fc1.{param_type}", + f"{dst_prefix}{cfg.up_proj}{param_type}", + mapping_type='column' ) + else: + dst_names = {'gate_proj': cfg.gate_proj, 'up_proj': cfg.up_proj} + if cfg.use_merged_gate_up: + dst_names = {'gate_up_proj': cfg.gate_up_proj} - self._inner_map_for_tensor_parallel( - f"{src_prefix}linear_fc2.weight", - f"{dst_prefix}down_proj.weight", - mapping_type='row' - ) - - if module.config.add_bias_linear: - self._inner_map_for_full_shape( - f"{src_prefix}linear_fc2.bias", - f"{dst_prefix}down_proj.bias" + for dst_type, dst_name in dst_names.items(): + self._inner_map_for_gate_up_proj( + f"{src_prefix}linear_fc1.{param_type}", + f"{dst_prefix}{dst_name}{param_type}", + proj_type=dst_type + ) + self._inner_map_for_tensor_parallel( + f"{src_prefix}linear_fc2.{param_type}", + f"{dst_prefix}{cfg.down_proj}{param_type}", + mapping_type='row' ) def _map_group_mlp(self, module: 'TEGroupedMLP', src_prefix: str='', dst_prefix: str=''): @@ -287,7 +333,8 @@ def _map_group_mlp(self, module: 'TEGroupedMLP', src_prefix: str='', dst_prefix: mapping_type='row' ) - def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_prefix: str=''): + def _map_mla_selfattn(self, module: 'MLASelfAttention', cfg: MLASelfAttnKeyMapping, src_prefix: str='', dst_prefix: str=''): + # pylint: disable=unused-argument if module.config.q_lora_rank is None: self._inner_map_for_tensor_parallel( f"{src_prefix}linear_q_proj.weight", @@ -333,37 +380,44 @@ def _map_mla_selfattn(self, module: 'MLASelfAttention', src_prefix: str='', dst_ mapping_type='row' ) - def _map_selfattn(self, module: 'SelfAttention', src_prefix: str='', dst_prefix: str=''): + def _map_selfattn( + self, + module: 'SelfAttention', + cfg: SelfAttnKeyMapping, + src_prefix: str='', + dst_prefix: str='' + ): if module.config.qk_layernorm: - self._map_norm_layer(module.q_layernorm, f"{src_prefix}q_layernorm.", f"{dst_prefix}q_norm.") - self._map_norm_layer(module.k_layernorm, f"{src_prefix}k_layernorm.", f"{dst_prefix}k_norm.") - - dst_names = ['q_proj', 'k_proj', 'v_proj'] - if self._mapper_config.merge_qkv: - dst_names = ['qkv_proj'] - - for dst_name in dst_names: - self._inner_map_for_qkv_proj( - f"{src_prefix}linear_qkv.weight", - f"{dst_prefix}{dst_name}.weight", - proj_type=dst_name, - num_attention_heads = module.config.num_attention_heads, - num_query_groups = module.config.num_query_groups - ) - if module.config.add_qkv_bias: + self._map_norm_layer(module.q_layernorm, f"{src_prefix}q_layernorm.", f"{dst_prefix}{cfg.q_layernorm}") + self._map_norm_layer(module.k_layernorm, f"{src_prefix}k_layernorm.", f"{dst_prefix}{cfg.k_layernorm}") + + qkv_dst_names = {'qkv_proj': cfg.qkv_proj} + if not cfg.use_merged_qkv: + qkv_dst_names = {'q_proj': cfg.q_proj, 'k_proj': cfg.k_proj, 'v_proj': cfg.v_proj} + + param_types = ['weight'] + if module.config.add_qkv_bias: + param_types = ['weight', 'bias'] + + for param_type in param_types: + for dst_type, dst_name in qkv_dst_names.items(): self._inner_map_for_qkv_proj( - f"{src_prefix}linear_qkv.bias", - f"{dst_prefix}{dst_name}.bias", - proj_type=dst_name, + f"{src_prefix}linear_qkv.{param_type}", + f"{dst_prefix}{dst_name}{param_type}", + proj_type=dst_type, num_attention_heads = module.config.num_attention_heads, num_query_groups = module.config.num_query_groups ) - self._inner_map_for_tensor_parallel( - f"{src_prefix}linear_proj.weight", - f"{dst_prefix}o_proj.weight", - mapping_type='row' - ) + param_types = ['weight'] + if module.config.add_bias_linear: + param_types = ['weight', 'bias'] + for param_type in param_types: + self._inner_map_for_tensor_parallel( + f"{src_prefix}linear_proj.{param_type}", + f"{dst_prefix}{cfg.out_proj}{param_type}", + mapping_type='row' + ) def _map_preprocess_layer(self, module: 'LanguageModelEmbedding', src_prefix='', dst_prefix=''): if module.add_position_embedding: diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py index 6930acd9..f2d35f8b 100644 --- a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -28,12 +28,17 @@ VLLM_HELPERS, HF_HELPERS ) +from .metadata import ( + SelfAttnKeyMapping, + MLPKeyMapping, + DecoderLayerKeyMapping, + LanguageModelKeyMapping +) from .megatron_llm_mapper import MegatronLLMMapper if TYPE_CHECKING: from chatlearn.models.megatron_module import MegatronModule - from megatron.core.transformer.transformer_layer import TransformerLayer class MegatronVLMMapper(MegatronLLMMapper): """MegatronVLMMapper""" @@ -59,6 +64,18 @@ def _map_model(self): """Mapping the local name of src model to global name of dst model """ + # TODO: clean this config object + cfg = LanguageModelKeyMapping( + word_embeddings=self._mapper_config.dst_language_prefix, + decoder_layer=f"{self._mapper_config.dst_language_prefix}layers.", + decoder_layer_cfg=DecoderLayerKeyMapping( + self_attn_cfg=SelfAttnKeyMapping(use_merged_qkv=self._mapper_config.merge_qkv), + mlp_cfg=MLPKeyMapping(use_merged_gate_up=self._mapper_config.merge_gate_up) + ), + final_layernorm=f"{self._mapper_config.dst_language_prefix}norm.", + output_layer=self._mapper_config.dst_lm_head_prefix + ) + for vp_stage, model in enumerate(self.model): if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) @@ -74,151 +91,64 @@ def _map_model(self): raise NotImplementedError("Currently, the mapper does not support MTP") if hasattr(model, 'vision_model'): - self._map_vlm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) - else: - # llm model - self._map_llm_model(model, vp_stage=vp_stage, layer_offset=layer_offset) - - mapping = self._mapping - self._mapping = None - return mapping - - def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): - dst_language_prefix = self._mapper_config.dst_language_prefix - dst_vision_prefix = self._mapper_config.dst_vision_prefix - dst_lm_head_prefix = self._mapper_config.dst_lm_head_prefix - - if model.pre_process: - self._map_preprocess_layer( - model.language_model.embedding, - src_prefix=f"{vp_stage}-language_model.embedding.", - dst_prefix=f"{dst_language_prefix}", - ) - - self._inner_map_for_full_shape( - f"{vp_stage}-vision_model.patch_embed.proj.weight", - f"{dst_vision_prefix}patch_embed.proj.weight" - ) - - # vision model decoder - for layer_idx in range(model.vision_config.num_layers): - global_layer_id = layer_offset + layer_idx - self._map_vision_layer( - model.vision_model.decoder.layers[layer_idx], - src_prefix=f"{vp_stage}-vision_model.decoder.layers.{layer_idx}.", - dst_prefix=f"{dst_vision_prefix}blocks.{global_layer_id}.", - num_attention_heads=model.vision_config.num_attention_heads, - num_query_groups=model.vision_config.num_query_groups + # assert layer_offset == 0 + self._map_vision_model( + model.vision_model, + src_prefix=f"{vp_stage}-vision_model.", + dst_prefix=self._mapper_config.dst_vision_prefix ) - # vision model projection - self._inner_map_for_full_shape( - f"{vp_stage}-vision_model.decoder.final_layernorm.weight", - f"{dst_vision_prefix}merger.ln_q.weight" - ) - - self._inner_map_for_tensor_parallel( - f"{vp_stage}-vision_model.projection.encoder.linear_fc1.weight", - f"{dst_vision_prefix}merger.mlp.0.weight", - mapping_type='column' - ) - - self._inner_map_for_tensor_parallel( - f"{vp_stage}-vision_model.projection.encoder.linear_fc1.bias", - f"{dst_vision_prefix}merger.mlp.0.bias", - mapping_type='column' - ) - - self._inner_map_for_tensor_parallel( - f"{vp_stage}-vision_model.projection.encoder.linear_fc2.weight", - f"{dst_vision_prefix}merger.mlp.2.weight", - mapping_type='row' + # llm model + self._map_llm_model( + model.language_model, + cfg=cfg, + layer_offset=layer_offset, + src_prefix=f"{vp_stage}-language_model.", + dst_prefix="" ) - # bias for row is not slice, so we need to map it to full shape - self._inner_map_for_full_shape( - f"{vp_stage}-vision_model.projection.encoder.linear_fc2.bias", - f"{dst_vision_prefix}merger.mlp.2.bias" - ) - - for layer_idx in range(model.language_model.decoder.num_layers_per_pipeline_rank): - global_layer_id = layer_offset + layer_idx - self._map_decoder_layer( - model.language_model.decoder.layers[layer_idx], - src_prefix=f"{vp_stage}-language_model.decoder.layers.{layer_idx}.", - dst_prefix=f"{dst_language_prefix}layers.{global_layer_id}.", - ) - - if model.post_process: - self._map_norm_layer( - model.language_model.decoder.final_layernorm, - src_prefix=f"{vp_stage}-language_model.decoder.final_layernorm.", - dst_prefix=f"{dst_language_prefix}norm.", - ) - - if model.share_embeddings_and_output_weights and model.pre_process: - self._map_postprocess_layer( - model.language_model.embedding, - src_prefix=f"{vp_stage}-language_model.embedding.word_embeddings.", - dst_prefix=f"{dst_lm_head_prefix}", - ) - else: - self._map_postprocess_layer( - model.language_model.output_layer, - src_prefix=f"{vp_stage}-language_model.output_layer.", - dst_prefix=f"{dst_lm_head_prefix}", - ) + mapping = self._mapping + self._mapping = None + return mapping - def _map_vision_layer( - self, - module: 'TransformerLayer', + def _map_vision_model(self, + model: nn.Module, src_prefix: str = '', - dst_prefix: str = '', - num_attention_heads: int = None, - num_query_groups: int = None + dst_prefix: str = '' ): - # module.self_attention - # linear_proj - self._inner_map_for_tensor_parallel( - f"{src_prefix}self_attention.linear_proj.weight", - f"{dst_prefix}attn.proj.weight", - mapping_type='row' - ) - - # bias for row is not slice, so we need to map it to full shape self._inner_map_for_full_shape( - f"{src_prefix}self_attention.linear_proj.bias", - f"{dst_prefix}attn.proj.bias" + f"{src_prefix}patch_embed.proj.weight", + f"{dst_prefix}patch_embed.proj.weight" ) - # linear_qkv - self._inner_map_for_qkv_proj( - f"{src_prefix}self_attention.linear_qkv.weight", - f"{dst_prefix}attn.qkv.weight", - proj_type='qkv_proj', - num_attention_heads=num_attention_heads, - num_query_groups=num_query_groups + # vision model decoder + decoder_layer_cfg = DecoderLayerKeyMapping( + input_layernorm='norm1.', + self_attn='attn.', + self_attn_cfg=SelfAttnKeyMapping( + qkv_proj='qkv.', + out_proj='proj.', + use_merged_qkv=True + ), + pre_mlp_layernorm='norm2.' ) - if module.config.add_qkv_bias: - self._inner_map_for_qkv_proj( - f"{src_prefix}self_attention.linear_qkv.bias", - f"{dst_prefix}attn.qkv.bias", - proj_type='qkv_proj', - num_attention_heads=num_attention_heads, - num_query_groups=num_query_groups + for layer_idx in range(model.config.num_layers): + self._map_decoder_layer( + model.decoder.layers[layer_idx], + decoder_layer_cfg, + src_prefix=f"{src_prefix}decoder.layers.{layer_idx}.", + dst_prefix=f"{dst_prefix}blocks.{layer_idx}.", ) - # linear_qkv_norm + # vision model projection self._inner_map_for_full_shape( - f"{src_prefix}self_attention.linear_qkv.layer_norm_weight", - f"{dst_prefix}norm1.weight" + f"{src_prefix}decoder.final_layernorm.weight", + f"{dst_prefix}merger.ln_q.weight" ) - - # module.mlp - self._map_mlp(module.mlp, src_prefix=f"{src_prefix}mlp.", dst_prefix=f"{dst_prefix}mlp.", is_vision_block=True) - - # mlp norm - self._inner_map_for_full_shape( - f"{src_prefix}mlp.linear_fc1.layer_norm_weight", - f"{dst_prefix}norm2.weight" + mlp_cfg = MLPKeyMapping(up_proj='0.', down_proj='2.') + self._map_mlp( + model.projection.encoder, + mlp_cfg, + src_prefix=f"{src_prefix}projection.encoder.", + dst_prefix=f"{dst_prefix}merger.mlp." ) diff --git a/chatlearn/synchronizer/mappers/metadata.py b/chatlearn/synchronizer/mappers/metadata.py new file mode 100644 index 00000000..a205b8dd --- /dev/null +++ b/chatlearn/synchronizer/mappers/metadata.py @@ -0,0 +1,52 @@ +# pylint: disable=missing-module-docstring, missing-class-docstring +from typing import Union +from dataclasses import dataclass, field + + +@dataclass +class SelfAttnKeyMapping: + q_layernorm: str = 'q_norm.' + k_layernorm: str = 'k_norm.' + qkv_proj: str = 'qkv_proj.' + q_proj: str = 'q_proj.' + k_proj: str = 'k_proj.' + v_proj: str = 'v_proj.' + out_proj: str = 'o_proj.' + use_merged_qkv: bool = False + +@dataclass +class MLASelfAttnKeyMapping: + # NOTE: currently not used + pass + +@dataclass +class MLPKeyMapping: + gate_proj: str = 'gate_proj.' + up_proj: str = 'up_proj.' + down_proj: str = 'down_proj.' + gate_up_proj: str = 'gate_up_proj.' + use_merged_gate_up: bool = False + +@dataclass +class MoELayerKeyMapping: + # NOTE: currently not used + pass + + +@dataclass +class DecoderLayerKeyMapping: + input_layernorm: str = 'input_layernorm.' # if is_vision_block, norm1. + self_attn: str = 'self_attn.' + self_attn_cfg: Union[SelfAttnKeyMapping, MLASelfAttnKeyMapping] = field(default=SelfAttnKeyMapping) + pre_mlp_layernorm: str = 'post_attention_layernorm.' # if is_vision_block, norm2. + mlp: str = 'mlp.' + mlp_cfg: Union[MLPKeyMapping, MoELayerKeyMapping] = field(default=MLPKeyMapping) + + +@dataclass +class LanguageModelKeyMapping: + word_embeddings: str = 'model.' + decoder_layer: str = 'model.layers.' + decoder_layer_cfg: DecoderLayerKeyMapping = field(default=DecoderLayerKeyMapping) + final_layernorm: str = 'model.norm.' + output_layer: str = '' From e6f7aa2bb699659d2a59ea20a89ecb7bc3a254c0 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Tue, 30 Sep 2025 14:43:54 +0800 Subject: [PATCH 10/28] init commit --- .../grpo_utils/megatron_policy_trainer.py | 269 ++---------------- .../grpo_utils/megatron_utils/__init__.py | 6 - .../grpo_utils/megatron_utils/policy_model.py | 187 ------------ .../megatron_utils/policy_model_vl.py | 184 ------------ .../grpo_utils/megatron_utils/train_helper.py | 105 ++++++- chatlearn/configs/megatron_config.py | 5 + chatlearn/synchronizer/mappers/mapper.py | 6 +- 7 files changed, 134 insertions(+), 628 deletions(-) delete mode 100644 chatlearn/algorithm/grpo_utils/megatron_utils/policy_model.py delete mode 100644 chatlearn/algorithm/grpo_utils/megatron_utils/policy_model_vl.py diff --git a/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py b/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py index 2599c346..a445add9 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py +++ b/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py @@ -13,78 +13,43 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -import inspect import os -from contextlib import nullcontext from functools import partial import itertools -from typing import List, Dict, Any, Sequence, Optional +from typing import List, Dict, Any from collections import defaultdict -import numpy as np -from copy import deepcopy +from importlib import import_module import torch from megatron.core import mpu from megatron.core.distributed import finalize_model_grads from megatron.core.enums import ModelType -from megatron.core.models.gpt.gpt_layer_specs import ( - get_gpt_decoder_block_spec, - get_gpt_layer_local_spec, - get_gpt_mtp_block_spec -) -from megatron.core.num_microbatches_calculator import get_num_microbatches from megatron.core.pipeline_parallel import get_forward_backward_func from megatron.core.transformer.spec_utils import import_module from megatron.core.utils import get_model_config -from megatron.training import get_args, get_model, get_timers, print_rank_0, get_tokenizer -from megatron.training.arguments import core_transformer_config_from_args +from megatron.training import get_args, get_model, get_timers from megatron.training.checkpointing import load_checkpoint from megatron.training.training import setup_model_and_optimizer from megatron.training.utils import ( calc_params_l2_norm, logical_and_across_model_parallel_group ) -from megatron.training.yaml_arguments import core_transformer_config_from_yaml -import warnings -try: - from megatron_patch.model.qwen2_5_vl.transformer_config import ( - Qwen2VLTransformerConfig, - get_vision_model_config, - get_vision_projection_config - ) - - from megatron_patch.model.qwen2_5_vl.layer_specs import ( - get_qwen2vl_vision_model_spec, - get_mlp_module_spec - - ) - from chatlearn.algorithm.grpo_utils.megatron_utils import Qwen2_5VLPolicyModel - HAVE_MEGATRON_PATCH = True -except ImportError: - from unittest.mock import MagicMock - Qwen2_5VLPolicyModel = MagicMock() - HAVE_MEGATRON_PATCH = False - -import chatlearn + from chatlearn import MegatronModule from chatlearn.utils.utils import even_slice from chatlearn.runtime.decorator import timeit, compute_decorator, monitor_error from chatlearn.algorithm.grpo_utils.megatron_utils import ( - GPTPolicyModel, forward_step, training_log ) - from chatlearn.algorithm.grpo_utils.trainer_utils import ( - logprobs_from_logits, - entropy_from_logits_with_chunking, - sp_split, generate_loss_mask_position_ids, split_microbatch, batching, split_and_unpadding ) + class MegatronPolicyTrainer(MegatronModule): """MegatronPolicyTrainer""" @monitor_error() @@ -100,6 +65,15 @@ def setup(self): self._metric_prefix = "policy_trainer" + try: + module = import_module(self.module_args.model_provider_module) + except ImportError as e: + raise ImportError(f"Failed to import {self.module_args.model_provider_module} in current PYTHONPATH, get error: {e.msg}") + + # module should has attr model_provider + if not hasattr(module, 'model_provider'): + raise ValueError(f"Cannot find model_provider() in the given module {self.module_args.model_provider_module}, location: {module.__file__}") + if self.trainable: # TODO: move this hardcoded resumedir elsewhere resume_dir = f"{self.runtime_args.output_dir}/save_model/{self.name}" @@ -109,29 +83,19 @@ def setup(self): get_args().no_load_rng = False get_args().no_load_scheduler = False self._logger.info(f"Overwrite load path for resuming training.") - if self.runtime_args.model_type == 'vlm': - assert HAVE_MEGATRON_PATCH, "megatron_patch is nessary for vl. Please set env var MEGATRON_PATCH_PATH to include megatron_patch" - self.model, self.optimizer, self.opt_param_scheduler = ( - setup_model_and_optimizer( - self.model_provider_vl, ModelType.encoder_or_decoder - ) - ) - else: - self.model, self.optimizer, self.opt_param_scheduler = ( - setup_model_and_optimizer( - self.model_provider, ModelType.encoder_or_decoder - ) + + self.model, self.optimizer, self.opt_param_scheduler = ( + setup_model_and_optimizer( + # TODO: currently we only support ModelType.encoder_or_decoder + module.model_provider, ModelType.encoder_or_decoder ) + ) self.config = get_model_config(self.model[0]) self.config.grad_scale_func = self.optimizer.scale_loss self.config.finalize_model_grads_func = finalize_model_grads else: - if self.runtime_args.model_type == 'vlm': - assert HAVE_MEGATRON_PATCH, "megatron_patch is nessary for vl. Please set env var MEGATRON_PATCH_PATH to include megatron_patch" - self.model = get_model(self.model_provider_vl, wrap_with_ddp=False) - else: - self.model = get_model(self.model_provider, wrap_with_ddp=False) + self.model = get_model(module.model_provider, wrap_with_ddp=False) if self.args.load is not None: print(f"reference loading : {self.args.load}") _, _ = load_checkpoint( @@ -146,194 +110,6 @@ def setup(self): device_ids=[int(os.environ.get("LOCAL_RANK", 0))] ) - def model_provider(self, pre_process=True, post_process=True) -> GPTPolicyModel: - from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec - - args = get_args() - use_te = args.transformer_impl == "transformer_engine" - - if args.record_memory_history: - torch.cuda.memory._record_memory_history( - True, - # keep 100,000 alloc/free events from before the snapshot - trace_alloc_max_entries=100000, - # record stack information for the trace events - trace_alloc_record_context=True, - ) - - def oom_observer(device, alloc, device_alloc, device_free): - # snapshot right after an OOM happened - print("saving allocated state during OOM") - snapshot = torch.cuda.memory._snapshot() - from pickle import dump - - dump( - snapshot, - open( - f"oom_rank-{torch.distributed.get_rank()}_{args.memory_snapshot_path}", - "wb", - ), - ) - - torch._C._cuda_attach_out_of_memory_observer(oom_observer) - - print_rank_0("building GPT model ...") - # Experimental loading arguments from yaml - - if args.yaml_cfg is not None: - config = core_transformer_config_from_yaml(args, "language_model") - else: - config = core_transformer_config_from_args(args) - - if args.spec is not None: - transformer_layer_spec = import_module(args.spec) - else: - if args.num_experts: - # Define the decoder block spec - transformer_layer_spec = get_gpt_decoder_block_spec( - config, - use_transformer_engine=use_te, - normalization=args.normalization, - ) - else: - # Define the decoder layer spec - if use_te: - transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec( - args.num_experts, - args.moe_grouped_gemm, - args.qk_layernorm, - args.multi_latent_attention, - args.moe_use_legacy_grouped_gemm, - ) - else: - transformer_layer_spec = get_gpt_layer_local_spec( - args.num_experts, - args.moe_grouped_gemm, - args.qk_layernorm, - args.multi_latent_attention, - args.moe_use_legacy_grouped_gemm, - normalization=args.normalization, - ) - mtp_block_spec = None - if args.mtp_num_layers is not None: - mtp_block_spec = get_gpt_mtp_block_spec( - config, transformer_layer_spec, use_transformer_engine=use_te - ) - - build_model_context = nullcontext - build_model_context_args = {} - if args.fp8_param_gather: - try: - from transformer_engine.pytorch import fp8_model_init - - build_model_context = fp8_model_init - build_model_context_args["enabled"] = True - - # Check if fp8_model_init supports preserve_high_precision_init_val - if ( - "preserve_high_precision_init_val" - in inspect.signature(fp8_model_init).parameters - ): - build_model_context_args["preserve_high_precision_init_val"] = True - except: - raise RuntimeError( - "--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found." - ) - - with build_model_context(**build_model_context_args): - model = GPTPolicyModel( - config=config, - transformer_layer_spec=transformer_layer_spec, - vocab_size=args.padded_vocab_size, - max_sequence_length=args.max_position_embeddings, - pre_process=pre_process, - post_process=post_process, - fp16_lm_cross_entropy=args.fp16_lm_cross_entropy, - parallel_output=True, - share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights, - position_embedding_type=args.position_embedding_type, - rotary_percent=args.rotary_percent, - rotary_base=args.rotary_base, - rope_scaling=args.use_rope_scaling, - mtp_block_spec=mtp_block_spec, - module_args=self.module_args - ) - - return model - - def model_provider_vl( - self, pre_process=True, post_process=True, add_encoder=True, add_decoder=True, vp_stage: Optional[int] = None - ) -> Qwen2_5VLPolicyModel: - from megatron_patch.model.qwen2_5_vl.layer_specs import get_gpt_layer_with_transformer_engine_spec - - args = get_args() - - print_rank_0("start building qwen2-vl model ...") - - # Config of vit, llm and projector - config = core_transformer_config_from_args(args, Qwen2VLTransformerConfig) - use_te = args.transformer_impl == "transformer_engine" - if not use_te: - raise NotImplementedError("The Qwen2-VL model is only implemented with TransformerEngine!") - - if args.rotary_seq_len_interpolation_factor is not None or args.rotary_seq_len_interpolation_factor != 1: - print_rank_0('Multimodal RoPE currently not support RoPE interpolation, set to None...') - args.rotary_seq_len_interpolation_factor = None - - vision_config = get_vision_model_config(args, deepcopy(config)) - vision_config.pipeline_model_parallel_size = 1 - vision_config.num_layers_in_first_pipeline_stage = None - vision_projector_config = get_vision_projection_config(deepcopy(config), vision_config.hidden_size, vision_config.spatial_merge_size) - - print_rank_0("building Qwen2-5-VL model in TE...") - # Layer Specs of vit, llm and projector - transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec(args.qk_layernorm) - vision_model_spec = get_qwen2vl_vision_model_spec() - vision_projector_spec = get_mlp_module_spec(add_norm=False).submodules - - model = Qwen2_5VLPolicyModel( - language_transformer_config=config, - language_transformer_layer_spec=transformer_layer_spec, - language_vocab_size=args.padded_vocab_size, - language_max_sequence_length=args.max_position_embeddings, - - vision_transformer_config=vision_config, - vision_transformer_layer_spec=vision_model_spec, - drop_vision_class_token=False, # NOTE: no class token to drop? - - vision_projection_config=vision_projector_config, - vision_projection_layer_spec=vision_projector_spec, - vision_projection_type='mlp', - allow_missing_vision_projection_checkpoint= False, # TODO: may parameterized - - language_position_embedding_type=args.position_embedding_type, - language_rotary_percent=args.rotary_percent, - language_rotary_base=args.rotary_base, - - pre_process=pre_process, - post_process=post_process, - add_decoder=add_decoder, - add_encoder=add_encoder, - - fp16_lm_cross_entropy=args.fp16_lm_cross_entropy, - parallel_output=True, - language_share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights, - vp_stage=vp_stage, - - module_args=self.module_args - ) - - # TODO: support model freeze for vl model - assert not (self.module_args.megatron_model_cfg.freeze_LM or self.module_args.megatron_model_cfg.freeze_ViT or self.module_args.megatron_model_cfg.freeze_VP), \ - "VL models do not support model freeze currently. Please set freeze_LM, freeze_ViT, and freeze_VP to False." - model.freeze( - freeze_language_model=self.module_args.megatron_model_cfg.freeze_LM, - freeze_vision_model=self.module_args.megatron_model_cfg.freeze_ViT, - freeze_vision_projection=self.module_args.megatron_model_cfg.freeze_VP, - ) - - return model - @monitor_error() @compute_decorator(trainable=True, rollout=False) @timeit() @@ -373,7 +149,8 @@ def train_step(self, data_list: List[Dict[str, Any]], **kwargs): forward_step_func=partial( forward_step, is_training=True, - is_packing=self.module_args.packing + is_packing=self.module_args.packing, + module_args=self.module_args ), data_iterator=data_iterator, model=self.model, diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/__init__.py b/chatlearn/algorithm/grpo_utils/megatron_utils/__init__.py index 78311647..c5283252 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/__init__.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/__init__.py @@ -1,8 +1,2 @@ """utils function megatron policy trainer""" -from .policy_model import GPTPolicyModel -try: - from .policy_model_vl import Qwen2_5VLPolicyModel -except ImportError: - import warnings - warnings.warn("megatron_patch is not installed.") from .train_helper import forward_step, training_log diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/policy_model.py b/chatlearn/algorithm/grpo_utils/megatron_utils/policy_model.py deleted file mode 100644 index f3e09b49..00000000 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/policy_model.py +++ /dev/null @@ -1,187 +0,0 @@ -# pylint: skip-file -# Copyright 2024 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from typing import Literal, Optional, Dict, Any, Union - -import torch - -from flash_attn.bert_padding import pad_input - -from megatron.core import mpu -from megatron.core.inference.contexts import BaseInferenceContext -from megatron.core.models.gpt import GPTModel -from megatron.core.packed_seq_params import PackedSeqParams -from megatron.core.transformer.spec_utils import ModuleSpec -from megatron.core.transformer.transformer_config import TransformerConfig - -from megatron.training import get_args -from torch import Tensor - -from chatlearn.configs.base import BaseModelConfig - -from ..loss_gallery import calculate_grpo_loss, calculate_gspo_loss -from .train_helper import entropy_from_tensor_parallel_logits, reduce_from_context_parallel_region - - -# TODO: replace this class with GPTModel -class GPTPolicyModel(GPTModel): - """PolicyModel""" - - def __init__(self, *args, module_args: Optional[BaseModelConfig] = None, **kwargs): - """Create a Megatron-Core Policy Model. For more descriptions, please - refer to `megatron.core.models.gpt.GPTModel` - - Args: - module_args (Optional[BaseModelConfig], optional): Arguments for chatlearn modules. - Defaults to None. - """ - super().__init__(*args, **kwargs) - self.module_args = module_args - - def forward( - self, - input_ids: Tensor, - position_ids: Tensor, - attention_mask: Tensor, - decoder_input: Tensor = None, - labels: Tensor = None, - inference_context: BaseInferenceContext = None, - packed_seq_params: PackedSeqParams = None, - extra_block_kwargs: dict = None, - runtime_gather_output: Optional[bool] = None, - *, - inference_params: Optional[BaseInferenceContext] = None, - loss_mask: Optional[Tensor] = None, - training_inputs: dict = None, - ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: - # untransposed hidden_states or transposed logits with shape [b, s, h] - hidden_states_or_logits = super().forward( - input_ids=input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - decoder_input=decoder_input, - labels=None, - loss_mask=loss_mask, - inference_context=inference_context, - packed_seq_params=packed_seq_params, - extra_block_kwargs=extra_block_kwargs, - runtime_gather_output=runtime_gather_output, - inference_params=inference_params, - ) - - if not self.post_process: - return hidden_states_or_logits - - if training_inputs is None: - return ( - self.compute_language_model_loss( - labels, - hidden_states_or_logits.transpose( - 0, 1 - ).contiguous(), # [b s h] => [s b h] - ) - if labels is not None - else hidden_states_or_logits - ) - - return self._compute_all_losses( - all_token_logits=hidden_states_or_logits.transpose(0, 1).contiguous(), - labels=labels, - training_inputs=training_inputs - ) - - def _compute_all_losses( - self, - all_token_logits: torch.Tensor, - labels: torch.Tensor, - training_inputs: Dict[str, Any], - ) -> Dict[str, torch.Tensor]: - """Compute all required losses. - - Args: - all_token_logits (torch.Tensor): logits of input tokens. shape: [s, b, h] or [total_nnz, 1, h] - labels (torch.Tensor): labels of input tokens. shape: [s, b] or [total_nnz, 1] - training_inputs (Dict[str, Any]): All training inputs. - - """ - forward_logprob = ( - self.compute_language_model_loss(labels, all_token_logits) * -1 - ) - - forward_logprob = reduce_from_context_parallel_region(forward_logprob, self.module_args.packing, training_inputs) - - old_logprobs = training_inputs["old_logprobs"] - ref_logprobs = training_inputs["ref_logprobs"] - advantages = training_inputs["advantages"] - - - if self.module_args.use_group_sequence_policy: - ( - pg_loss, - is_positive_clipped, - is_negative_clipped, - is_clipped, - ) = calculate_gspo_loss( - log_probs=forward_logprob, - old_log_probs=old_logprobs, - advantages=advantages, - diff_clip_ratio=self.module_args.diff_clip_ratio, - pos_clip_ratio=self.module_args.pos_clip_ratio, - neg_clip_ratio=self.module_args.neg_clip_ratio, - final_clip_ratio=self.module_args.final_clip_ratio, - loss_mask = training_inputs['all_token_loss_mask'] - ) - else: - pg_loss = calculate_grpo_loss( - log_probs=forward_logprob, - old_log_probs=old_logprobs, - advantages=advantages, - diff_clip_ratio=self.module_args.diff_clip_ratio, - pos_clip_ratio=self.module_args.pos_clip_ratio, - neg_clip_ratio=self.module_args.neg_clip_ratio, - final_clip_ratio=self.module_args.final_clip_ratio - ) - - entropy_loss = entropy_from_tensor_parallel_logits(all_token_logits).permute(1, 0) - - entropy_loss = reduce_from_context_parallel_region(entropy_loss, self.module_args.packing, training_inputs) - - kl = ref_logprobs - forward_logprob - ratio = torch.exp(kl) - ratio[~training_inputs['all_token_loss_mask'].bool()] = 1 - assert not torch.isinf(ratio).any(), "kl loss ratio has inf values" - assert not torch.isnan(ratio).any(), "kl loss ratio has nan values" - kld = (ratio - kl - 1).contiguous() - kl_loss = torch.clamp(kld, min=-10, max=10) - - if self.module_args.use_group_sequence_policy: - return { - 'pg_loss': pg_loss, - 'entropy_loss': entropy_loss, - 'kl_loss': kl_loss, - 'is_positive_clipped': is_positive_clipped, - 'is_negative_clipped': is_negative_clipped, - 'is_clipped': is_clipped, - 'is_positive_clipped_sample_average': is_positive_clipped, - 'is_negative_clipped_sample_average': is_negative_clipped, - 'is_clipped_sample_average': is_clipped, - } - else: - return { - 'pg_loss': pg_loss, - 'entropy_loss': entropy_loss, - 'kl_loss': kl_loss - } \ No newline at end of file diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/policy_model_vl.py b/chatlearn/algorithm/grpo_utils/megatron_utils/policy_model_vl.py deleted file mode 100644 index f0612c1d..00000000 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/policy_model_vl.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -# Copyright 2024 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from typing import Optional, Dict, Any, Union - -import torch - -from megatron.core.inference.contexts import BaseInferenceContext -from megatron.core.packed_seq_params import PackedSeqParams - -from torch import Tensor -from chatlearn.configs.base import BaseModelConfig - -from ..loss_gallery import calculate_grpo_loss, calculate_gspo_loss -from .train_helper import entropy_from_tensor_parallel_logits, reduce_from_context_parallel_region - -try: - from megatron_patch.model.qwen2_5_vl.model import Qwen2_5VLModel -except ImportError: - from unittest.mock import MagicMock - Qwen2_5VLModel = MagicMock() - -class Qwen2_5VLPolicyModel(Qwen2_5VLModel): - """PolicyModel""" - - def __init__(self, *args, module_args: Optional[BaseModelConfig] = None, **kwargs): - """Create a Megatron-Core Policy Model. For more descriptions, please - refer to `megatron_patch.model.qwen2_5_vl.model import Qwen2_5VLModel` - - Args: - module_args (Optional[BaseModelConfig], optional): Arguments for chatlearn modules. - Defaults to None. - """ - super().__init__(*args, **kwargs) - self.module_args = module_args - self.vision_config = kwargs['vision_transformer_config'] - - def forward( - self, - input_ids: Tensor, - position_ids: Tensor, - labels: Tensor = None, - pixel_values: Tensor = None, - image_grid_thw: Tensor = None, - image_input_mask: Tensor = None, - packed_seq_params: PackedSeqParams = None, - extra_block_kwargs: dict = None, - *, - inference_params: Optional[BaseInferenceContext] = None, - training_inputs: dict = None, - ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: - # untransposed hidden_states or transposed logits with shape [b, s, h] - hidden_states_or_logits = super().forward( - input_ids=input_ids, - position_ids=position_ids, - vision_data=pixel_values, - vision_grid_thw=image_grid_thw, - video_start_index=image_input_mask.sum().cpu().item(), - image_input_mask=image_input_mask, - video_input_mask= None, - attention_mask=None, - labels=None, - inference_params=inference_params, - packed_seq_params=packed_seq_params, - extra_block_kwargs=extra_block_kwargs, - ) - - if not self.post_process: - return hidden_states_or_logits - - if training_inputs is None: - return ( - self.language_model.compute_language_model_loss( - labels, - hidden_states_or_logits.transpose( - 0, 1 - ).contiguous(), # [b s h] => [s b h] - ) - if labels is not None - else hidden_states_or_logits - ) - - return self._compute_all_losses( - all_token_logits=hidden_states_or_logits.transpose(0, 1).contiguous(), - labels=labels, - training_inputs=training_inputs - ) - - def _compute_all_losses( - self, - all_token_logits: torch.Tensor, - labels: torch.Tensor, - training_inputs: Dict[str, Any], - ) -> Dict[str, torch.Tensor]: - """Compute all required losses. - - Args: - all_token_logits (torch.Tensor): logits of input tokens. shape: [s, b, h] or [total_nnz, 1, h] - labels (torch.Tensor): labels of input tokens. shape: [s, b] or [total_nnz, 1] - training_inputs (Dict[str, Any]): All training inputs. - - """ - forward_logprob = ( - self.language_model.compute_language_model_loss(labels, all_token_logits) * -1 - ) - - forward_logprob = reduce_from_context_parallel_region(forward_logprob, self.module_args.packing, training_inputs) - - old_logprobs = training_inputs["old_logprobs"] - ref_logprobs = training_inputs["ref_logprobs"] - advantages = training_inputs["advantages"] - - - if self.module_args.use_group_sequence_policy: - ( - pg_loss, - is_positive_clipped, - is_negative_clipped, - is_clipped, - ) = calculate_gspo_loss( - log_probs=forward_logprob, - old_log_probs=old_logprobs, - advantages=advantages, - diff_clip_ratio=self.module_args.diff_clip_ratio, - pos_clip_ratio=self.module_args.pos_clip_ratio, - neg_clip_ratio=self.module_args.neg_clip_ratio, - final_clip_ratio=self.module_args.final_clip_ratio, - loss_mask = training_inputs['all_token_loss_mask'] - ) - else: - pg_loss = calculate_grpo_loss( - log_probs=forward_logprob, - old_log_probs=old_logprobs, - advantages=advantages, - diff_clip_ratio=self.module_args.diff_clip_ratio, - pos_clip_ratio=self.module_args.pos_clip_ratio, - neg_clip_ratio=self.module_args.neg_clip_ratio, - final_clip_ratio=self.module_args.final_clip_ratio - ) - - entropy_loss = entropy_from_tensor_parallel_logits(all_token_logits).permute(1, 0) - - entropy_loss = reduce_from_context_parallel_region(entropy_loss, self.module_args.packing, training_inputs) - - kl = ref_logprobs - forward_logprob - ratio = torch.exp(kl) - - ratio[~training_inputs['all_token_loss_mask'].bool()] = 1 - assert not torch.isinf(ratio).any(), "kl loss ratio has inf values" - assert not torch.isnan(ratio).any(), "kl loss ratio has nan values" - kld = (ratio - kl - 1).contiguous() - kl_loss = torch.clamp(kld, min=-10, max=10) - - if self.module_args.use_group_sequence_policy: - return { - 'pg_loss': pg_loss, - 'entropy_loss': entropy_loss, - 'kl_loss': kl_loss, - 'is_positive_clipped': is_positive_clipped, - 'is_negative_clipped': is_negative_clipped, - 'is_clipped': is_clipped, - 'is_positive_clipped_sample_average': is_positive_clipped, - 'is_negative_clipped_sample_average': is_negative_clipped, - 'is_clipped_sample_average': is_clipped, - } - else: - return { - 'pg_loss': pg_loss, - 'entropy_loss': entropy_loss, - 'kl_loss': kl_loss - } diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index f230b7ed..d80d4c7f 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -43,6 +43,7 @@ from chatlearn.utils import to_device +from ..loss_gallery import calculate_grpo_loss, calculate_gspo_loss # TODO: simplify this function def training_log( @@ -462,8 +463,92 @@ def loss_func( return total_loss_for_bp, num_tokens, reporting_losses +def _compute_all_losses( + module_args, + model, + all_token_logits: torch.Tensor, + labels: torch.Tensor, + training_inputs: Dict[str, Any], + ) -> Dict[str, torch.Tensor]: + """Compute all required losses. + + Args: + all_token_logits (torch.Tensor): logits of input tokens. shape: [s, b, h] or [total_nnz, 1, h] + labels (torch.Tensor): labels of input tokens. shape: [s, b] or [total_nnz, 1] + training_inputs (Dict[str, Any]): All training inputs. + + """ + forward_logprob = ( + model.compute_language_model_loss(labels, all_token_logits) * -1 + ) + + forward_logprob = reduce_from_context_parallel_region(forward_logprob, module_args.packing, training_inputs) + + old_logprobs = training_inputs["old_logprobs"] + ref_logprobs = training_inputs["ref_logprobs"] + advantages = training_inputs["advantages"] + + + if module_args.use_group_sequence_policy: + ( + pg_loss, + is_positive_clipped, + is_negative_clipped, + is_clipped, + ) = calculate_gspo_loss( + log_probs=forward_logprob, + old_log_probs=old_logprobs, + advantages=advantages, + diff_clip_ratio=module_args.diff_clip_ratio, + pos_clip_ratio=module_args.pos_clip_ratio, + neg_clip_ratio=module_args.neg_clip_ratio, + final_clip_ratio=module_args.final_clip_ratio, + loss_mask = training_inputs['all_token_loss_mask'] + ) + else: + pg_loss = calculate_grpo_loss( + log_probs=forward_logprob, + old_log_probs=old_logprobs, + advantages=advantages, + diff_clip_ratio=module_args.diff_clip_ratio, + pos_clip_ratio=module_args.pos_clip_ratio, + neg_clip_ratio=module_args.neg_clip_ratio, + final_clip_ratio=module_args.final_clip_ratio + ) + + entropy_loss = entropy_from_tensor_parallel_logits(all_token_logits).permute(1, 0) + + entropy_loss = reduce_from_context_parallel_region(entropy_loss, module_args.packing, training_inputs) + + kl = ref_logprobs - forward_logprob + ratio = torch.exp(kl) + ratio[~training_inputs['all_token_loss_mask'].bool()] = 1 + assert not torch.isinf(ratio).any(), "kl loss ratio has inf values" + assert not torch.isnan(ratio).any(), "kl loss ratio has nan values" + kld = (ratio - kl - 1).contiguous() + kl_loss = torch.clamp(kld, min=-10, max=10) + + if module_args.use_group_sequence_policy: + return { + 'pg_loss': pg_loss, + 'entropy_loss': entropy_loss, + 'kl_loss': kl_loss, + 'is_positive_clipped': is_positive_clipped, + 'is_negative_clipped': is_negative_clipped, + 'is_clipped': is_clipped, + 'is_positive_clipped_sample_average': is_positive_clipped, + 'is_negative_clipped_sample_average': is_negative_clipped, + 'is_clipped_sample_average': is_clipped, + } + else: + return { + 'pg_loss': pg_loss, + 'entropy_loss': entropy_loss, + 'kl_loss': kl_loss + } -def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: bool=False): + +def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: bool=False, module_args = None): """Forward step.""" inputs = get_batch( @@ -475,7 +560,8 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b kwargs = { 'input_ids': inputs["all_tokens"], 'position_ids': inputs["all_token_position_ids"], - 'labels': inputs["labels"], + # 'labels': inputs["labels"], + 'labels': None, 'training_inputs': inputs if is_training else None, 'packed_seq_params': inputs['packed_seq_params'] if is_packing else None } @@ -493,6 +579,21 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b output_tensor = model(**kwargs) + if is_training: + output_tensor = _compute_all_losses( + module_args=module_args, + model=model, + all_token_logits=output_tensor.transpose(0, 1).contiguous(), + labels=inputs['labels'], + training_inputs=inputs + ) + else: + output_tensor = model.compute_language_model_loss( + inputs["labels"], + output_tensor.transpose( + 0, 1 + ).contiguous(), # [b s h] => [s b h] + ) if is_training: wrapped_loss_func = partial(loss_func, inputs) diff --git a/chatlearn/configs/megatron_config.py b/chatlearn/configs/megatron_config.py index ad244cc4..3ba9f4fd 100644 --- a/chatlearn/configs/megatron_config.py +++ b/chatlearn/configs/megatron_config.py @@ -212,6 +212,11 @@ def _post_init_impl(self): @dataclass class MegatronConfig(BaseConfig): """configs for megatron model""" + model_provider_module: str = field( + default='pretrain_gpt', metadata={ + "help": "A module with a model_provider() that could be imported in the current PYTHONPATH to build mcore model" + } + ) # NOTE: model parallel config tensor_model_parallel_size: int = field( default=1, metadata={"help": "tensor model parallel size"} diff --git a/chatlearn/synchronizer/mappers/mapper.py b/chatlearn/synchronizer/mappers/mapper.py index 0ab95c40..9d003858 100644 --- a/chatlearn/synchronizer/mappers/mapper.py +++ b/chatlearn/synchronizer/mappers/mapper.py @@ -129,14 +129,14 @@ def _map_vlm_model(self, model: nn.Module, vp_stage: int, layer_offset: int): )) # vision model decoder - for layer_idx in range(model.vision_config.num_layers): + for layer_idx in range(model.vision_model.config.num_layers): global_layer_id = layer_offset + layer_idx self._update_mapping(self._map_vision_layer( model.vision_model.decoder.layers[layer_idx], src_prefix=f"{vp_stage}-vision_model.decoder.layers.{layer_idx}.", dst_prefix=f"{dst_vision_prefix}blocks.{global_layer_id}.", - num_attention_heads=model.vision_config.num_attention_heads, - num_query_groups=model.vision_config.num_query_groups + num_attention_heads=model.vision_model.config.num_attention_heads, + num_query_groups=model.vision_model.config.num_query_groups )) # vision model projection From 1d4ed891e1a4c38a7e3f390cf794b7f10793c484 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Thu, 9 Oct 2025 10:49:51 +0800 Subject: [PATCH 11/28] test qwen2_5_vl --- .../algorithm/grpo_utils/megatron_policy_trainer.py | 6 +++--- .../grpo_utils/megatron_utils/train_helper.py | 13 ++++++------- .../train_mcore_sglang_qwen2_5_vl_7b_grpo.sh | 3 ++- ...ore_sglang_qwen2_5_vl_7b_grpo_rollout_manager.sh | 3 ++- .../train_mcore_vllm_qwen2_5_vl_7b_grpo.sh | 3 ++- template/grpo_megatron.yaml | 2 ++ 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py b/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py index a445add9..0e356f6f 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py +++ b/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py @@ -18,7 +18,7 @@ import itertools from typing import List, Dict, Any from collections import defaultdict -from importlib import import_module +import importlib import torch from megatron.core import mpu @@ -66,9 +66,9 @@ def setup(self): self._metric_prefix = "policy_trainer" try: - module = import_module(self.module_args.model_provider_module) + module = importlib.import_module(self.module_args.model_provider_module) except ImportError as e: - raise ImportError(f"Failed to import {self.module_args.model_provider_module} in current PYTHONPATH, get error: {e.msg}") + raise ImportError(f"Failed to import {self.module_args.model_provider_module} in current PYTHONPATH, get error: {e.msg} {importlib.import_module('examples').__file__}") # module should has attr model_provider if not hasattr(module, 'model_provider'): diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index d80d4c7f..d60a47af 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -479,7 +479,7 @@ def _compute_all_losses( """ forward_logprob = ( - model.compute_language_model_loss(labels, all_token_logits) * -1 + unwrap_model(model).compute_language_model_loss(labels, all_token_logits) * -1 ) forward_logprob = reduce_from_context_parallel_region(forward_logprob, module_args.packing, training_inputs) @@ -560,17 +560,16 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b kwargs = { 'input_ids': inputs["all_tokens"], 'position_ids': inputs["all_token_position_ids"], - # 'labels': inputs["labels"], 'labels': None, - 'training_inputs': inputs if is_training else None, 'packed_seq_params': inputs['packed_seq_params'] if is_packing else None } if 'pixel_values' in inputs: kwargs.update({ - 'pixel_values': inputs["pixel_values"], - 'image_grid_thw': inputs["image_grid_thw"], - 'image_input_mask': inputs["image_input_mask"] + 'vision_data': inputs["pixel_values"], + 'vision_grid_thw': inputs["image_grid_thw"], + 'image_input_mask': inputs["image_input_mask"], + 'video_start_index': inputs["image_input_mask"].sum().cpu().item() }) else: kwargs.update({ @@ -588,7 +587,7 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b training_inputs=inputs ) else: - output_tensor = model.compute_language_model_loss( + output_tensor = unwrap_model(model).compute_language_model_loss( inputs["labels"], output_tensor.transpose( 0, 1 diff --git a/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo.sh b/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo.sh index 50775fb8..09f84d84 100644 --- a/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo.sh +++ b/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo.sh @@ -10,7 +10,7 @@ export VLLM_USE_RAY_COMPILED_DAG=1 export CHATLEARN=$(pwd) export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250624 -export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch +export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch:${CHATLEARN}/../Pai-Megatron-Patch/examples export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATCH_PATH}:${MEGATRON_PATH}:${PYTHONPATH} source scripts/base_env.sh @@ -43,6 +43,7 @@ python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ runtime_args.eval_episode_interval=1 \ runtime_args.enable_eval_before_training=True \ runtime_args.model_type=vlm \ + models.policy_trainer.model_provider_module=qwen2_5_vl.pretrain_qwen \ models.policy_trainer.num_gpu=${num_device} \ models.policy_trainer.packing=False \ models.policy_trainer.max_token_in_packing=8192 \ diff --git a/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo_rollout_manager.sh b/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo_rollout_manager.sh index a73769a8..9c9cecbe 100644 --- a/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo_rollout_manager.sh +++ b/scripts/mcore_sglang/train_mcore_sglang_qwen2_5_vl_7b_grpo_rollout_manager.sh @@ -10,7 +10,7 @@ export VLLM_USE_RAY_COMPILED_DAG=1 export CHATLEARN=$(pwd) export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250624 -export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch +export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch:${CHATLEARN}/../Pai-Megatron-Patch/examples export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATCH_PATH}:${MEGATRON_PATH}:${PYTHONPATH} source scripts/base_env.sh @@ -45,6 +45,7 @@ python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ runtime_args.eval_episode_interval=1 \ runtime_args.enable_eval_before_training=True \ runtime_args.model_type=vlm \ + models.policy_trainer.model_provider_module=qwen2_5_vl.pretrain_qwen \ models.policy_trainer.num_gpu=${num_device} \ models.policy_trainer.packing=False \ models.policy_trainer.max_token_in_packing=8192 \ diff --git a/scripts/mcore_vllm/train_mcore_vllm_qwen2_5_vl_7b_grpo.sh b/scripts/mcore_vllm/train_mcore_vllm_qwen2_5_vl_7b_grpo.sh index 8134f949..2e0594ac 100644 --- a/scripts/mcore_vllm/train_mcore_vllm_qwen2_5_vl_7b_grpo.sh +++ b/scripts/mcore_vllm/train_mcore_vllm_qwen2_5_vl_7b_grpo.sh @@ -10,7 +10,7 @@ export VLLM_USE_RAY_COMPILED_DAG=1 export CHATLEARN=$(pwd) export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250624 -export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch +export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch:${CHATLEARN}/../Pai-Megatron-Patch/examples export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATCH_PATH}:${MEGATRON_PATH}:${PYTHONPATH} source scripts/base_env.sh @@ -42,6 +42,7 @@ python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ runtime_args.eval_episode_interval=1 \ runtime_args.enable_eval_before_training=True \ runtime_args.model_type=vlm \ + models.policy_trainer.model_provider_module=qwen2_5_vl.pretrain_qwen \ models.policy_trainer.num_gpu=${num_device} \ models.policy_trainer.packing=False \ models.policy_trainer.max_token_in_packing=8192 \ diff --git a/template/grpo_megatron.yaml b/template/grpo_megatron.yaml index 1f0e0dd3..7853256d 100644 --- a/template/grpo_megatron.yaml +++ b/template/grpo_megatron.yaml @@ -76,6 +76,7 @@ models: use_group_sequence_policy: False packing: true max_token_in_packing: ${models.policy_trainer.seq_length} + model_provider_module: pretrain_gpt ref_policy: free_gpu_memory: offload_weights: True @@ -101,6 +102,7 @@ models: load: ${models.policy_trainer.load} packing: ${models.policy_trainer.packing} max_token_in_packing: ${models.policy_trainer.max_token_in_packing} + model_provider_module: ${models.policy_trainer.model_provider_module} policy: free_gpu_memory: offload_weights: True From c262631723b8e31ea292060262fb146e81d0b2dd Mon Sep 17 00:00:00 2001 From: lostkevin Date: Thu, 9 Oct 2025 10:53:16 +0800 Subject: [PATCH 12/28] fix pylint --- .../grpo_utils/megatron_utils/train_helper.py | 160 +++++++++--------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index d60a47af..e355c837 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -464,88 +464,88 @@ def loss_func( return total_loss_for_bp, num_tokens, reporting_losses def _compute_all_losses( - module_args, - model, - all_token_logits: torch.Tensor, - labels: torch.Tensor, - training_inputs: Dict[str, Any], - ) -> Dict[str, torch.Tensor]: - """Compute all required losses. - - Args: - all_token_logits (torch.Tensor): logits of input tokens. shape: [s, b, h] or [total_nnz, 1, h] - labels (torch.Tensor): labels of input tokens. shape: [s, b] or [total_nnz, 1] - training_inputs (Dict[str, Any]): All training inputs. - - """ - forward_logprob = ( - unwrap_model(model).compute_language_model_loss(labels, all_token_logits) * -1 - ) + module_args, + model, + all_token_logits: torch.Tensor, + labels: torch.Tensor, + training_inputs: Dict[str, Any], +) -> Dict[str, torch.Tensor]: + """Compute all required losses. + + Args: + all_token_logits (torch.Tensor): logits of input tokens. shape: [s, b, h] or [total_nnz, 1, h] + labels (torch.Tensor): labels of input tokens. shape: [s, b] or [total_nnz, 1] + training_inputs (Dict[str, Any]): All training inputs. + + """ + forward_logprob = ( + unwrap_model(model).compute_language_model_loss(labels, all_token_logits) * -1 + ) - forward_logprob = reduce_from_context_parallel_region(forward_logprob, module_args.packing, training_inputs) - - old_logprobs = training_inputs["old_logprobs"] - ref_logprobs = training_inputs["ref_logprobs"] - advantages = training_inputs["advantages"] - - - if module_args.use_group_sequence_policy: - ( - pg_loss, - is_positive_clipped, - is_negative_clipped, - is_clipped, - ) = calculate_gspo_loss( - log_probs=forward_logprob, - old_log_probs=old_logprobs, - advantages=advantages, - diff_clip_ratio=module_args.diff_clip_ratio, - pos_clip_ratio=module_args.pos_clip_ratio, - neg_clip_ratio=module_args.neg_clip_ratio, - final_clip_ratio=module_args.final_clip_ratio, - loss_mask = training_inputs['all_token_loss_mask'] - ) - else: - pg_loss = calculate_grpo_loss( - log_probs=forward_logprob, - old_log_probs=old_logprobs, - advantages=advantages, - diff_clip_ratio=module_args.diff_clip_ratio, - pos_clip_ratio=module_args.pos_clip_ratio, - neg_clip_ratio=module_args.neg_clip_ratio, - final_clip_ratio=module_args.final_clip_ratio - ) + forward_logprob = reduce_from_context_parallel_region(forward_logprob, module_args.packing, training_inputs) - entropy_loss = entropy_from_tensor_parallel_logits(all_token_logits).permute(1, 0) - - entropy_loss = reduce_from_context_parallel_region(entropy_loss, module_args.packing, training_inputs) - - kl = ref_logprobs - forward_logprob - ratio = torch.exp(kl) - ratio[~training_inputs['all_token_loss_mask'].bool()] = 1 - assert not torch.isinf(ratio).any(), "kl loss ratio has inf values" - assert not torch.isnan(ratio).any(), "kl loss ratio has nan values" - kld = (ratio - kl - 1).contiguous() - kl_loss = torch.clamp(kld, min=-10, max=10) - - if module_args.use_group_sequence_policy: - return { - 'pg_loss': pg_loss, - 'entropy_loss': entropy_loss, - 'kl_loss': kl_loss, - 'is_positive_clipped': is_positive_clipped, - 'is_negative_clipped': is_negative_clipped, - 'is_clipped': is_clipped, - 'is_positive_clipped_sample_average': is_positive_clipped, - 'is_negative_clipped_sample_average': is_negative_clipped, - 'is_clipped_sample_average': is_clipped, - } - else: - return { - 'pg_loss': pg_loss, - 'entropy_loss': entropy_loss, - 'kl_loss': kl_loss - } + old_logprobs = training_inputs["old_logprobs"] + ref_logprobs = training_inputs["ref_logprobs"] + advantages = training_inputs["advantages"] + + + if module_args.use_group_sequence_policy: + ( + pg_loss, + is_positive_clipped, + is_negative_clipped, + is_clipped, + ) = calculate_gspo_loss( + log_probs=forward_logprob, + old_log_probs=old_logprobs, + advantages=advantages, + diff_clip_ratio=module_args.diff_clip_ratio, + pos_clip_ratio=module_args.pos_clip_ratio, + neg_clip_ratio=module_args.neg_clip_ratio, + final_clip_ratio=module_args.final_clip_ratio, + loss_mask = training_inputs['all_token_loss_mask'] + ) + else: + pg_loss = calculate_grpo_loss( + log_probs=forward_logprob, + old_log_probs=old_logprobs, + advantages=advantages, + diff_clip_ratio=module_args.diff_clip_ratio, + pos_clip_ratio=module_args.pos_clip_ratio, + neg_clip_ratio=module_args.neg_clip_ratio, + final_clip_ratio=module_args.final_clip_ratio + ) + + entropy_loss = entropy_from_tensor_parallel_logits(all_token_logits).permute(1, 0) + + entropy_loss = reduce_from_context_parallel_region(entropy_loss, module_args.packing, training_inputs) + + kl = ref_logprobs - forward_logprob + ratio = torch.exp(kl) + ratio[~training_inputs['all_token_loss_mask'].bool()] = 1 + assert not torch.isinf(ratio).any(), "kl loss ratio has inf values" + assert not torch.isnan(ratio).any(), "kl loss ratio has nan values" + kld = (ratio - kl - 1).contiguous() + kl_loss = torch.clamp(kld, min=-10, max=10) + + if module_args.use_group_sequence_policy: + return { + 'pg_loss': pg_loss, + 'entropy_loss': entropy_loss, + 'kl_loss': kl_loss, + 'is_positive_clipped': is_positive_clipped, + 'is_negative_clipped': is_negative_clipped, + 'is_clipped': is_clipped, + 'is_positive_clipped_sample_average': is_positive_clipped, + 'is_negative_clipped_sample_average': is_negative_clipped, + 'is_clipped_sample_average': is_clipped, + } + else: + return { + 'pg_loss': pg_loss, + 'entropy_loss': entropy_loss, + 'kl_loss': kl_loss + } def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: bool=False, module_args = None): From cd83710dfff82f0738e50a5693cf8ac721f9697d Mon Sep 17 00:00:00 2001 From: lostkevin Date: Thu, 9 Oct 2025 14:32:56 +0800 Subject: [PATCH 13/28] fix issues when PP > 1 --- .../grpo_utils/megatron_utils/train_helper.py | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index e355c837..cba8c9f9 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -560,7 +560,7 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b kwargs = { 'input_ids': inputs["all_tokens"], 'position_ids': inputs["all_token_position_ids"], - 'labels': None, + 'labels': inputs["labels"] if not is_training else None, 'packed_seq_params': inputs['packed_seq_params'] if is_packing else None } @@ -576,34 +576,29 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b 'attention_mask': inputs["all_token_attention_mask"] }) + # NOTE: + # 1) when post_process is False, model returns hidden states + # 2) when post_process is True: + # 1) if is_training is False, model returns logprobs + # 2) otherwise, model returns logits and loss should be computed by `_compute_all_losses` output_tensor = model(**kwargs) - if is_training: - output_tensor = _compute_all_losses( - module_args=module_args, - model=model, - all_token_logits=output_tensor.transpose(0, 1).contiguous(), - labels=inputs['labels'], - training_inputs=inputs - ) - else: - output_tensor = unwrap_model(model).compute_language_model_loss( - inputs["labels"], - output_tensor.transpose( - 0, 1 - ).contiguous(), # [b s h] => [s b h] - ) + if unwrap_model(model).post_process: + if is_training: + output_tensor = _compute_all_losses( + module_args=module_args, + model=model, + all_token_logits=output_tensor.transpose(0, 1).contiguous(), + labels=inputs['labels'], + training_inputs=inputs + ) + else: + output_tensor = reduce_from_context_parallel_region(output_tensor, is_packing, inputs) + # NOTE: just returns the output tensor (the first argument). + wrapped_loss_func = lambda x, **_: x # pylint: disable=unnecessary-lambda-assignment if is_training: wrapped_loss_func = partial(loss_func, inputs) - - else: - if unwrap_model(model).post_process: - output_tensor = reduce_from_context_parallel_region(output_tensor, is_packing, inputs) - - # NOTE: just returns the output tensor (the first argument). - wrapped_loss_func = lambda x, **_: x # pylint: disable=unnecessary-lambda-assignment - return output_tensor, wrapped_loss_func class _VocabParallelEntropy(torch.autograd.Function): From 44801fc3fb842ecfe41e6892c2e3e5799b0a5161 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Thu, 9 Oct 2025 17:11:41 +0800 Subject: [PATCH 14/28] passing a copy to avoid inplace modification on fp32 logits --- .../algorithm/grpo_utils/megatron_utils/train_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index cba8c9f9..140bdd34 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -479,7 +479,7 @@ def _compute_all_losses( """ forward_logprob = ( - unwrap_model(model).compute_language_model_loss(labels, all_token_logits) * -1 + unwrap_model(model).compute_language_model_loss(labels, all_token_logits.clone()) * -1 ) forward_logprob = reduce_from_context_parallel_region(forward_logprob, module_args.packing, training_inputs) @@ -576,14 +576,14 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b 'attention_mask': inputs["all_token_attention_mask"] }) - # NOTE: + # NOTE: # 1) when post_process is False, model returns hidden states # 2) when post_process is True: # 1) if is_training is False, model returns logprobs # 2) otherwise, model returns logits and loss should be computed by `_compute_all_losses` output_tensor = model(**kwargs) - if unwrap_model(model).post_process: + if model.post_process: if is_training: output_tensor = _compute_all_losses( module_args=module_args, From b56aa20da7393397509e2a011aded56f53cacdf6 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Thu, 9 Oct 2025 17:23:23 +0800 Subject: [PATCH 15/28] fix issue --- chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index 140bdd34..f2a33677 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -583,7 +583,7 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b # 2) otherwise, model returns logits and loss should be computed by `_compute_all_losses` output_tensor = model(**kwargs) - if model.post_process: + if unwrap_model(model).post_process: if is_training: output_tensor = _compute_all_losses( module_args=module_args, From 682a144bba13441a2a3a6008e842784f76959c88 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 13 Oct 2025 17:24:28 +0800 Subject: [PATCH 16/28] add draft version of qwen3-vl --- .../grpo_utils/megatron_utils/train_helper.py | 4 +- chatlearn/configs/megatron_config.py | 5 + chatlearn/models/sglang_module.py | 14 +- chatlearn/runtime/engine.py | 2 +- .../mappers/base_megatron_mapper.py | 41 ++- .../synchronizer/mappers/mapping_helpers.py | 153 +++++++--- .../mappers/megatron_llm_mapper.py | 267 +++++++++++++----- .../mappers/megatron_vlm_mapper.py | 19 +- chatlearn/utils/mappings/megatron_helpers.py | 43 +++ .../utils/mappings/sharded_tensor_info.py | 32 +++ chatlearn/utils/megatron_utils.py | 46 +++ chatlearn/utils/utils.py | 2 +- .../train_fsdp_sglang_qwen3_next_small.sh | 51 ++++ .../train_mcore_sglang_qwen3_next_grpo.sh | 71 +++++ ...rain_mcore_sglang_qwen3_next_small_grpo.sh | 73 +++++ tests/parameter_sync/test_mapper_helpers.py | 18 +- 16 files changed, 703 insertions(+), 138 deletions(-) create mode 100644 scripts/fsdp_sglang/train_fsdp_sglang_qwen3_next_small.sh create mode 100644 scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh create mode 100644 scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index f2a33677..1ec2e042 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -561,9 +561,11 @@ def forward_step(data_iterator, model, *, is_training: bool=False, is_packing: b 'input_ids': inputs["all_tokens"], 'position_ids': inputs["all_token_position_ids"], 'labels': inputs["labels"] if not is_training else None, - 'packed_seq_params': inputs['packed_seq_params'] if is_packing else None } + if is_packing: + kwargs.update({'packed_seq_params': inputs['packed_seq_params']}) + if 'pixel_values' in inputs: kwargs.update({ 'vision_data': inputs["pixel_values"], diff --git a/chatlearn/configs/megatron_config.py b/chatlearn/configs/megatron_config.py index ff40b33b..72f6b59f 100644 --- a/chatlearn/configs/megatron_config.py +++ b/chatlearn/configs/megatron_config.py @@ -245,6 +245,11 @@ class MegatronModelArchitectureConfig(BaseConfig): freeze_VP: bool = field( default=False, metadata={"help": "Freeze vision projection layers"} ) + + hybrid_override_pattern: Optional[str] = None + is_hybrid_model: bool = False + apply_layernorm_1p: bool = False + def _post_init_impl(self): if self.moe_aux_loss_coeff == 0: self.moe_router_load_balancing_type = 'none' diff --git a/chatlearn/models/sglang_module.py b/chatlearn/models/sglang_module.py index dfb91970..b4093690 100644 --- a/chatlearn/models/sglang_module.py +++ b/chatlearn/models/sglang_module.py @@ -32,7 +32,7 @@ from torch.distributed.device_mesh import init_device_mesh from transformers import AutoTokenizer, AutoModelForImageTextToText, AutoModelForCausalLM, AutoConfig, AutoProcessor -from chatlearn.runtime.decorator import timeit, compute_decorator +from chatlearn.runtime.decorator import timeit, compute_decorator, monitor_error from chatlearn.utils.utils import get_full_proc_memory_info from chatlearn.utils.mappings import ShardedTensorInfo from chatlearn.utils.mappings.huggingface_helpers import build_sharded_info_for_huggingface_model @@ -412,6 +412,12 @@ def generate(self, query: List[Dict], is_eval: bool) -> List[Dict]: self.flush_cache() return outputs + def dump_parameters(self, dump_path_root): + os.makedirs(dump_path_root, exist_ok=True) + self.onload() + self.llm.save_sharded_model(path=dump_path_root, pattern=None, max_size=None) + self.offload() + def update_weights_from_ipc_handles(self, reduce_data): gathered_data = None if self.is_engine(): @@ -725,6 +731,12 @@ async def generate(self, query: List[Dict], is_eval: bool) -> List[Dict]: ) return outputs + async def dump_parameters(self, dump_path_root): + os.makedirs(dump_path_root, exist_ok=True) + await self.onload() + self.llm.save_sharded_model(path=dump_path_root, pattern=None, max_size=None) + await self.offload() + async def generate_per_request(self, query: Dict, is_eval: bool) -> Dict: outputs = None if self.is_engine(): diff --git a/chatlearn/runtime/engine.py b/chatlearn/runtime/engine.py index 59100992..87d3968e 100644 --- a/chatlearn/runtime/engine.py +++ b/chatlearn/runtime/engine.py @@ -556,7 +556,7 @@ def _resume_from_data_checkpoint(self): def dump_parameters(self, dump_path): for _, model in enumerate(self.models): replic_0 = model.replicas[0] - if isinstance(replic_0, DistVLLMActor): + if isinstance(replic_0, (DistVLLMActor, DistSGLangActor)): future.wait(replic_0.engine.dump_parameters.remote(dump_path)) def save_checkpoint(self, episode_id): diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index 082c10ee..be25594e 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -15,7 +15,7 @@ """Basic Mapper for Megatron to rollout framework""" from collections import defaultdict -from typing import List, Dict, TYPE_CHECKING, Union +from typing import List, Dict, TYPE_CHECKING, Union, Tuple from megatron.training.utils import unwrap_model @@ -27,6 +27,7 @@ process_normal_tensor, process_gate_up_tensor, process_qkv_tensor, + process_merged_linear_tensor, VLLM_HELPERS, HF_HELPERS ) @@ -165,7 +166,7 @@ def _inner_map_for_gate_up_proj(self, src_key: str, dst_key: str, proj_type: str self._update_mapping(mapping) return mapping - def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, num_attention_heads: int, num_query_groups: int): + def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, num_attention_heads: int, num_query_groups: int, is_gated_attention: bool=False): src_info = self._src_name_to_metadata[src_key] dst_info = self._dst_name_to_metadata[dst_key] mapping = defaultdict(list) @@ -174,7 +175,8 @@ def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, nu num_attention_heads, num_query_groups, self._dst_tp_size, - proj_type=proj_type + proj_type=proj_type, + is_gated_attention=is_gated_attention ): src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype @@ -192,6 +194,39 @@ def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): self._update_mapping(results) return results + def _inner_map_for_merged_linear( + self, + src_key: str, + dst_key: str, + src_layout: List[Tuple[str, int]], + required_layout: List[str], + *, + global_expert_id: int=None, + num_experts: int=None, + axis: int = 0 + ): + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + mapping = {} + for src_meta, dst_meta in process_merged_linear_tensor( + src_info, + self._dst_tp_size, + src_layout=src_layout, + required_layout=required_layout, + axis=axis + ): + src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id + src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype + if global_expert_id is not None: + dst_meta = ( + dst_meta + .unsqueeze(offset=global_expert_id, length=num_experts, axis=0) + .refragment(1, axis=0) # 1 is dst EP + ) + mapping[src_meta] = [dst_meta] + self._update_mapping(mapping) + return mapping + def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]) -> None: if self._mapping is None: self._mapping = defaultdict(list) diff --git a/chatlearn/synchronizer/mappers/mapping_helpers.py b/chatlearn/synchronizer/mappers/mapping_helpers.py index b1c65377..4a495629 100644 --- a/chatlearn/synchronizer/mappers/mapping_helpers.py +++ b/chatlearn/synchronizer/mappers/mapping_helpers.py @@ -19,6 +19,7 @@ from itertools import chain from chatlearn.utils.mappings import ShardedTensorInfo +from chatlearn.utils.utils import slice_data_list_by_index def process_normal_tensor( sharded_info: ShardedTensorInfo, @@ -48,25 +49,6 @@ def process_normal_tensor( ) for tensor_part_info in sharded_info.fragment(dst_tp_size, axis) ] -def _build_gate_up_layout(src_tp_size: int, dst_tp_size: int): - """ - build layout with 2 * lcm(src_tp, dst_tp) chunks - """ - n_chunks = math.lcm(src_tp_size, dst_tp_size) - flatten = lambda x: list(chain.from_iterable(x)) # pylint: disable=unnecessary-lambda-assignment - mcore_layout = flatten([ - [ f"g{c_id + tp_rank * (n_chunks // src_tp_size)}" for c_id in range(n_chunks // src_tp_size) ] + - [ f"u{c_id + tp_rank * (n_chunks // src_tp_size)}" for c_id in range(n_chunks // src_tp_size) ] - for tp_rank in range(src_tp_size) - ]) - - vllm_layout = flatten([ - [ f"g{c_id + tp_rank * (n_chunks // dst_tp_size)}" for c_id in range(n_chunks // dst_tp_size) ] + - [ f"u{c_id + tp_rank * (n_chunks // dst_tp_size)}" for c_id in range(n_chunks // dst_tp_size) ] - for tp_rank in range(dst_tp_size) - ]) - - return mcore_layout, vllm_layout def process_gate_up_tensor( sharded_info: ShardedTensorInfo, @@ -83,74 +65,147 @@ def process_gate_up_tensor( Returns: List[Tuple[ShardedTensorInfo, ...]]: The layout mapping. """ - src_tp_size = sharded_info.axis_fragmentations[0] - - mcore_layout, vllm_layout = _build_gate_up_layout(src_tp_size, dst_tp_size) - mcore_id_to_frags = { - part.global_offset[0]: part.refragment(src_tp_size) - for part in sharded_info.fragment(math.lcm(src_tp_size, dst_tp_size) * 2) - } + gate_up = sharded_info.global_shape[0] if proj_type == 'gate_up_proj': - n_chunks = math.lcm(src_tp_size, dst_tp_size) * 2 - full_dst_info = ShardedTensorInfo.from_global_shape(sharded_info.global_shape) + layout = ['gate', 'up'] + elif proj_type == 'up_proj': + layout = ['up'] else: - n_chunks = math.lcm(src_tp_size, dst_tp_size) - full_dst_info = ShardedTensorInfo.from_global_shape( - (sharded_info.global_shape[0] // 2, ) + sharded_info.global_shape[1:] + layout = ['gate'] + return process_merged_linear_tensor( + sharded_info, + dst_tp_size, + src_layout=[('gate', gate_up // 2), ('up', gate_up // 2)], + required_layout=layout + ) + + +def _build_merged_linear_layout( + layout: List[Tuple[str, int]], + n_chunks: int, + tp_size: int, +) -> List[Tuple[str, int, int]]: + flatten = lambda x: list(chain.from_iterable(x)) # pylint: disable=unnecessary-lambda-assignment + mcore_layout = flatten([ + flatten([ + [ (key, c_id + tp_rank * (n_chunks // tp_size), size // n_chunks) for c_id in range(n_chunks // tp_size) ] + for key, size in layout + ]) + for tp_rank in range(tp_size) + ]) + return mcore_layout + +def process_merged_linear_tensor( + sharded_info: ShardedTensorInfo, + dst_tp_size: int, + src_layout: List[Tuple[str, int]], + required_layout: List[str], + axis: int = 0 +) -> List[Tuple[ShardedTensorInfo, ...]]: + """ + A generalized implementation to resolve mapping on a column-merged linear + """ + src_tp_rank = sharded_info.global_offset[axis] + src_tp_size = sharded_info.axis_fragmentations[axis] + n_chunks = math.lcm(src_tp_size, dst_tp_size) + keyname_to_size = {item[0] : item[1] for item in src_layout} + + src_names = [item[0] for item in src_layout] + if not set(required_layout).issubset(set(src_names)): + raise ValueError(f"Expect all keys of the required layout is the subset of source layout {src_names}, but {required_layout}") + + mcore_layout = slice_data_list_by_index(_build_merged_linear_layout( + src_layout, + n_chunks, + src_tp_size + ), (src_tp_rank, src_tp_size)) + + id_to_frags = { + (item[0], item[1]): part + for item, part in zip( + mcore_layout, + sharded_info.chunk(sections=[item[2] for item in mcore_layout], axis=axis) ) + } + + full_dst_size = sum(keyname_to_size[name] for name in required_layout) + full_dst_info = ShardedTensorInfo.from_global_shape( + (full_dst_size, ) + sharded_info.global_shape[1:] + ) + vllm_layout = _build_merged_linear_layout( + [(name, keyname_to_size[name]) for name in required_layout], + n_chunks, + dst_tp_size + ) results = [] - for chunk_idx, dst_part in enumerate(full_dst_info.fragment(n_chunks)): - if proj_type == 'gate_up_proj': - chunk_name = vllm_layout[chunk_idx] - else: - chunk_name = f"{proj_type[:1]}{chunk_idx}" - mcore_idx = mcore_layout.index(chunk_name) - if mcore_idx not in mcore_id_to_frags: + for (name, chunk_id, _), dst_part in zip( + vllm_layout, + full_dst_info.chunk(sections=[item[2] for item in vllm_layout], axis=axis) + ): + if (name, chunk_id) not in id_to_frags: continue results.append(( - mcore_id_to_frags[mcore_idx], - dst_part.refragment(dst_tp_size) + id_to_frags[(name, chunk_id)], + dst_part.refragment(dst_tp_size, axis=axis) )) return __maybe_merge(results) - def _build_qkv_layout( num_heads: int, num_query_group: int, - dst_tp_size: int + dst_tp_size: int, + is_gated_attention: bool = False ): """Generate a mapping between mcore qkv heads (mix-style qkv) and vllm qkv heads (no mix-style qkv). + is_gated_attention=False: Mcore layout of first dim per tp rank when nh=24, ng=8, tp=4, nq=3: [q q q k v q q q k v], while vLLM: [q q q q q q k k v v] + is_gated_attention=True: + Mcore layout of first dim per tp rank when + nh=48, ng=8, tp=4, nq=3: [q q q g g g k v q q q g g g k v], + while vLLM: [q g q g q g q g q g q g k k v v] + Args: - num_heads (int): The num of attention heads + num_heads (int): The num of attention heads. If is_gated_attention is True, the number should be + the total amount of query heads and gate heads num_query_group (int): The num of query groups dst_tp_size (int): The dst tensor parallel size + is_gated_attention (bool, optional): whether query heads have corresponding gate head """ flatten = lambda x: list(chain.from_iterable(x)) # pylint: disable=unnecessary-lambda-assignment + if is_gated_attention: + num_heads //= 2 nq = num_heads // num_query_group mcore_layout = [] vllm_layout = [] mcore_layout = flatten([ - [ f"q{g_id * nq + q_id}" for q_id in range(nq)] + [f"k{g_id}", f"v{g_id}"] + [ f"q{g_id * nq + q_id}" for q_id in range(nq)] + + ([ f"g{g_id * nq + q_id}" for q_id in range(nq)] if is_gated_attention else []) + + [f"k{g_id}", f"v{g_id}"] for g_id in range(num_query_group) ]) vllm_nq = num_heads // dst_tp_size if dst_tp_size < num_query_group: vllm_layout = flatten([ - [f"q{r_id * vllm_nq + q_id}" for q_id in range(num_heads // dst_tp_size)] + + flatten([ + (f"q{r_id * vllm_nq + q_id}", f"g{r_id * vllm_nq + q_id}") if is_gated_attention else (f"q{r_id * vllm_nq + q_id}", ) + for q_id in range(num_heads // dst_tp_size) + ]) + [f"k{g_id + r_id * (num_query_group // dst_tp_size)}" for g_id in range(num_query_group // dst_tp_size)] + [f"v{g_id + r_id * (num_query_group // dst_tp_size)}" for g_id in range(num_query_group // dst_tp_size)] for r_id in range(dst_tp_size) ]) else: vllm_layout = flatten([ - [f"q{r_id * vllm_nq + q_id}" for q_id in range(num_heads // dst_tp_size)] + + flatten([ + (f"q{r_id * vllm_nq + q_id}", f"g{r_id * vllm_nq + q_id}" if is_gated_attention else (f"q{r_id * vllm_nq + q_id}",)) + for q_id in range(num_heads // dst_tp_size) + ]) + [f"k{r_id * num_query_group // dst_tp_size}", f"v{r_id * num_query_group // dst_tp_size}"] for r_id in range(dst_tp_size) ]) @@ -162,7 +217,8 @@ def process_qkv_tensor( num_heads: int, num_query_groups: Optional[int], dst_tp_size: int, - proj_type: Literal['qkv_proj', 'q_proj', 'k_proj', 'v_proj'] + proj_type: Literal['qkv_proj', 'q_proj', 'k_proj', 'v_proj'], + is_gated_attention: bool = False ) -> List[Tuple[ShardedTensorInfo, ...]]: """Process qkv weight/bias to generate shard mapping. @@ -172,9 +228,12 @@ def process_qkv_tensor( num_query_group (int): The number of query groups dst_tp_size (int): The target tensor parallel size proj_type (Literal['qkv_proj', 'q_proj', 'k_proj', 'v_proj']): the projection type + is_gated_attention (bool, optional): whether query heads have corresponding gate head """ if num_query_groups is None: num_query_groups = num_heads + if is_gated_attention: + num_heads *= 2 if num_query_groups % dst_tp_size != 0 and dst_tp_size % num_query_groups != 0: raise ValueError(f"num_query_groups {num_query_groups} must be divisible or multiple by dst_tp_size {dst_tp_size}") head_dim = sharded_info.global_shape[0] // (num_heads + 2 * num_query_groups) diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index f1e1ba1f..413ea1af 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -14,7 +14,7 @@ # ============================================================================== """Mapper for Megatron to vLLM""" -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Union, Dict import inspect from torch import nn @@ -24,6 +24,10 @@ from megatron.core.transformer.transformer_layer import get_transformer_layer_offset from megatron.core.transformer.moe.moe_layer import MoELayer from megatron.core.transformer.moe.experts import TEGroupedMLP +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.ssm.mamba_block import MambaStack +from megatron.core.ssm.mamba_layer import MambaLayer from chatlearn.configs import PolicyConfig @@ -41,6 +45,8 @@ ) from .base_megatron_mapper import BaseMegatronMapper +from chatlearn.utils.logger import logger + if TYPE_CHECKING: from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding from megatron.core.transformer.transformer_layer import TransformerLayer @@ -70,6 +76,25 @@ def __init__( super().__init__(dst_model_config=dst_model_config, model=model, mapper_config=mapper_config) # NOTE: the following function implements the module-wise sync mapping + def _build_layer_index_mapping(self, decoder, vp_stage): + """ + Map the local layer index (ranged from 0 to model.decoder.num_layers_per_pipeline_rank) + to the global huggingface layer index + """ + if isinstance(decoder, TransformerBlock): + layer_offset = get_transformer_layer_offset(decoder.config, vp_stage=vp_stage) + num_layers_per_pipeline_rank = decoder.num_layers_per_pipeline_rank + return {n: layer_offset + n for n in range(num_layers_per_pipeline_rank)} + elif isinstance(decoder, MambaStack): + assert vp_stage == 0, "Mamba do not support VPP" + # NOTE: currently we assume MambaLayer just replaces some of Attention + # layout should be: ((mamba | attn) mlp) x n + num_layers_per_pipeline_rank = decoder.num_layers_per_pipeline_rank + layer_offset = num_layers_per_pipeline_rank * mpu.get_pipeline_model_parallel_rank() + return {n: (n + layer_offset) // 2 for n in range(num_layers_per_pipeline_rank)} + else: + raise ValueError(f"Unexpected decoder type: {type(decoder)}") + def _map_model(self): """Mapping the local name of src model to global name of dst model @@ -81,23 +106,16 @@ def _map_model(self): ) ) for vp_stage, model in enumerate(self.model): - if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: - layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) - else: - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) - layer_offset = get_transformer_layer_offset(model.config) - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(None) - - # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch if getattr(model, 'mtp_process', False): raise NotImplementedError("Currently, the mapper does not support MTP") self._map_llm_model( model, cfg, - layer_offset=layer_offset, + index_mapping=self._build_layer_index_mapping( + model.decoder, + vp_stage + ), src_prefix=f"{vp_stage}-", dst_prefix="" ) @@ -110,7 +128,7 @@ def _map_llm_model( self, model: nn.Module, cfg: LanguageModelKeyMapping, - layer_offset: int, + index_mapping: Dict[int, int], src_prefix: str='', dst_prefix: str='' ): @@ -122,20 +140,34 @@ def _map_llm_model( ) for layer_idx in range(model.decoder.num_layers_per_pipeline_rank): - global_layer_id = layer_offset + layer_idx - self._map_decoder_layer( - model.decoder.layers[layer_idx], - cfg=cfg.decoder_layer_cfg, - src_prefix=f"{src_prefix}decoder.layers.{layer_idx}.", - dst_prefix=f"{dst_prefix}{cfg.decoder_layer}{global_layer_id}.", - ) + global_layer_id = index_mapping[layer_idx] + if isinstance(model.decoder.layers[layer_idx], MambaLayer): + self._map_mamba_layer( + model.decoder.layers[layer_idx], + src_prefix=f"{src_prefix}decoder.layers.{layer_idx}.", + dst_prefix=f"{dst_prefix}{cfg.decoder_layer}{global_layer_id}.", + ) + else: + self._map_transformer_layer( + model.decoder.layers[layer_idx], + cfg=cfg.decoder_layer_cfg, + src_prefix=f"{src_prefix}decoder.layers.{layer_idx}.", + dst_prefix=f"{dst_prefix}{cfg.decoder_layer}{global_layer_id}.", + ) if model.post_process: - self._map_norm_layer( - model.decoder.final_layernorm, - src_prefix=f"{src_prefix}decoder.final_layernorm.", - dst_prefix=f"{dst_prefix}{cfg.final_layernorm}", - ) + if isinstance(model.decoder, MambaStack): + self._map_norm_layer( + model.decoder.final_norm, + src_prefix=f"{src_prefix}decoder.final_norm.", + dst_prefix=f"{dst_prefix}{cfg.final_layernorm}", + ) + else: + self._map_norm_layer( + model.decoder.final_layernorm, + src_prefix=f"{src_prefix}decoder.final_layernorm.", + dst_prefix=f"{dst_prefix}{cfg.final_layernorm}", + ) if model.share_embeddings_and_output_weights and model.pre_process: self._map_postprocess_layer( @@ -171,51 +203,145 @@ def _map_norm_layer(self, module: nn.Module, src_prefix: str='', dst_prefix: str f"{dst_prefix}{_keynames[item]}" ) - def _map_decoder_layer(self, module: 'TransformerLayer', cfg: DecoderLayerKeyMapping, src_prefix: str='', dst_prefix: str=''): - if module.config.multi_latent_attention: - map_attn_func = self._map_mla_selfattn - norm_layer = module.input_layernorm - norm_src_key = f"{src_prefix}input_layernorm." - is_norm_layer = True - else: - map_attn_func = self._map_selfattn - norm_layer = module.self_attention.linear_qkv - norm_src_key = f"{src_prefix}self_attention.linear_qkv." - is_norm_layer = False - map_attn_func( - module.self_attention, - cfg=cfg.self_attn_cfg, - src_prefix=f"{src_prefix}self_attention.", - dst_prefix=f"{dst_prefix}{cfg.self_attn}", - ) + def _map_transformer_layer(self, module: 'TransformerLayer', cfg: DecoderLayerKeyMapping, src_prefix: str='', dst_prefix: str=''): + submodule_config = module.submodules_config + has_self_attention = submodule_config.self_attention is not IdentityOp + has_mlp = submodule_config.mlp is not IdentityOp + assert has_self_attention or has_mlp, f"The TransformerLayer should at least contains one of self_attn or mlp!" + + if has_self_attention: + if module.config.multi_latent_attention: + map_attn_func = self._map_mla_selfattn + norm_layer = module.input_layernorm + norm_src_key = f"{src_prefix}input_layernorm." + is_norm_layer = True + else: + map_attn_func = self._map_selfattn + is_gated_attention = hasattr(module.self_attention, 'linear_qgkv') + if is_gated_attention: + norm_layer = module.self_attention.linear_qgkv + norm_src_key = f"{src_prefix}self_attention.linear_qgkv." + else: + norm_layer = module.self_attention.linear_qkv + norm_src_key = f"{src_prefix}self_attention.linear_qkv." + is_norm_layer = False + map_attn_func( + module.self_attention, + cfg=cfg.self_attn_cfg, + src_prefix=f"{src_prefix}self_attention.", + dst_prefix=f"{dst_prefix}{cfg.self_attn}", + ) + self._map_norm_layer( + norm_layer, + norm_src_key, + dst_prefix=f"{dst_prefix}{cfg.input_layernorm}", + is_norm_layer=is_norm_layer + ) + + if has_mlp: + if isinstance(module.mlp, MoELayer): + map_mlp_func = self._map_moe_layer + norm_layer = module.pre_mlp_layernorm + norm_src_key = f"{src_prefix}pre_mlp_layernorm." + is_norm_layer = True + else: + map_mlp_func = self._map_mlp + norm_layer = module.mlp.linear_fc1 + norm_src_key = f"{src_prefix}mlp.linear_fc1." + is_norm_layer = False + map_mlp_func( + module.mlp, + cfg=cfg.mlp_cfg, + src_prefix=f"{src_prefix}mlp.", + dst_prefix=f"{dst_prefix}{cfg.mlp}", + ) + self._map_norm_layer( + norm_layer, + norm_src_key, + dst_prefix=f"{dst_prefix}{cfg.pre_mlp_layernorm}", + is_norm_layer=is_norm_layer + ) + + def _map_mamba_layer(self, module, src_prefix='', dst_prefix=''): + # NOTE: the API is experimental as MambaLayer is not general enough currently self._map_norm_layer( - norm_layer, - norm_src_key, - dst_prefix=f"{dst_prefix}{cfg.input_layernorm}", - is_norm_layer=is_norm_layer + module.mixer.in_proj, + f"{src_prefix}mixer.in_proj.", + dst_prefix=f"{dst_prefix}input_layernorm.", + is_norm_layer=False + ) + self._map_mamba_mixer( + module.mixer, + src_prefix=f"{src_prefix}mixer.", + dst_prefix=f"{dst_prefix}linear_attn.", ) - if isinstance(module.mlp, MoELayer): - map_mlp_func = self._map_moe_layer - norm_layer = module.pre_mlp_layernorm - norm_src_key = f"{src_prefix}pre_mlp_layernorm." - is_norm_layer = True - else: - map_mlp_func = self._map_mlp - norm_layer = module.mlp.linear_fc1 - norm_src_key = f"{src_prefix}mlp.linear_fc1." - is_norm_layer = False - map_mlp_func( - module.mlp, - cfg=cfg.mlp_cfg, - src_prefix=f"{src_prefix}mlp.", - dst_prefix=f"{dst_prefix}{cfg.mlp}", + def _map_mamba_mixer(self, module, src_prefix='', dst_prefix=''): + Nk, Nv, Dk, Dv = ( + module.ngroups, + module.nheads, + module.d_state, + module.headdim + ) + + # in_proj + src_layout = [ + ('z', Dv * Nv), + ('v', Dv * Nv), + ('q', Dk * Nk), + ('k', Dk * Nk), + ('b', Nv), + ('a', Nv) + ] + self._inner_map_for_merged_linear( + f"{src_prefix}in_proj.weight", + f"{dst_prefix}in_proj_qkvz.weight", + src_layout=src_layout, + required_layout=['q', 'k', 'v', 'z'] + ) + self._inner_map_for_merged_linear( + f"{src_prefix}in_proj.weight", + f"{dst_prefix}in_proj_ba.weight", + src_layout=src_layout, + required_layout=['b', 'a'] + ) + # conv1d + src_layout = [ + ('conv_v', Dv * Nv), + ('conv_q', Dk * Nk), + ('conv_k', Dk * Nk), + ] + self._inner_map_for_merged_linear( + f"{src_prefix}conv1d.weight", + f"{dst_prefix}conv1d.weight", + src_layout=src_layout, + required_layout=['conv_q', 'conv_k', 'conv_v'] ) + self._inner_map_for_tensor_parallel( + f"{src_prefix}dt_bias", + f"{dst_prefix}dt_bias", + mapping_type='column' + ) + + logger.info(f"RANK {mpu.get_pipeline_model_parallel_rank()}: mapping {src_prefix}A_log to {dst_prefix}A_log, data: {module.A_log}") + self._inner_map_for_tensor_parallel( + f"{src_prefix}A_log", + f"{dst_prefix}A_log", + mapping_type='column' + ) + if module.D is not None: + raise NotImplementedError() + self._map_norm_layer( - norm_layer, - norm_src_key, - dst_prefix=f"{dst_prefix}{cfg.pre_mlp_layernorm}", - is_norm_layer=is_norm_layer + module.norm, + f"{src_prefix}norm.", + dst_prefix=f"{dst_prefix}norm.", + is_norm_layer=True + ) + self._inner_map_for_tensor_parallel( + f"{src_prefix}out_proj.weight", + f"{dst_prefix}out_proj.weight", + mapping_type='row' ) def _map_moe_layer(self, module: 'MoELayer', cfg: MoELayerKeyMapping, src_prefix='', dst_prefix=''): @@ -399,14 +525,21 @@ def _map_selfattn( if module.config.add_qkv_bias: param_types = ['weight', 'bias'] + # TODO: make better condition + is_gated_attention = hasattr(module, 'linear_qgkv') for param_type in param_types: for dst_type, dst_name in qkv_dst_names.items(): + if is_gated_attention: + src_key = f"{src_prefix}linear_qgkv.{param_type}" + else: + src_key = f"{src_prefix}linear_qkv.{param_type}" self._inner_map_for_qkv_proj( - f"{src_prefix}linear_qkv.{param_type}", + src_key, f"{dst_prefix}{dst_name}{param_type}", proj_type=dst_type, num_attention_heads = module.config.num_attention_heads, - num_query_groups = module.config.num_query_groups + num_query_groups = module.config.num_query_groups, + is_gated_attention = is_gated_attention ) param_types = ['weight'] diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py index f2d35f8b..8945119e 100644 --- a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -77,17 +77,7 @@ def _map_model(self): ) for vp_stage, model in enumerate(self.model): - if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: - layer_offset = get_transformer_layer_offset(model.config, vp_stage=vp_stage) - else: - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) - layer_offset = get_transformer_layer_offset(model.config) - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(None) - - # TODO: VLM model does not have mtp_process, fix it in Pai-Megatron-Patch - if getattr(model, 'mtp_process', None): + if getattr(model, 'mtp_process', False): raise NotImplementedError("Currently, the mapper does not support MTP") if hasattr(model, 'vision_model'): @@ -102,7 +92,10 @@ def _map_model(self): self._map_llm_model( model.language_model, cfg=cfg, - layer_offset=layer_offset, + index_mapping=self._build_layer_index_mapping( + model.language_model.decoder, + vp_stage + ), src_prefix=f"{vp_stage}-language_model.", dst_prefix="" ) @@ -133,7 +126,7 @@ def _map_vision_model(self, pre_mlp_layernorm='norm2.' ) for layer_idx in range(model.config.num_layers): - self._map_decoder_layer( + self._map_transformer_layer( model.decoder.layers[layer_idx], decoder_layer_cfg, src_prefix=f"{src_prefix}decoder.layers.{layer_idx}.", diff --git a/chatlearn/utils/mappings/megatron_helpers.py b/chatlearn/utils/mappings/megatron_helpers.py index c81b3547..e9dd8758 100644 --- a/chatlearn/utils/mappings/megatron_helpers.py +++ b/chatlearn/utils/mappings/megatron_helpers.py @@ -36,6 +36,8 @@ VocabParallelEmbedding, ColumnParallelLinear ) + from megatron.core.transformer.moe.shared_experts import SharedExpertMLP + from megatron.core.ssm.mamba_mixer import MambaMixer HAVE_MEGATRON = True except ImportError: HAVE_MEGATRON = False @@ -181,6 +183,47 @@ def _prepare_metadata(prefix: str, module: nn.Module): results['weight'] = ShardedTensorInfo.from_global_shape( tuple(module.weight.shape), dtype=module.weight.dtype ) + elif isinstance(module, MambaMixer): + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + L = module.dt_bias.shape[0] + results['dt_bias'] = ShardedTensorInfo( + dtype=module.dt_bias.dtype, + global_shape=(L * tp_size, ), + axis_fragmentations=(tp_size, ), + global_offset=(tp_rank,) + ) + results['A_log'] = ShardedTensorInfo( + dtype=module.A_log.dtype, + global_shape=(L * tp_size, ), + axis_fragmentations=(tp_size, ), + global_offset=(tp_rank,) + ) + if module.D is not None: + raise NotImplementedError() + if module.rmsnorm: + results['norm.weight'] = ShardedTensorInfo( + dtype=module.norm.weight.dtype, + global_shape=(module.norm.weight.shape[0] * tp_size, ), + axis_fragmentations=(tp_size, ), + global_offset=(tp_rank,) + ) + conv_dim, _, d_conv = module.conv1d.weight.shape + results['conv1d.weight'] = ShardedTensorInfo( + dtype=module.conv1d.weight.dtype, + global_shape=(conv_dim * tp_size, 1, d_conv), + axis_fragmentations=(tp_size, 1, 1), + global_offset=(tp_rank, 0, 0) + ) + elif isinstance(module, SharedExpertMLP): + if module.use_shared_expert_gate: + results['gate_weight'] = ShardedTensorInfo( + dtype=module.gate_weight.dtype, + global_shape=(1, module.gate_weight.shape[1]), + axis_fragmentations=(1, 1), + global_offset=(0, 0) + ) + return results def build_sharded_info_for_mcore_model( diff --git a/chatlearn/utils/mappings/sharded_tensor_info.py b/chatlearn/utils/mappings/sharded_tensor_info.py index ddf0fd93..0f9b4173 100644 --- a/chatlearn/utils/mappings/sharded_tensor_info.py +++ b/chatlearn/utils/mappings/sharded_tensor_info.py @@ -329,3 +329,35 @@ def __contains__(self, other: 'ShardedTensorInfo'): if si > oi or si + sj < oi + oj: return False return True + + def chunk(self, sections: List[int], axis: int=0) -> List['ShardedTensorInfo']: + """ + Chunk the sharded info on the given axis. + + Args: + sections (List[int]): a list of length for chunking, the total length of this + list should be equal to local_shape on the given axis. + axis (int, optional): The axis to be chunked. Defaults to 0. + """ + local_size = self.local_shape[axis] + assert local_size == sum(sections), f"Failed to chunk {self} on axis {axis}, given sections {sections}" + offset = self.local_offset[axis] + + chunks = [] + for section in sections: + result = self.copy() + result.local_shape = result.local_shape[:axis] + (section, ) + result.local_shape[axis + 1:] + result.local_offset = result.local_offset[:axis] + (offset, ) + result.local_offset[axis + 1:] + offset += section + chunks.append(result) + return chunks + + @property + def offset(self): + """Return the offset of this shard in the global tensor""" + return tuple(l + g * s // a for l, g, s, a in zip( + self.local_offset, + self.global_offset, + self.global_shape, + self.axis_fragmentations + )) \ No newline at end of file diff --git a/chatlearn/utils/megatron_utils.py b/chatlearn/utils/megatron_utils.py index c465d08d..b5a76b5d 100644 --- a/chatlearn/utils/megatron_utils.py +++ b/chatlearn/utils/megatron_utils.py @@ -21,6 +21,9 @@ def update_cfg(cfg): hf_transformer_config = AutoConfig.from_pretrained(cfg.models.policy.load) + if hf_transformer_config.architectures[0] == "Qwen3NextForCausalLM": + return update_qwen3_next_cfg(cfg, hf_transformer_config) + # common cfgs cfg.models.policy_trainer.megatron_model_cfg.attention_dropout = hf_transformer_config.attention_dropout cfg.models.policy_trainer.megatron_model_cfg.num_layers = hf_transformer_config.num_hidden_layers @@ -107,3 +110,46 @@ def update_cfg(cfg): cfg.models.ref_policy.megatron_model_cfg = cfg.models.policy_trainer.megatron_model_cfg return cfg + +def update_qwen3_next_cfg(cfg, hf_transformer_config): + cfg.models.policy_trainer.megatron_model_cfg.attention_dropout = hf_transformer_config.attention_dropout + cfg.models.policy_trainer.megatron_model_cfg.num_layers = hf_transformer_config.num_hidden_layers * 2 + + full_attention_interval = hf_transformer_config.full_attention_interval + hybrid_pattern = ['*-' if (i + 1) % full_attention_interval == 0 else 'M-' for i in range(hf_transformer_config.num_hidden_layers)] + cfg.models.policy_trainer.megatron_model_cfg.hybrid_override_pattern = ''.join(hybrid_pattern) + + cfg.models.policy_trainer.megatron_model_cfg.is_hybrid_model = True + + cfg.models.policy_trainer.megatron_model_cfg.hidden_size = hf_transformer_config.hidden_size + cfg.models.policy_trainer.megatron_model_cfg.num_attention_heads = hf_transformer_config.num_attention_heads + cfg.models.policy_trainer.megatron_model_cfg.ffn_hidden_size = hf_transformer_config.intermediate_size + cfg.models.policy_trainer.megatron_model_cfg.max_position_embeddings = hf_transformer_config.max_position_embeddings + cfg.models.policy_trainer.megatron_model_cfg.add_bias_linear = False + cfg.models.policy_trainer.megatron_model_cfg.rotary_base = hf_transformer_config.rope_theta + cfg.models.policy_trainer.megatron_model_cfg.rotary_percent = hf_transformer_config.partial_rotary_factor + cfg.models.policy_trainer.megatron_model_cfg.norm_epsilon = hf_transformer_config.rms_norm_eps + cfg.models.policy_trainer.megatron_model_cfg.untie_embeddings_and_output_weights = not hf_transformer_config.tie_word_embeddings + cfg.models.policy_trainer.megatron_model_cfg.vocab_size = hf_transformer_config.vocab_size + cfg.models.policy_trainer.megatron_model_cfg.qk_layernorm = True + + cfg.models.policy_trainer.megatron_model_cfg.kv_channels = hf_transformer_config.head_dim + cfg.models.policy_trainer.megatron_model_cfg.add_qkv_bias = False + + cfg.models.policy_trainer.megatron_model_cfg.moe_shared_expert_intermediate_size = hf_transformer_config.shared_expert_intermediate_size + + cfg.models.policy_trainer.megatron_model_cfg.group_query_attention = True + cfg.models.policy_trainer.megatron_model_cfg.num_query_groups = hf_transformer_config.num_key_value_heads + + cfg.models.policy_trainer.megatron_model_cfg.moe_grouped_gemm = True + cfg.models.policy_trainer.megatron_model_cfg.moe_token_dispatcher_type = "alltoall" + cfg.models.policy_trainer.megatron_model_cfg.moe_router_topk = hf_transformer_config.num_experts_per_tok + cfg.models.policy_trainer.megatron_model_cfg.moe_ffn_hidden_size = hf_transformer_config.moe_intermediate_size + cfg.models.policy_trainer.megatron_model_cfg.moe_router_dtype= 'fp32' + cfg.models.policy_trainer.megatron_model_cfg.num_experts = hf_transformer_config.num_experts + + + cfg.models.policy_trainer.megatron_model_cfg.apply_layernorm_1p = True + + cfg.models.ref_policy.megatron_model_cfg = cfg.models.policy_trainer.megatron_model_cfg + return cfg \ No newline at end of file diff --git a/chatlearn/utils/utils.py b/chatlearn/utils/utils.py index 1a22152c..7cdacccd 100644 --- a/chatlearn/utils/utils.py +++ b/chatlearn/utils/utils.py @@ -326,7 +326,7 @@ def even_slice(total_sample:int, total_slice:int): slice_index.append(total_sample) return slice_index -def slice_data_list_by_index(batched_input: List[Dict[str, Any]], index): +def slice_data_list_by_index(batched_input: List[Any], index): """ Slice input data_list by slice index """ diff --git a/scripts/fsdp_sglang/train_fsdp_sglang_qwen3_next_small.sh b/scripts/fsdp_sglang/train_fsdp_sglang_qwen3_next_small.sh new file mode 100644 index 00000000..b069a829 --- /dev/null +++ b/scripts/fsdp_sglang/train_fsdp_sglang_qwen3_next_small.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -x + +export CHATLEARN=$(pwd) +export PYTHONPATH=${CHATLEARN}:${PYTHONPATH} +source scripts/base_env.sh +export RAY_DEDUP_LOGS=1 + +export DEBUG_SYNC_PARAMETERS_PATH="/mnt/data/gushen.hkw/logs/qwen3_next_small_cvt_fsdp_async" + +export exp_name=qwen_next_response8192_0926 +python chatlearn/entrypoint.py grpo \ + --config-file template/grpo_fsdp.yaml \ + runtime_args.exp_name=${exp_name} \ + runtime_args.rollout_backend=sglang \ + runtime_args.data_path=${CHATLEARN}/dataset/MATH-lighteval/train.json \ + runtime_args.eval_data_path=${CHATLEARN}/dataset/MATH-lighteval/test.json \ + runtime_args.output_dir=${CHATLEARN}/output/${exp_name} \ + runtime_args.num_episode=200 \ + runtime_args.sample_per_episode=2048 \ + runtime_args.train_global_batch_size=2048 \ + runtime_args.train_micro_batch_size=64 \ + runtime_args.save_episode_interval=50 \ + runtime_args.eval_episode_interval=5 \ + runtime_args.enable_eval_before_training=False \ + runtime_args.log_args_dict.enable_wandb=False \ + models.policy_trainer.num_gpu=${num_device} \ + models.policy_trainer.packing=True \ + models.policy_trainer.max_token_in_packing=8192 \ + models.policy_trainer.meta_init=True \ + models.policy_trainer.groupgemm=True \ + models.policy_trainer.generation_batch_size=64 \ + models.policy_trainer.ulysses_sequence_parallel_size=1 \ + models.policy_trainer.load=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-small \ + models.policy_trainer.save_hf=False \ + models.policy_trainer.optimizer.lr=2e-6 \ + models.policy_trainer.pos_clip_ratio=0.2 \ + models.policy_trainer.neg_clip_ratio=0.2 \ + models.ref_policy.generation_batch_size=64 \ + models.policy.generation_batch_size=128 \ + models.policy.enforce_eager=False \ + models.policy.is_sync_mode=False \ + models.policy.tensor_model_parallel_size=4 \ + models.policy.max_prompt_tokens_length=1024 \ + models.policy.max_response_tokens_length=8192 \ + models.policy.num_inference_per_prompt=32 \ + models.policy.gpu_memory_utilization=0.85 \ + models.policy.enable_thinking=False \ + models.reward.generation_batch_size=256 \ + 2>&1 | tee log_${exp_name}.log ; exit ${PIPESTATUS[0]} \ No newline at end of file diff --git a/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh new file mode 100644 index 00000000..95f4ef1f --- /dev/null +++ b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -x + +# Tested on 8xH20-3e with 140G VRAM +export RAY_CGRAPH_get_timeout=200 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export RAY_DEDUP_LOGS=0 +export VLLM_USE_RAY_SPMD_WORKER=1 +export VLLM_USE_RAY_COMPILED_DAG=1 + +export CHATLEARN=$(pwd) +export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250908 +export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch:${CHATLEARN}/../Pai-Megatron-Patch/examples +export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATCH_PATH}:${MEGATRON_PATH}:${PYTHONPATH} +source scripts/base_env.sh + +hf_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct +mcore_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-to-mcore + +exp_name="test_qwen3_next" +export output_dir=${CHATLEARN}/output/${exp_name} +mkdir -p $output_dir/ +export log_dir=${output_dir}/logs +mkdir -p $log_dir +log_file=$log_dir/${exp_name}_rank${RANK}.log + +python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ + runtime_args.exp_name=${exp_name} \ + runtime_args.log_args_dict.enable_tensorboard=True \ + runtime_args.train_backend=megatron \ + runtime_args.rollout_backend=sglang \ + runtime_args.data_path=${CHATLEARN}/dataset/MATH-lighteval/train.json \ + runtime_args.eval_data_path=${CHATLEARN}/dataset/MATH-lighteval/test.json \ + runtime_args.output_dir=${CHATLEARN}/output/${exp_name} \ + runtime_args.num_episode=50 \ + runtime_args.sample_per_episode=64 \ + runtime_args.train_global_batch_size=64 \ + runtime_args.train_micro_batch_size=1 \ + runtime_args.save_episode_interval=1000000 \ + runtime_args.log_args_dict.enable_tensorboard=True \ + runtime_args.log_args_dict.tensorboard_dir=${output_dir}/tensorboard \ + runtime_args.eval_episode_interval=1 \ + runtime_args.enable_eval_before_training=False \ + models.policy_trainer.model_provider_module=qwen3_next.pretrain_qwen3_next \ + models.policy_trainer.num_gpu=${num_device} \ + models.policy_trainer.packing=False \ + models.policy_trainer.max_token_in_packing=4096 \ + models.policy_trainer.bf16=True \ + models.policy_trainer.sequence_parallel=True \ + models.policy_trainer.use_distributed_optimizer=True \ + models.policy_trainer.recompute_granularity=null \ + models.policy_trainer.tensor_model_parallel_size=2 \ + models.policy_trainer.pipeline_model_parallel_size=2 \ + models.policy_trainer.expert_tensor_parallel_size=1 \ + models.policy_trainer.expert_model_parallel_size=2 \ + models.policy_trainer.generation_batch_size=32 \ + models.policy_trainer.load=${mcore_ckpt_path} \ + models.policy_trainer.optimizer.lr=2e-6 \ + models.policy_trainer.optimizer.min_lr=2e-6 \ + models.policy_trainer.pos_clip_ratio=0.2 \ + models.policy_trainer.neg_clip_ratio=0.2 \ + models.reward.generation_batch_size=8 \ + models.policy.load=${hf_ckpt_path} \ + models.policy.generation_batch_size=16 \ + models.policy.tensor_model_parallel_size=2 \ + models.policy.max_prompt_tokens_length=1024 \ + models.policy.max_response_tokens_length=2048 \ + models.policy.num_inference_per_prompt=32 \ + models.policy.gpu_memory_utilization=0.75 \ + models.policy.enable_thinking=False \ + 2>&1 | tee ${log_file} ; exit ${PIPESTATUS[0]} \ No newline at end of file diff --git a/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh new file mode 100644 index 00000000..b856d0d7 --- /dev/null +++ b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -x + +# Tested on 8xH20-3e with 140G VRAM +export RAY_CGRAPH_get_timeout=200 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export RAY_DEDUP_LOGS=0 +export VLLM_USE_RAY_SPMD_WORKER=1 +export VLLM_USE_RAY_COMPILED_DAG=1 + +export CHATLEARN=$(pwd) +export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250908 +export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch:${CHATLEARN}/../Pai-Megatron-Patch/examples +export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATCH_PATH}:${MEGATRON_PATH}:${PYTHONPATH} +source scripts/base_env.sh + +hf_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-small +mcore_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-small-to-mcore + +export DEBUG_SYNC_PARAMETERS_PATH="/mnt/data/gushen.hkw/logs/qwen3_next_small_cvt" + +exp_name="test_qwen3_next" +export output_dir=${CHATLEARN}/output/${exp_name} +mkdir -p $output_dir/ +export log_dir=${output_dir}/logs +mkdir -p $log_dir +log_file=$log_dir/${exp_name}_rank${RANK}.log + +python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ + runtime_args.exp_name=${exp_name} \ + runtime_args.log_args_dict.enable_tensorboard=True \ + runtime_args.train_backend=megatron \ + runtime_args.rollout_backend=sglang \ + runtime_args.data_path=${CHATLEARN}/dataset/MATH-lighteval/train.json \ + runtime_args.eval_data_path=${CHATLEARN}/dataset/MATH-lighteval/test.json \ + runtime_args.output_dir=${CHATLEARN}/output/${exp_name} \ + runtime_args.num_episode=50 \ + runtime_args.sample_per_episode=64 \ + runtime_args.train_global_batch_size=64 \ + runtime_args.train_micro_batch_size=1 \ + runtime_args.save_episode_interval=1000000 \ + runtime_args.log_args_dict.enable_tensorboard=True \ + runtime_args.log_args_dict.tensorboard_dir=${output_dir}/tensorboard \ + runtime_args.eval_episode_interval=1 \ + runtime_args.enable_eval_before_training=False \ + models.policy_trainer.model_provider_module=qwen3_next.pretrain_qwen3_next \ + models.policy_trainer.num_gpu=${num_device} \ + models.policy_trainer.packing=False \ + models.policy_trainer.max_token_in_packing=4096 \ + models.policy_trainer.bf16=True \ + models.policy_trainer.sequence_parallel=True \ + models.policy_trainer.use_distributed_optimizer=True \ + models.policy_trainer.recompute_granularity=null \ + models.policy_trainer.tensor_model_parallel_size=2 \ + models.policy_trainer.pipeline_model_parallel_size=2 \ + models.policy_trainer.expert_tensor_parallel_size=1 \ + models.policy_trainer.expert_model_parallel_size=2 \ + models.policy_trainer.generation_batch_size=32 \ + models.policy_trainer.load=${mcore_ckpt_path} \ + models.policy_trainer.optimizer.lr=2e-6 \ + models.policy_trainer.optimizer.min_lr=2e-6 \ + models.policy_trainer.pos_clip_ratio=0.2 \ + models.policy_trainer.neg_clip_ratio=0.2 \ + models.reward.generation_batch_size=8 \ + models.policy.load=${hf_ckpt_path} \ + models.policy.generation_batch_size=16 \ + models.policy.tensor_model_parallel_size=4 \ + models.policy.max_prompt_tokens_length=1024 \ + models.policy.max_response_tokens_length=2048 \ + models.policy.num_inference_per_prompt=32 \ + models.policy.gpu_memory_utilization=0.75 \ + models.policy.enable_thinking=False \ + 2>&1 | tee ${log_file} ; exit ${PIPESTATUS[0]} \ No newline at end of file diff --git a/tests/parameter_sync/test_mapper_helpers.py b/tests/parameter_sync/test_mapper_helpers.py index d0b878bf..3a46e3c5 100644 --- a/tests/parameter_sync/test_mapper_helpers.py +++ b/tests/parameter_sync/test_mapper_helpers.py @@ -129,7 +129,8 @@ def test_process_gate_up_tensor(): # Case 1: src tp == dst tp assert process_gate_up_tensor( ShardedTensorInfo(axis_fragmentations=(4, 1), global_shape=(16, 8), global_offset=(0, 0)), - 4 + 4, + proj_type='gate_up_proj' ) == [ ( ShardedTensorInfo(axis_fragmentations=(4, 1), global_shape=(16, 8), global_offset=(0, 0)), @@ -140,7 +141,8 @@ def test_process_gate_up_tensor(): # Case 2: src tp < dst tp assert sorted(process_gate_up_tensor( ShardedTensorInfo(axis_fragmentations=(2, ), global_shape=(16, ), global_offset=(1, )), - 4 + 4, + proj_type='gate_up_proj' ), key=lambda x: x[0].local_offset[0]) == [ ( ShardedTensorInfo(axis_fragmentations=(2, ), global_shape=(16, ), global_offset=(1, ), local_shape=(2, ), local_offset=(0, )), @@ -164,6 +166,7 @@ def test_process_gate_up_tensor(): assert sorted(process_gate_up_tensor( ShardedTensorInfo(axis_fragmentations=(8, ), global_shape=(16, ), global_offset=(3, )), 2, + proj_type='gate_up_proj' ), key=lambda x: x[0].local_offset[0]) == [ ( ShardedTensorInfo(axis_fragmentations=(8, ), global_shape=(16, ), global_offset=(3, ), local_shape=(1, ), local_offset=(0, )), @@ -179,6 +182,7 @@ def test_process_gate_up_tensor(): assert sorted(process_gate_up_tensor( ShardedTensorInfo(axis_fragmentations=(3, 1), global_shape=(24, 7), global_offset=(1, 0)), 2, + proj_type='gate_up_proj' ), key=lambda x: x[0].local_offset[0]) == [ ( ShardedTensorInfo(axis_fragmentations=(3, 1), global_shape=(24, 7), global_offset=(1, 0), local_shape=(2, 7), local_offset=(0, 0)), @@ -202,6 +206,7 @@ def test_process_gate_up_tensor(): assert sorted(process_gate_up_tensor( ShardedTensorInfo(axis_fragmentations=(3, ), global_shape=(48, ), global_offset=(1, )), 8, + proj_type='gate_up_proj' ), key=lambda x: x[0].local_offset[0]) == [ ( ShardedTensorInfo(axis_fragmentations=(3, ), global_shape=(48, ), global_offset=(1, ), local_offset=(0, ), local_shape=(1, )), @@ -242,7 +247,8 @@ def test_process_qkv_tensor_no_gqa(): ShardedTensorInfo(axis_fragmentations=(4, 1), global_shape=(96, 8), global_offset=(0, 0)), 8, None, - 4 + 4, + proj_type='qkv_proj' ) == [ ( ShardedTensorInfo(axis_fragmentations=(4, 1), global_shape=(96, 8), global_offset=(0, 0), local_shape=(4, 8), local_offset=(0, 0)), @@ -275,7 +281,8 @@ def test_process_qkv_tensor_no_gqa(): ShardedTensorInfo(axis_fragmentations=(2, ), global_shape=(96, ), global_offset=(1, )), 8, None, - 4 + 4, + proj_type='qkv_proj' ), key=lambda x: x[0].local_offset[0]) == [ ( ShardedTensorInfo(axis_fragmentations=(2, ), global_shape=(96, ), global_offset=(1, ), local_shape=(4, ), local_offset=(0, )), @@ -333,6 +340,7 @@ def test_process_qkv_tensor_no_gqa(): 8, None, 2, + proj_type='qkv_proj' ), key=lambda x: x[0].local_offset[0]) == [ ( ShardedTensorInfo(axis_fragmentations=(8, ), global_shape=(48, ), global_offset=(3, ), local_shape=(2, ), local_offset=(0, )), @@ -355,6 +363,7 @@ def test_process_qkv_tensor_no_gqa(): 12, None, 2, + proj_type='qkv_proj' ), key=lambda x: x[0].local_offset[0] ) @@ -382,6 +391,7 @@ def test_process_qkv_tensor_no_gqa(): 12, None, 4, + proj_type='qkv_proj' ), key=lambda x: x[0].local_offset[0] ) From 0fb9e2bab99ecc3cc0f10902f392a50a90cff76e Mon Sep 17 00:00:00 2001 From: Peng Li Date: Tue, 14 Oct 2025 11:36:36 +0800 Subject: [PATCH 17/28] add how to build image for qwen3-next --- .../tutorial_grpo_mcore_qwen3_next.md | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md diff --git a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md new file mode 100644 index 00000000..0f35a9b5 --- /dev/null +++ b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md @@ -0,0 +1,101 @@ +# 基于 Mcore 的端到端GRPO训练流程 + +本文档提供使用 ChatLearn、Mcore 和 SGLANG 框架来对Qwen3-next进行GRPO训练的快速开始指南。 + +## 开发环境配置 +建议在PAI平台DSW环境中基于nvcr.io/nvidia/pytorch:24.12-py3来构建镜像。 +```bash + +#安装SGLAN，注意这将移除NGC自带的Pytorch，而自动重新安装pytorch==2.8.0 +pip install --no-cache-dir "sglang[all]==0.5.2" -i https://mirrors.aliyun.com/pypi/simple/ + +#安装最新版的Transformers，这将包含最新的qwen3-next的transfomrers实现 +pip install git+https://github.com/huggingface/transformers.git@5f6e278a5177d8b85945a2cdb6b776dacee34914 -i https://mirrors.aliyun.com/pypi/simple/ + + +#安装Chatlearn的依赖包 +pip install modelscope==1.30.0 tensordict==0.10.0 torchdata==0.11.0 codetiming==1.4.0 blobfile==3.0.0 numpy==1.26.4 accelerate==1.10.0 wandb==0.19.11 datasets==3.6.0 grpcio==1.71.0 omegaconf==2.3.0 hydra-core==1.3.2 msgspec==0.19.0 mathruler==0.1.0 pylatexenc==2.10 langgraph==0.6.6 ray[default]==2.46.0 -i https://mirrors.aliyun.com/pypi/simple/ + +#由于安装VLLM会重新安装pytorch，因此需要重新安装flash attention以及apex +pip uninstall -y flash_attn && pip install https://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/csrc/flash-attention/torch2.6.0-cu12x/flash_attn-2.4.2-cp312-cp312-linux_x86_64.whl --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ + +pip uninstall -y apex && pip install https://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/csrc/apex/torch2.6.0-cuda12x/apex-0.1-cp312-cp312-linux_x86_64.whl --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ + + + +#升级Transformer Engine +pip uninstall -y transformer-engine transformer-engine-cu12 transformer-engine-torch +git clone --recursive https://github.com/NVIDIA/TransformerEngine.git +cd TransformerEngine +git submodule update --init --recursive +git checkout release_v2.7 +export CUDNN_PATH=/usr/local/lib/python3.12/dist-packages/nvidia/cudnn/ +cp /usr/local/lib/python3.12/dist-packages/nvidia/cudnn/include/* /usr/local/cuda/include/ +python setup.py bdist_wheel -vvv +cd dist +export NVTE_FRAMEWORK=pytorch +pip install transformer_engine-2.7.0-cp312-cp312-linux_x86_64.whl --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.cloud.aliyuncs.com + +#安装mamba-ssm依赖 +pip install --no-build-isolation "mamba-ssm" -i https://mirrors.aliyun.com/pypi/simple/ + +#安装causal-conv1d依赖 +git clone https://github.com/Dao-AILab/causal-conv1d.git +cd causal-conv1d +git checkout v1.5.2 +export CAUSAL_CONV1D_FORCE_BUILD=TRUE +python setup.py bdist_wheel -vvv +cd dist +export NVTE_FRAMEWORK=pytorch +pip install causal_conv1d-1.5.2-cp312-cp312-linux_x86_64.whl --no-cache-dir --no-build-isolation -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.cloud.aliyuncs.com + +# 安装flash-linear-attention +pip install --no-build-isolation flash-linear-attention -i https://mirrors.aliyun.com/pypi/simple/ + +``` +## 代码准备 + +```bash +git clone https://github.com/alibaba/ChatLearn.git +git clone --recurse-submodules https://github.com/alibaba/Pai-Megatron-Patch.git +``` + +## 数据&模型准备 +以[MATH-lighteval](https://www.modelscope.cn/datasets/AI-ModelScope/MATH-lighteval)数据集作为示例. +```bash +cd ChatLearn +# 下载数据集 +mkdir -p dataset +modelscope download --dataset AI-ModelScope/MATH-lighteval --local_dir dataset/MATH-lighteval +# preprocess dataset +python chatlearn/data/data_preprocess/math_lighteval.py --input_dir dataset/MATH-lighteval --local_dir dataset/MATH-lighteval +# download model weight +modelscope download --model Qwen/Qwen3-Next-80B-A3B-Instruct --local_dir Qwen3-Next-80B-A3B-Instruct + +``` + +## 模型转换 +使用下述脚本将Moonlight和DeepSeek-V3的Huggingface格式的模型转换到MCore格式 +```bash +CHATLEARN_ROOT=$(pwd) +cd ../Pai-Megatron-Patch/toolkits/distributed_checkpoints_convertor +bash scripts/qwen3_next/run_8xH20.sh \ +A3B \ +${CHATLEARN_ROOT}/pretrained_models/Qwen3-Next-80B-A3B-Instruct \ +${CHATLEARN_ROOT}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-to-mcore \ +false \ +true \ +bf16 + +``` + +## Qwen3-Next强化学习训练以及训练稳定性指引 +运行以下命令可以对Qwen3-Next进行GRPO训练： + +```bash +cd ${CHATLEARN_ROOT} +bash scripts/mcore_vllm/train_mcore_vllm_qwen3_next_grpo.sh +``` + +## 使用 Wandb 监控 +如需使用 Wandb 记录训练过程，请参考其他最佳实践进行修改。 From 900dcac0032d36c67cc0834c4028867ed0d518e5 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Tue, 14 Oct 2025 11:58:25 +0800 Subject: [PATCH 18/28] fix param_sync --- chatlearn/models/megatron_module.py | 11 ++---- .../mappers/base_megatron_mapper.py | 36 +++++++++++++++++++ .../synchronizer/mappers/mapping_helpers.py | 30 ++++++++++++---- .../mappers/megatron_llm_mapper.py | 11 +++--- chatlearn/synchronizer/parameter_sync.py | 2 +- 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/chatlearn/models/megatron_module.py b/chatlearn/models/megatron_module.py index e8ba6d40..4b947d09 100644 --- a/chatlearn/models/megatron_module.py +++ b/chatlearn/models/megatron_module.py @@ -255,17 +255,10 @@ def map_local_param_name_to_global(self): self.global_name_to_local_name = {} # NOTE: this regex is for model with TEGroupedGEMM # SequentialMLP or GroupedMLP is not supported - regex = re.compile(r"(.*)decoder.layers\.(\d+)\.([a-z0-9_.]+)([\._])([a-z]+)([0-9]*)") + regex = re.compile(r"(.*)decoder.layers\.(\d+)\.([a-zA-Z0-9_.]+)([\._])([a-zA-Z]+)([0-9]*)") for vp_stage, model_chunk in enumerate(self.model): model_config = unwrap_model(model_chunk).config - if 'vp_stage' in inspect.signature(get_transformer_layer_offset).parameters: - offset = get_transformer_layer_offset(model_config, vp_stage=vp_stage) - else: - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(vp_stage) - offset = get_transformer_layer_offset(model_config) - if len(self.model) > 1: - mpu.set_virtual_pipeline_model_parallel_rank(None) + offset = get_transformer_layer_offset(model_config, vp_stage=vp_stage) if model_config.num_moe_experts is not None: ep_rank = mpu.get_expert_model_parallel_rank() ep_size = mpu.get_expert_model_parallel_world_size() diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index be25594e..c945ffe9 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -28,6 +28,7 @@ process_gate_up_tensor, process_qkv_tensor, process_merged_linear_tensor, + process_linear_attn_tensor, VLLM_HELPERS, HF_HELPERS ) @@ -227,6 +228,41 @@ def _inner_map_for_merged_linear( self._update_mapping(mapping) return mapping + def _inner_map_for_linear_attn( + self, + src_key: str, + dst_key: str, + src_layout: List[Tuple[str, int]], + required_layout: List[str], + *, + global_expert_id: int=None, + num_experts: int=None, + axis: int = 0, + n_groups: int = 1 + ): + src_info = self._src_name_to_metadata[src_key] + dst_info = self._dst_name_to_metadata[dst_key] + mapping = {} + for src_meta, dst_meta in process_linear_attn_tensor( + src_info, + self._dst_tp_size, + n_groups=n_groups, + src_layout=src_layout, + required_layout=required_layout, + axis=axis + ): + src_meta.param_id, dst_meta.param_id = src_info.param_id, dst_info.param_id + src_meta.dtype, dst_meta.dtype = src_info.dtype, dst_info.dtype + if global_expert_id is not None: + dst_meta = ( + dst_meta + .unsqueeze(offset=global_expert_id, length=num_experts, axis=0) + .refragment(1, axis=0) # 1 is dst EP + ) + mapping[src_meta] = [dst_meta] + self._update_mapping(mapping) + return mapping + def _update_mapping(self, results: Dict[ShardedTensorInfo, List[ShardedTensorInfo]]) -> None: if self._mapping is None: self._mapping = defaultdict(list) diff --git a/chatlearn/synchronizer/mappers/mapping_helpers.py b/chatlearn/synchronizer/mappers/mapping_helpers.py index 4a495629..bd664c32 100644 --- a/chatlearn/synchronizer/mappers/mapping_helpers.py +++ b/chatlearn/synchronizer/mappers/mapping_helpers.py @@ -151,6 +151,26 @@ def process_merged_linear_tensor( )) return __maybe_merge(results) +def process_linear_attn_tensor( + sharded_info: ShardedTensorInfo, + dst_tp_size: int, + n_groups: int, + src_layout: List[Tuple[str, int]], + required_layout: List[str], + axis: int = 0 +) -> List[Tuple[ShardedTensorInfo, ...]]: + if n_groups % dst_tp_size != 0: + raise ValueError("n_groups of linear attn should be divided by tp!") + results = process_merged_linear_tensor( + sharded_info=sharded_info, + dst_tp_size=n_groups, + src_layout=src_layout, + required_layout=required_layout, + axis=axis + ) + return [(item[0], item[1].refragment(dst_tp_size, axis=axis)) for item in results] + + def _build_qkv_layout( num_heads: int, num_query_group: int, @@ -240,7 +260,7 @@ def process_qkv_tensor( src_tp_size = sharded_info.axis_fragmentations[0] src_global_shape = sharded_info.global_shape - mcore_layout, vllm_layout = _build_qkv_layout(num_heads, num_query_groups, dst_tp_size) + mcore_layout, vllm_layout = _build_qkv_layout(num_heads, num_query_groups, dst_tp_size, is_gated_attention=is_gated_attention) mcore_id_to_frags = { part.global_offset[0]: part.refragment(src_tp_size) for part in sharded_info.fragment(num_query_groups * (2 + num_heads // num_query_groups)) @@ -250,16 +270,14 @@ def process_qkv_tensor( n_heads = num_heads + 2 * num_query_groups * max(1, dst_tp_size // num_query_groups) elif proj_type == 'q_proj': n_heads = num_heads + vllm_layout = [item for item in vllm_layout if 'q' in item or 'g' in item] else: n_heads = num_query_groups * max(1, dst_tp_size // num_query_groups) + vllm_layout = [item for item in vllm_layout if proj_type[:1] in item] full_dst_info = ShardedTensorInfo.from_global_shape((n_heads * head_dim, ) + src_global_shape[1:]) results = [] - for head_idx, dst_part in enumerate(full_dst_info.fragment(n_heads)): - if proj_type == 'qkv_proj': - head_name = vllm_layout[head_idx] - else: - head_name = f"{proj_type[:1]}{head_idx}" # q0 / k1 / v2, etc. + for head_name, dst_part in zip(vllm_layout, full_dst_info.fragment(n_heads)): mcore_idx = mcore_layout.index(head_name) if mcore_idx not in mcore_id_to_frags: continue diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index 413ea1af..495fc2bb 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -293,17 +293,19 @@ def _map_mamba_mixer(self, module, src_prefix='', dst_prefix=''): ('b', Nv), ('a', Nv) ] - self._inner_map_for_merged_linear( + self._inner_map_for_linear_attn( f"{src_prefix}in_proj.weight", f"{dst_prefix}in_proj_qkvz.weight", src_layout=src_layout, - required_layout=['q', 'k', 'v', 'z'] + required_layout=['q', 'k', 'v', 'z'], + n_groups=Nk ) - self._inner_map_for_merged_linear( + self._inner_map_for_linear_attn( f"{src_prefix}in_proj.weight", f"{dst_prefix}in_proj_ba.weight", src_layout=src_layout, - required_layout=['b', 'a'] + required_layout=['b', 'a'], + n_groups=Nk ) # conv1d src_layout = [ @@ -323,7 +325,6 @@ def _map_mamba_mixer(self, module, src_prefix='', dst_prefix=''): mapping_type='column' ) - logger.info(f"RANK {mpu.get_pipeline_model_parallel_rank()}: mapping {src_prefix}A_log to {dst_prefix}A_log, data: {module.A_log}") self._inner_map_for_tensor_parallel( f"{src_prefix}A_log", f"{dst_prefix}A_log", diff --git a/chatlearn/synchronizer/parameter_sync.py b/chatlearn/synchronizer/parameter_sync.py index 65ea0a75..90d2c59c 100644 --- a/chatlearn/synchronizer/parameter_sync.py +++ b/chatlearn/synchronizer/parameter_sync.py @@ -153,7 +153,7 @@ def generate_global_param_ids(self, model: 'DistModel') -> Dict[str, int]: param_names = set() for res in results: param_names.update(res) - return {name: idx for idx, name in enumerate(param_names)} + return {name: idx for idx, name in enumerate(sorted(param_names))} def validate_sync_mapping( self, From 805a0166cf30df5517dabc4d4559c723ae211eb5 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Tue, 14 Oct 2025 14:45:11 +0800 Subject: [PATCH 19/28] Add SGLANG PATCH to README --- .../tutorial_grpo_mcore_qwen3_next.md | 6 +- .../train_mcore_vllm_qwen3_next_grpo.sh | 74 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 scripts/mcore_vllm/train_mcore_vllm_qwen3_next_grpo.sh diff --git a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md index 0f35a9b5..97bf5a34 100644 --- a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md +++ b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md @@ -6,9 +6,13 @@ 建议在PAI平台DSW环境中基于nvcr.io/nvidia/pytorch:24.12-py3来构建镜像。 ```bash -#安装SGLAN，注意这将移除NGC自带的Pytorch，而自动重新安装pytorch==2.8.0 +#安装SGLANG，注意这将移除NGC自带的Pytorch，而自动重新安装pytorch==2.8.0 pip install --no-cache-dir "sglang[all]==0.5.2" -i https://mirrors.aliyun.com/pypi/simple/ +#添加SGLANG PATCH +wget https://gist.github.com/lostkevin/9b668c24de6f0e9974c9ad069ef03ed9 +cp memory_pool.py /usr/local/lib/python3.12/dist-packages/sglang/srt/mem_cache/ + #安装最新版的Transformers，这将包含最新的qwen3-next的transfomrers实现 pip install git+https://github.com/huggingface/transformers.git@5f6e278a5177d8b85945a2cdb6b776dacee34914 -i https://mirrors.aliyun.com/pypi/simple/ diff --git a/scripts/mcore_vllm/train_mcore_vllm_qwen3_next_grpo.sh b/scripts/mcore_vllm/train_mcore_vllm_qwen3_next_grpo.sh new file mode 100644 index 00000000..542d18ab --- /dev/null +++ b/scripts/mcore_vllm/train_mcore_vllm_qwen3_next_grpo.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -x + +export RAY_CGRAPH_get_timeout=200 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export RAY_num_server_call_thread=1 +export RAY_DEDUP_LOGS=0 +export VLLM_USE_RAY_SPMD_WORKER=1 +export VLLM_USE_RAY_COMPILED_DAG=1 + +export CHATLEARN=$(pwd) +export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250908 +export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATH}:${PYTHONPATH} +source scripts/base_env.sh + +hf_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct +mcore_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-to-mcore + + +exp_name="test_qwen3_next_grpo" +export output_dir=${CHATLEARN}/output/${exp_name} +mkdir -p $output_dir/ +export log_dir=${output_dir}/logs +mkdir -p $log_dir +log_file=$log_dir/${exp_name}_rank${RANK}.log + + + +python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ + runtime_args.exp_name=${exp_name} \ + runtime_args.log_args_dict.enable_tensorboard=True \ + runtime_args.train_backend=megatron \ + runtime_args.rollout_backend=sglang \ + runtime_args.data_path=${CHATLEARN}/dataset/MATH-lighteval/train.json \ + runtime_args.eval_data_path=${CHATLEARN}/dataset/MATH-lighteval/test.json \ + runtime_args.output_dir=${CHATLEARN}/output/${exp_name} \ + runtime_args.num_episode=200 \ + runtime_args.sample_per_episode=2048 \ + runtime_args.train_global_batch_size=16 \ + runtime_args.train_micro_batch_size=1 \ + runtime_args.save_episode_interval=1000000 \ + runtime_args.log_args_dict.enable_tensorboard=True \ + runtime_args.log_args_dict.tensorboard_dir=${output_dir}/tensorboard \ + runtime_args.eval_episode_interval=1 \ + runtime_args.enable_eval_before_training=False \ + models.policy_trainer.num_gpu=${num_device} \ + models.policy_trainer.packing=True \ + models.policy_trainer.trust_remote_code=True \ + models.policy_trainer.max_token_in_packing=3072 \ + models.policy_trainer.bf16=True \ + models.policy_trainer.sequence_parallel=True \ + models.policy_trainer.use_distributed_optimizer=True \ + models.policy_trainer.recompute_granularity=null \ + models.policy_trainer.tensor_model_parallel_size=4 \ + models.policy_trainer.pipeline_model_parallel_size=4 \ + models.policy_trainer.expert_tensor_parallel_size=1 \ + models.policy_trainer.expert_model_parallel_size=8 \ + models.policy_trainer.context_parallel_size=1 \ + models.policy_trainer.generation_batch_size=512 \ + models.policy_trainer.load=${mcore_ckpt_path} \ + models.policy_trainer.optimizer.lr=2e-6 \ + models.policy_trainer.optimizer.min_lr=2e-6 \ + models.policy_trainer.pos_clip_ratio=0.2 \ + models.policy_trainer.neg_clip_ratio=0.2 \ + models.reward.generation_batch_size=128 \ + models.policy.load=${hf_ckpt_path} \ + models.policy.generation_batch_size=512 \ + models.policy.tensor_model_parallel_size=2 \ + models.policy.max_prompt_tokens_length=1024 \ + models.policy.max_response_tokens_length=2048 \ + models.policy.num_inference_per_prompt=32 \ + models.policy.gpu_memory_utilization=0.75 \ + models.policy.enable_thinking=False \ + 2>&1 | tee ${log_file} ; exit ${PIPESTATUS[0]} \ No newline at end of file From da34f05cd7a27f2dac72223858f7677c2ee6acde Mon Sep 17 00:00:00 2001 From: Peng Li Date: Wed, 15 Oct 2025 22:32:05 +0800 Subject: [PATCH 20/28] fix readme and scripts --- .../tutorial_grpo_mcore_qwen3_next.md | 8 +- ...rain_mcore_sglang_qwen3_next_small_grpo.sh | 73 ------------------- 2 files changed, 2 insertions(+), 79 deletions(-) delete mode 100644 scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh diff --git a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md index 97bf5a34..c9e783e5 100644 --- a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md +++ b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md @@ -13,12 +13,9 @@ pip install --no-cache-dir "sglang[all]==0.5.2" -i https://mirrors.aliyun.com/p wget https://gist.github.com/lostkevin/9b668c24de6f0e9974c9ad069ef03ed9 cp memory_pool.py /usr/local/lib/python3.12/dist-packages/sglang/srt/mem_cache/ -#安装最新版的Transformers，这将包含最新的qwen3-next的transfomrers实现 -pip install git+https://github.com/huggingface/transformers.git@5f6e278a5177d8b85945a2cdb6b776dacee34914 -i https://mirrors.aliyun.com/pypi/simple/ - #安装Chatlearn的依赖包 -pip install modelscope==1.30.0 tensordict==0.10.0 torchdata==0.11.0 codetiming==1.4.0 blobfile==3.0.0 numpy==1.26.4 accelerate==1.10.0 wandb==0.19.11 datasets==3.6.0 grpcio==1.71.0 omegaconf==2.3.0 hydra-core==1.3.2 msgspec==0.19.0 mathruler==0.1.0 pylatexenc==2.10 langgraph==0.6.6 ray[default]==2.46.0 -i https://mirrors.aliyun.com/pypi/simple/ +pip install transformers==4.57.1 modelscope==1.30.0 tensordict==0.10.0 torchdata==0.11.0 codetiming==1.4.0 blobfile==3.0.0 numpy==1.26.4 accelerate==1.10.0 wandb==0.19.11 datasets==3.6.0 grpcio==1.71.0 omegaconf==2.3.0 hydra-core==1.3.2 msgspec==0.19.0 mathruler==0.1.0 pylatexenc==2.10 langgraph==0.6.6 ray[default]==2.46.0 -i https://mirrors.aliyun.com/pypi/simple/ #由于安装VLLM会重新安装pytorch，因此需要重新安装flash attention以及apex pip uninstall -y flash_attn && pip install https://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/csrc/flash-attention/torch2.6.0-cu12x/flash_attn-2.4.2-cp312-cp312-linux_x86_64.whl --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ @@ -26,7 +23,6 @@ pip uninstall -y flash_attn && pip install https://pai-vision-data-hz.oss-cn-zha pip uninstall -y apex && pip install https://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/csrc/apex/torch2.6.0-cuda12x/apex-0.1-cp312-cp312-linux_x86_64.whl --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ - #升级Transformer Engine pip uninstall -y transformer-engine transformer-engine-cu12 transformer-engine-torch git clone --recursive https://github.com/NVIDIA/TransformerEngine.git @@ -98,7 +94,7 @@ bf16 ```bash cd ${CHATLEARN_ROOT} -bash scripts/mcore_vllm/train_mcore_vllm_qwen3_next_grpo.sh +bash scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh ``` ## 使用 Wandb 监控 diff --git a/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh deleted file mode 100644 index b856d0d7..00000000 --- a/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_small_grpo.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -x - -# Tested on 8xH20-3e with 140G VRAM -export RAY_CGRAPH_get_timeout=200 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export RAY_DEDUP_LOGS=0 -export VLLM_USE_RAY_SPMD_WORKER=1 -export VLLM_USE_RAY_COMPILED_DAG=1 - -export CHATLEARN=$(pwd) -export MEGATRON_PATH=${CHATLEARN}/../Pai-Megatron-Patch/backends/megatron/Megatron-LM-250908 -export MEGATRON_PATCH_PATH=${CHATLEARN}/../Pai-Megatron-Patch:${CHATLEARN}/../Pai-Megatron-Patch/examples -export PYTHONPATH=${CHATLEARN}:${MEGATRON_PATCH_PATH}:${MEGATRON_PATH}:${PYTHONPATH} -source scripts/base_env.sh - -hf_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-small -mcore_ckpt_path=${CHATLEARN}/pretrained_models/Qwen3-Next-80B-A3B-Instruct-small-to-mcore - -export DEBUG_SYNC_PARAMETERS_PATH="/mnt/data/gushen.hkw/logs/qwen3_next_small_cvt" - -exp_name="test_qwen3_next" -export output_dir=${CHATLEARN}/output/${exp_name} -mkdir -p $output_dir/ -export log_dir=${output_dir}/logs -mkdir -p $log_dir -log_file=$log_dir/${exp_name}_rank${RANK}.log - -python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ - runtime_args.exp_name=${exp_name} \ - runtime_args.log_args_dict.enable_tensorboard=True \ - runtime_args.train_backend=megatron \ - runtime_args.rollout_backend=sglang \ - runtime_args.data_path=${CHATLEARN}/dataset/MATH-lighteval/train.json \ - runtime_args.eval_data_path=${CHATLEARN}/dataset/MATH-lighteval/test.json \ - runtime_args.output_dir=${CHATLEARN}/output/${exp_name} \ - runtime_args.num_episode=50 \ - runtime_args.sample_per_episode=64 \ - runtime_args.train_global_batch_size=64 \ - runtime_args.train_micro_batch_size=1 \ - runtime_args.save_episode_interval=1000000 \ - runtime_args.log_args_dict.enable_tensorboard=True \ - runtime_args.log_args_dict.tensorboard_dir=${output_dir}/tensorboard \ - runtime_args.eval_episode_interval=1 \ - runtime_args.enable_eval_before_training=False \ - models.policy_trainer.model_provider_module=qwen3_next.pretrain_qwen3_next \ - models.policy_trainer.num_gpu=${num_device} \ - models.policy_trainer.packing=False \ - models.policy_trainer.max_token_in_packing=4096 \ - models.policy_trainer.bf16=True \ - models.policy_trainer.sequence_parallel=True \ - models.policy_trainer.use_distributed_optimizer=True \ - models.policy_trainer.recompute_granularity=null \ - models.policy_trainer.tensor_model_parallel_size=2 \ - models.policy_trainer.pipeline_model_parallel_size=2 \ - models.policy_trainer.expert_tensor_parallel_size=1 \ - models.policy_trainer.expert_model_parallel_size=2 \ - models.policy_trainer.generation_batch_size=32 \ - models.policy_trainer.load=${mcore_ckpt_path} \ - models.policy_trainer.optimizer.lr=2e-6 \ - models.policy_trainer.optimizer.min_lr=2e-6 \ - models.policy_trainer.pos_clip_ratio=0.2 \ - models.policy_trainer.neg_clip_ratio=0.2 \ - models.reward.generation_batch_size=8 \ - models.policy.load=${hf_ckpt_path} \ - models.policy.generation_batch_size=16 \ - models.policy.tensor_model_parallel_size=4 \ - models.policy.max_prompt_tokens_length=1024 \ - models.policy.max_response_tokens_length=2048 \ - models.policy.num_inference_per_prompt=32 \ - models.policy.gpu_memory_utilization=0.75 \ - models.policy.enable_thinking=False \ - 2>&1 | tee ${log_file} ; exit ${PIPESTATUS[0]} \ No newline at end of file From 6f6f3a8cd7684e19058f2c120496cb7277cc9449 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Thu, 16 Oct 2025 11:31:55 +0800 Subject: [PATCH 21/28] fix memory_pool.py overwrite in readme --- docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md index 97bf5a34..948c6d64 100644 --- a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md +++ b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md @@ -9,8 +9,9 @@ #安装SGLANG，注意这将移除NGC自带的Pytorch，而自动重新安装pytorch==2.8.0 pip install --no-cache-dir "sglang[all]==0.5.2" -i https://mirrors.aliyun.com/pypi/simple/ -#添加SGLANG PATCH -wget https://gist.github.com/lostkevin/9b668c24de6f0e9974c9ad069ef03ed9 +#添加SGLANG PATCH。 从https://gist.github.com/lostkevin/9b668c24de6f0e9974c9ad069ef03ed9下载修改后的memory_pool.py文件 +cd /usr/local/lib/python3.12/dist-packages/sglang/srt/mem_cache/ +mv memory_pool.py memory_pool.py.bak cp memory_pool.py /usr/local/lib/python3.12/dist-packages/sglang/srt/mem_cache/ #安装最新版的Transformers，这将包含最新的qwen3-next的transfomrers实现 From 5509f13576830e29b7ba2187a7e81e7a21a63dd2 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 20 Oct 2025 11:10:11 +0800 Subject: [PATCH 22/28] demo --- chatlearn/configs/megatron_config.py | 8 ++++++++ chatlearn/models/megatron_module.py | 2 ++ .../synchronizer/planners/tensor_planner.py | 4 ++++ chatlearn/utils/megatron_utils.py | 16 +++++++++++++--- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/chatlearn/configs/megatron_config.py b/chatlearn/configs/megatron_config.py index 72f6b59f..c2fc27d5 100644 --- a/chatlearn/configs/megatron_config.py +++ b/chatlearn/configs/megatron_config.py @@ -70,6 +70,7 @@ class MegatronModelArchitectureConfig(BaseConfig): default=1000000, metadata={"help": "Base to use for rotary positional embeddings"}, ) + rotary_percent: float = 1.0 group_query_attention: bool = field( default=False, metadata={"help": "Use group-query attention."} ) @@ -334,6 +335,12 @@ class MegatronConfig(BaseConfig): } ) + use_expandable_segments: bool = field( + default=False, metadata={"help": "Whether to use expandable_segments in PYTORCH_CUDA_ALLOC_CONF, \ + avoid big reseverd memory in ref and policy trainer worker, expandable_segments should be False \ + while in parameter sync for efficiency"} + ) + def _validate_impl(self): assert self.num_gpu > 0, "Megatron-Core requires at least one GPU" assert self.num_gpu % self.num_replica == 0, \ @@ -448,6 +455,7 @@ class MegatronPolicyTrainerConfig(PolicyTrainerConfig, MegatronConfig): "help": "Load model for finetuning. Do not load optimizer or rng state from checkpoint and set iteration to 0." }, ) + distributed_timeout_minutes: int = 10 def _validate_impl(self): assert self.calculate_per_token_loss, "Per-Token-Loss is required for Training." diff --git a/chatlearn/models/megatron_module.py b/chatlearn/models/megatron_module.py index 4b947d09..38394889 100644 --- a/chatlearn/models/megatron_module.py +++ b/chatlearn/models/megatron_module.py @@ -123,6 +123,8 @@ def model_setup(self): """ :meta private: """ + if self.module_args.use_expandable_segments: + torch.cuda.memory._set_allocator_settings("expandable_segments:True") super().model_setup() # TODO: we may need to let setup return model, optimizer and opt_param_scheduler diff --git a/chatlearn/synchronizer/planners/tensor_planner.py b/chatlearn/synchronizer/planners/tensor_planner.py index a9fd879e..08fc62fa 100644 --- a/chatlearn/synchronizer/planners/tensor_planner.py +++ b/chatlearn/synchronizer/planners/tensor_planner.py @@ -115,6 +115,10 @@ def build_iteration( continue is_added.add(dst_param.param_id) dst_param_id_to_src_params[dst_param.param_id].append(src_param) + t = list(dst_param_id_to_src_params.keys()) + import random + random.shuffle(t) + dst_param_id_to_src_params = {k: dst_param_id_to_src_params[k] for k in t} src_shard_to_sender = {} for sender, plan_per_rank in unbucketized_plan.items(): diff --git a/chatlearn/utils/megatron_utils.py b/chatlearn/utils/megatron_utils.py index b5a76b5d..b36f1098 100644 --- a/chatlearn/utils/megatron_utils.py +++ b/chatlearn/utils/megatron_utils.py @@ -145,11 +145,21 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): cfg.models.policy_trainer.megatron_model_cfg.moe_token_dispatcher_type = "alltoall" cfg.models.policy_trainer.megatron_model_cfg.moe_router_topk = hf_transformer_config.num_experts_per_tok cfg.models.policy_trainer.megatron_model_cfg.moe_ffn_hidden_size = hf_transformer_config.moe_intermediate_size - cfg.models.policy_trainer.megatron_model_cfg.moe_router_dtype= 'fp32' + cfg.models.policy_trainer.megatron_model_cfg.moe_router_dtype= 'fp64' cfg.models.policy_trainer.megatron_model_cfg.num_experts = hf_transformer_config.num_experts - - cfg.models.policy_trainer.megatron_model_cfg.apply_layernorm_1p = True + cfg.models.policy_trainer.megatron_model_cfg.moe_router_load_balancing_type = "none" + cfg.models.policy_trainer.megatron_model_cfg.moe_aux_loss_coeff = 0 + cfg.models.policy_trainer.megatron_model_cfg.moe_permute_fusion = True + cfg.models.policy_trainer.megatron_model_cfg.moe_router_fusion = False # try5: True + cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_loss_fusion = True + cfg.models.policy_trainer.megatron_model_cfg.moe_shared_expert_overlap = False + + cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_fusion_impl = 'te' + cfg.models.policy_trainer.megatron_model_cfg.gradient_accumulation_fusion = True # try5: False + # cfg.models.policy_trainer.megatron_model_cfg.async_tensor_model_parallel_allreduce = True + cfg.models.policy_trainer.distributed_timeout_minutes = 60 + cfg.models.ref_policy.megatron_model_cfg = cfg.models.policy_trainer.megatron_model_cfg return cfg \ No newline at end of file From 9cd419a47624569be7837461db8dea406e46b7b2 Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 20 Oct 2025 11:49:50 +0800 Subject: [PATCH 23/28] demo update --- chatlearn/utils/megatron_utils.py | 2 +- template/grpo_megatron.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/chatlearn/utils/megatron_utils.py b/chatlearn/utils/megatron_utils.py index b36f1098..09899889 100644 --- a/chatlearn/utils/megatron_utils.py +++ b/chatlearn/utils/megatron_utils.py @@ -158,7 +158,7 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_fusion_impl = 'te' cfg.models.policy_trainer.megatron_model_cfg.gradient_accumulation_fusion = True # try5: False - # cfg.models.policy_trainer.megatron_model_cfg.async_tensor_model_parallel_allreduce = True + cfg.models.policy_trainer.megatron_model_cfg.async_tensor_model_parallel_allreduce = True cfg.models.policy_trainer.distributed_timeout_minutes = 60 cfg.models.ref_policy.megatron_model_cfg = cfg.models.policy_trainer.megatron_model_cfg diff --git a/template/grpo_megatron.yaml b/template/grpo_megatron.yaml index 1324cefe..5e70319d 100644 --- a/template/grpo_megatron.yaml +++ b/template/grpo_megatron.yaml @@ -35,6 +35,7 @@ runtime_args: wandb_resume: allow models: policy_trainer: + use_expandable_segments: True free_gpu_memory: offload_weights: True offload_optimizer_states: True @@ -81,6 +82,7 @@ models: max_token_in_packing: ${models.policy_trainer.seq_length} model_provider_module: pretrain_gpt ref_policy: + use_expandable_segments: True free_gpu_memory: offload_weights: True generation_batch_size: ${models.policy_trainer.generation_batch_size} From 65f0195acc3b7e85a818ef0146013838fef83df3 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Mon, 20 Oct 2025 16:34:02 +0800 Subject: [PATCH 24/28] fix wandb logging --- .../grpo_utils/megatron_policy_trainer.py | 2 +- .../grpo_utils/megatron_utils/train_helper.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py b/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py index 2defaca7..7f72abff 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py +++ b/chatlearn/algorithm/grpo_utils/megatron_policy_trainer.py @@ -249,7 +249,7 @@ def train_step(self, data_list: List[Dict[str, Any]], **kwargs): num_zeros_in_grad, self.stats, {}, - "policy_trainer", + "", self._metric_list, ) diff --git a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py index 1ec2e042..d99ad5ad 100644 --- a/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py +++ b/chatlearn/algorithm/grpo_utils/megatron_utils/train_helper.py @@ -113,32 +113,32 @@ def training_log( if is_last_rank(): for key in loss_dict: - iter_dict[f"{name}/{key}"] = loss_dict[key] - consumed_train_samples_dict[f"{name}/" + key + " vs samples"] = loss_dict[ + iter_dict[f"{key}"] = loss_dict[key] + consumed_train_samples_dict[key + " vs samples"] = loss_dict[ key ] if grad_norm is not None: - iter_dict[f"{name}/" + "grad_norm"] = grad_norm - consumed_train_samples_dict[f"{name}/" + "grad-norm vs samples"] = grad_norm + iter_dict["grad_norm"] = grad_norm + consumed_train_samples_dict["grad-norm vs samples"] = grad_norm if more_grad_norm is not None: for k in more_grad_norm: - iter_dict[f"{name}/{k}" + " grad_norm"] = more_grad_norm[k] - consumed_train_samples_dict[f"{name}/{k}" + " grad-norm vs samples"] = ( + iter_dict[f"{k}" + " grad_norm"] = more_grad_norm[k] + consumed_train_samples_dict[f"{k}" + " grad-norm vs samples"] = ( more_grad_norm[k] ) if params_norm is not None: - iter_dict[f"{name}/" + "params-norm"] = params_norm - consumed_train_samples_dict[f"{name}/" + "params-norm vs samples"] = ( + iter_dict["params-norm"] = params_norm + consumed_train_samples_dict["params-norm vs samples"] = ( params_norm ) elapsed_time = 0 elapsed_time_per_iteration = elapsed_time / total_iterations if args.log_timers_to_tensorboard: - iter_dict[f"{name}/" + "iteration-time"] = elapsed_time_per_iteration + iter_dict["iteration-time"] = elapsed_time_per_iteration log_string = " iteration {:8d}/infinity |".format(iteration) log_string += " consumed samples: {:12d} |".format(args.consumed_train_samples) From 0bcaeaedb14d79062860db9eb0107822a979f29a Mon Sep 17 00:00:00 2001 From: lostkevin Date: Mon, 20 Oct 2025 14:42:22 +0800 Subject: [PATCH 25/28] demo update --- chatlearn/utils/megatron_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatlearn/utils/megatron_utils.py b/chatlearn/utils/megatron_utils.py index 09899889..9d519930 100644 --- a/chatlearn/utils/megatron_utils.py +++ b/chatlearn/utils/megatron_utils.py @@ -156,7 +156,7 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_loss_fusion = True cfg.models.policy_trainer.megatron_model_cfg.moe_shared_expert_overlap = False - cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_fusion_impl = 'te' + # cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_fusion_impl = 'te' cfg.models.policy_trainer.megatron_model_cfg.gradient_accumulation_fusion = True # try5: False cfg.models.policy_trainer.megatron_model_cfg.async_tensor_model_parallel_allreduce = True cfg.models.policy_trainer.distributed_timeout_minutes = 60 From 44311139b3b6d3de1c2c26f7dfe958ee8e879547 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Tue, 21 Oct 2025 11:39:34 +0800 Subject: [PATCH 26/28] fix convergence issue --- chatlearn/utils/megatron_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chatlearn/utils/megatron_utils.py b/chatlearn/utils/megatron_utils.py index 09899889..4324c87f 100644 --- a/chatlearn/utils/megatron_utils.py +++ b/chatlearn/utils/megatron_utils.py @@ -152,12 +152,10 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): cfg.models.policy_trainer.megatron_model_cfg.moe_router_load_balancing_type = "none" cfg.models.policy_trainer.megatron_model_cfg.moe_aux_loss_coeff = 0 cfg.models.policy_trainer.megatron_model_cfg.moe_permute_fusion = True - cfg.models.policy_trainer.megatron_model_cfg.moe_router_fusion = False # try5: True + cfg.models.policy_trainer.megatron_model_cfg.moe_router_fusion = False cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_loss_fusion = True cfg.models.policy_trainer.megatron_model_cfg.moe_shared_expert_overlap = False - - cfg.models.policy_trainer.megatron_model_cfg.cross_entropy_fusion_impl = 'te' - cfg.models.policy_trainer.megatron_model_cfg.gradient_accumulation_fusion = True # try5: False + cfg.models.policy_trainer.megatron_model_cfg.gradient_accumulation_fusion = True cfg.models.policy_trainer.megatron_model_cfg.async_tensor_model_parallel_allreduce = True cfg.models.policy_trainer.distributed_timeout_minutes = 60 From 8a20d07b2eb45d01402212b6271f0683810c4014 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Tue, 21 Oct 2025 14:14:17 +0800 Subject: [PATCH 27/28] update readme --- docs/images/qwen3_next.jpg | Bin 0 -> 142472 bytes .../tutorial_grpo_mcore_qwen3_next.md | 9 ++++++ .../train_mcore_sglang_qwen3_next_grpo.sh | 29 ++++++++++-------- 3 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 docs/images/qwen3_next.jpg diff --git a/docs/images/qwen3_next.jpg b/docs/images/qwen3_next.jpg new file mode 100644 index 0000000000000000000000000000000000000000..73320551dcc5d2596b97d275f76ede6f7232d59b GIT binary patch literal 142472 zcmb@t2|Sc-^alKl!6>qnCF>}AvSnlmLnT|5Hp*I21|dt?h7nn^7DWjoODZI?q-5+% zvWx6%DBFy!(QMy?*7yJa*5CKM?@wbqbKmzl*LALQ?)yAL`%D`H*bPqTp8y~b0JsbO z2hhd=FC8Dps{mkR1RMYWfC-?7@Bs|qEASRT6T<)d^$Cb10Nr|@4gkU(0s5c!7=!L!Xfy6PbqTiLo%7JGdrz1| zjuuvPo$kboDqi*o-m#OLXD=_Gn7D+b)B%LjVPzFnwPVM1PU!0C8=N_N&dmJ$1q(a- zD_0#*j!vFlH*R|S`1*z14h;*BxDy$laQ{JK(!)o|8JStxPjj9EUEMvOdiy^24~$PtPJNl4nf>~Wu)MOmw*H;C@nh322mswS3;ell*?;Wf z0PUiqr-#zRHtmAYd2brdLC>&HfpNF?DcB|VJ^K&cV&Xh{?@3|x4iUxEc&^JHojbWj z5#wTnP1Ck4`+sIw@c$>vejE13u0h}!K(}>4>FA&gP$-mvkpa9InHV=OrkzY%*Uq08 z%htuZd9nSzXy70aa145SdKmaWJM#`^_W#!xZ5X7bDQy5?fkHqsK{)^Nwo;WL5GqZw#picKXtoO#y0OUFi_`Zg0xa(}8Y3#N@1D3vF zVgRSpp2@A$flk~Em5uZRJE{B6y+Mc|^#sEU*g*@VZQ0-?^=%HZ3*&;K0V_1XjRs)O zkWtXniA6M^wAgKf29(hN+;b`dykiu+0Y;&i5Yl(adTtG%Or`lKI_;QqOMM0u;s)m_df+LeC-c`F~)KfOIz2V*MN|Hb0$RHq0!SAPMw zg-g~i5PmsFaz^|$@*nqYqW(7qpXT&0VE$vl>?Y8^AxKG9ZN2(0Fpl7uAZm@jyr}S- z48O3|&PQ#i;(k&G@?5{oS$W&FMC2wzK_+Zc=hxjp7qDz3vuO(@aMUgC$2aVxK8Okp z_>Gf+#ikcQK5w~s8$&XDllehgF8rIFAc(r{WXv|)9gKI|jQ$6SetGSek;ajm!-H1u z+vbqrUz`BRzWO_ywo%@-J;5&?_hijzfc80@Xu}lCY=_`~4eZW%&G{mQHc6laT!2Rp9pRk5(#tq06;Wnd9|AFxux6R&vP1tAH<}3&LrI`kp z{6r=5Lgx<}fU^8$Mb`|X~Oa<=Kf?RedG z0Ei(RNdO~hi_ltp$5avj@@SLBe{oz*dJ~~`^6j4-lKV$_f)MXIe@D?aD0)giC-^1W z2`Ou!;->PeQak=t?O!GuM}FjMi);V!)4u<*v`fQ` zlUe^t#n4S3f&*->`KJqh3HCOx0Y{dp|B?wL^QOywrlBCj>=sDKW%i%Y|3`j+uBHE` zcj4ZDl7fM;XD^t$y{^rWwzK=cR;o$MJF6j)Uw@hSpI|(bFSe;?e`5J3&9*25>Yaq@ zKT_~Bac^fAz_E*EYRiLIup0S4TR&&T^mh4yUEK~R1Eo}W!FuS=%3!m?0c(xdtv{2T zv&CWG-w7>eEAjqWoReOOwX83_sm(EwGMx*Y2}GzCPlIRkM5L4|EFX|Tgrx0KdmOe# zk%ZDZC}T5-Q#t@wtB<+CtDe=)d?!<}PpxxD6doK4-C2M1nJU0yZlIV}Vdij)227c6 zt`Yk~;ThBv#V3TB5_F*ffoI5))T$1!@Va}WjIv>Z&yO?!nyJ+txY=KS=3@+x$GpzR zEq%a*I;~)x7=6=Oo}|x$NxSatAbhPJSra!e$V|P}JNo?#vZe&bXS@n|KnZz*=};S_ zl5>csD7T*2Z{584n^_4b}^))_--fYzKFX&F0iQ8AqubO`ebf&2lp#Ejmj0Xvbe5SqCD zpR0QgsHU?N{I%XB66`1qNdIZPv1hx1Hz(vSViy~hm^Aon057@UqxELZh;L1xK12Fr z-*0P5zN)sD0QND^W9{US?e#&mJ676Kw|y;bnbI}3wxGp}{LBXrQ2PwlEu33R{A-3! zz{&|$&w-#rxK{{uu%7)jOUM%%@YAJcB@(JX?Wp`YvgF^GNn1p2`REUbKQRs{1Sudz z<;vFq*9bdC{~MN5AYfC!e=~aqL%dN%KoUTpeouOO*I!0C2WhvV(pj+cyV#J+tfgQP zD@EnRT%5IC{rSt*6g1!>>i0XG*w1`I`qZ}4Mv6PgGxwCWiTfRlDjokAdFx3C4G`YO z(vtH&xPm`K0*SyTN&_5z;`(Vuvxj9xi}#pz2bkvX+$crPfdqknR^y>Sw>|G1h)A7< zO8<>;CJA~JEL`sV#M?;{Dnk0pET~ooSh@Y2-=bDElU@k*?*Qp2zP%vwzmV&lK~(l^ z`3b%e{&Qg_^Sl0pgKw)00J8rg3iHb|pqo$uzXHnnG~lVlYx~!t<Q!#tDnI%x;=w$`}($7ppk#*=iKFq1tSKGV(iARkYEgXGLyQ^E3-1y?CLp? zZD7o3{9Xp-~#^A3Xlj}Y~G?iS8iLt zGv4W-Lt!CLKmzTayN2oeAA#LVc0AI-X#O|8x}bI?Y~H^uUOlFa;5&Z<^_NhVgn6D| zQvc5YS{(=l5cXeT-R2nZY|~Fikt@G~5{$av0pYZgY_TN_G~l6Yv3C`=4;tU1tsdlLvZIrnU`{ z{qZX({sJ!_4E#+u{if4Ttk)ZVJVKCV&u?Tl$Nmeabb+q1sSZXH5Ubxa>iumI z*GmXYHDfzZ{3GGNLg2q4ae6bhB*2Wal~79>Jo_TgXziW+TaG|ChOzxhD%Vb%g46${ z6d)p7blyto(&i~7Q^i)`3h3Aws#q_mRxkgDJ`=mV|4oH)pl(TkcW>wMXXz~98-IiJ zx2Uj8N3(-FWY|6qKm>;$y zDqjx*8t`{Y;l%wVlTMy#erMjKvw*X1Rw9)@ON~F-Vlzh~wkDWC{>=XW*uzzz55_E1 z^q*3Qk2O5a>t`?yO(($$wrdU2*WN8+Y-Y*d2tI=)%X_BvfVIkBK?;^aY`;sP=c>)y zCRnfjiiE!qvjC$V^v@r$KlhR>_heWL9}v4Yid9JE6R*kTonr*cRZ|$(cvSfk;)vzr zuwfCY(~mzU@a2sv3Y*jDkBU-u*@U_H-y`KrxW9YSf{!?(~KmWB;N1+h61vdv|#F>zDR-< zQeqH`P@H6x4>A=KeFcAuLbpS94@w?FE>sjC^)6t&2kw=mNsZFFj6AZO>v^hG0C7^?P z2lP3kXFH=6R|w0`Sh}%?M)EU(jMYwBvgwSk2r@TV-7ZGF9im&t^kk8UfG(4atmzln!Vp zY#{!Av?(N6J7q>m_Z+oD&3bg*xHDj?dCHAc>Dv9;yLA3)#ppPS7%Ni(C!QCb`Dpuk zY56-1;CVzXh$(MC3b<5;D z2+6C~qD~?wi>ZAF+|Uon6s&jafbyhZ+0`HfSo=1AwNcEJtG`7!Kqxr@$44(FxuV~v znXx^14c8=3-V9h7ct-i?|_-Q8_kHnfrqtB z>fi^!`)+z-rVoEy7@Zn(8GRko-(IPZ=8Q`RSdEQ9u8iwz0duW;XYx6{5y0t-GaAn} znXxCO3nZCVn{PVoG|NnyqoX9SXe0qTJv!LGGxllFPUBU;vpGARF47pJ9Z)@&%;<|a z)n32}oYFf2!V77)IMW*sVuk{i3>0+$frNn|cKbcx9&)Z8sxd1#3poLTZtQ6Qvq}bF zHh!l-IMQJ|*-FHu_NGG-A*yYT{-?B!7}3sILX{a0JPH-+s?X z^bOEl?{sLIkqiK`7^EE|n^hll0?yeYRPH!NVV!0wv8=M5!cL#x4=uXl-85SBXqH(B zc0^O6Z&0Tsl7DY#@AJSBqZD4!vYpuDPIh#gKbwbiS~-3x+)Hl9FqTWcf;F()`+Du} zW)DAvJ^Dk{xE*uTUb-NJmxhByxVamP&m$54cB!1Qw?SO76QURM?~C5Qaj=|Mct&{r zWp4!ys2X6*aoo{|)Nd+kffTJ~hK;HTP??=n!Ocln=e#Q24SFkfxW8Zdb==zQq8@?rND zP6u^{htIFY(tzt`cbBwwDy9{K(qXDL)j`j8!-cbRu;q|wg2QEQ$Ya6lYAOC_*S@43 z>{e&HF^Odxaef;zILCf(U5_i`f*$(i(*4QluT8_P7I8|N6S38IT%0|ljZQN}VNwNpI77)Wl# zBgv7yw%a=G2n3cq+EQYfD+$=OtsQwTZ}W7lr`A^k$l{jLa)zi(2XRMAp&$%$@5P`a zK|4=^B{(!&vT1J{s6x|8oFO9bHzPttV8tMPG8r(&?6k0ugn&$4L}di+lw#BZH^O_~ z0|Fh2BSEaH=?v)(eUXg1ev&{&5SyPQ#7GPf@N5-s_XFHHM?4(+vv`-Z7@(@Eh{@>F zrgWh0iL)9z)`Eutn~GO2Cfqc(qQ$JUwJb;Iz^g5kKL z>#4J8&>Z3AijM2VZHBEQCZJH z?8o3~rNqP3WtApmgi`2)8;t1Pz>OdD3G1+ydB+{SXn#fGC9j@}%9I1P%3RyUjVrt2 zX6dX*Bo|X8Bi)N=Kr=W<#Rxg!5V|JEV>0&6Az=Zp_;e!VgLlO4Z+Wa9Z&aJ0AtBbf z5jPf)wR}o#?_LyII-g@GjV=@o1z1%@gQO34S5bKi37k~vl2>Ox$trUvc?zAs6mj}y zaP4!b1on)j#mj_CFCAZhRmq(Yl(!YE6S^ku+Iw$u z7zRyqJSzb5s(swhPzT(Ww+gm$nnHAho4_Xv_lX4ie%i?jWI03YBM7H@B*|gV_nJNmR13cm;$|5Y*PP0JZ59 z$8>nUWE*(ELjV6uP#$Zlng~+IRQY`g!@abb^Q%ShmDIc{4m(wHy4vK!MTu~h6JPgw zrZE95^Ixkg-145jS61VGw2!n?C`qw^aVmDUBI5n>$b$q&eBsO0mR%C{YPqBRUn9Bq z3g2<*yHy6t65{r&x$sj9HiazggU19*-ZEv}IlBK@)wxqaz&wz?1B^*1SeF^Eg{IG8 z2Au;Z;^@^+WB%{om$|nsYMc6d5`bo!m=iXXqQIS(LQy%N}u5ZwO6kQ zuGtgs1l_C(OJ4=o3``5}f1~)ic>0iv!c&E!_uN68U8aymFjs$1XOksdEf^b=6ZE-R zYtbz#X?K8$FQTQwEX<;C{$0u(mX%V*&qRnM}5@GNn5WiOoS2KNJSfNtktn zVFJmFo8rcKN{f!vsRI=$0855jjS|TQk^ryZS4RI812s=Q#aI+X*VwNUd|@Q{wcP>$ z(}|(>X2j^oFb1(Z%z=We0oFTOQ`3>4;)A8R1q{617EbLREeXWpxdWuDNBBFZ2qv)= zM`546KHsUy#U;i`YFQE=du^limKxdSH-`l3T>4vEr1^g&KiJ+y*{jDlVx<%btpIBb__o1Z;gfmyelP5q?fD(Ja~2)7_`HW1~BCWt2XsiAT_Qytyj- zV)=JjCP6S~wp);&_F$ZGgUaTeIfTBDKxSRX@$(!nnMaR#A{MTtr_SAx=@08nZRXfJ zlO1q_Bpul0XTBgcW|?~9VAzBFuX|)FGq1uwo^!D7Y~jC-E|(wFp5nUH2IZ*RqfV;A zlq8l-vJGHuC}}x(%KMVo*zYgOS`>>5)=il{a1^_?tM?5~s!ZefDw$}&2Sc(Oc4ICN zGb?TT9JiNz9lI-#%jb3DwQ5_PZdccR?I%)ABh0A``_7Rao9P@9(NN-29@&Y2sf!Zp zmRUfnjxCszWNQ`=-;5`8X6-)W@C9Hvv3Gu=GNm9Ad8wVn0^{zTYSl`EVi68OnxACZzv#MPv!bmn{Q@~_>+!M)yKlFF>^>AG8yy;JW6o61J0 ztl1H)TNf)#y{`^l9Jio9(H@H7M(@IVkN3u1$(sw5UM6??*w^i!c4sK36SvuMfq5n! zAb>68Vuu9|g50?Go#HrvEK79rpOkU4D0P1J60k$v;di+)$nJ0uD;3b7WctCwQ&-Z8 z1lY6s7GP+K1Afjhc{YaYuoplJQFX-+R*d~?r4E8 zjq7(iyYg;`T9F90)_{9+K|PXl_AN33H}<>>HL~wj*{6fDSIc$}4j)fACTz{H*EOd4 zOfZC3Pzx~;Djs-?2wm@`JgyY^Ryx3XXQHmSXNmQ5{?b@-yQfg3*aLd8J7bdQgeu0$ z7fKuBBui=|oN0{)gbM8=gx^9RaTv$$_PN;>aFnNe!V={W+kdjCUQ_{yw&n^x+)-WF zHWX4$1L&8Ch52LyBBvL2x5@hBB_`|CL#pTFmo1p$^_UZek0I7i8 zRfBwtkQ1`Du)FbO8Ga%u*7A(_hTg;4m8Hv?vw{cYz=FS|F^Vihu<09PtHLttOe_g} zc-BpM{>UZ2rE!MoLs*UCUNK9bftYV8!=bNq-0E} z7+=s9fvj|w7?IS_=I@E4c+XnU)HOj(4cW;HB%3)@O4FTn=jSdByg?HCQd>R+P z!blleWF93eGo;U1kX@vfk}GYirA>c>-Ya+X6%RSt_Le93XvWtM-=a`FJ*y@m*y8qD zw%E75CFwppEz4zBxy;LXxTm=wDd_|Hs$Yx(7;K|y0RALq;{b*eId>cR5N(${q*z>Mjcqq%;Y=7J)KPB`_ zr2zFahN~DJsfMgE8HlT9Cmu5;@anCS=8o#THdrWxh9zFPaqM8?@5h{{5t**?rw<|9hI6$Z=o&Qpws3F`awcQb`Foe#MypBrRM1MWu> zC(lvd;2h&_TTmY$7Z~uBgZlXCTyc*4)2JLKV7N#(4V8Qx<{lHgr~At{z6e!#{#n+P zwSwKmMuI(&ZR~0ZA-qPPF|YI{`)L0iX5T%HN5mV%^jxxSMd;0|Z`zji)zg58O72bs z4G2cEov8~%`1`uv$kvl}D0)TbD-tRHY_>iPVLsHwz#kmbxbJkt;lNj2o{>bh^$1Fy z=Dxu85musQo--dq^guyWe*fA-(LKV&$&w3c_MyiX`7Io*RPC1I{0Ieb>M*icAdv^j zp~*>>xr86-EPDGg-_c{ioqM73x_k@YiQ9yG(|}*8htzUNJo3D6qk1y2Y~7KPS7IAi zsY!4PULtcL!$t5D#dp->AF;JaMm49NbS$!K0d|bZQw6Z+DUSnHzm68m)wIY(a3^B> zzhAy^a4G+o8@u(f;1Bt?9$c}%F)1t$yJy2gIuQ6NP@1?#U~5QY>sz%HJjBNO+~-Tl zihA>Jb&`Aq_a3{|huCyx8#2na%zCuCeSrT0(R_l9Et`7l>EQ8>W|ZL4O!=E){fr2@ z5gR`9{*>#gQ+Tv@-Dp|Pf+D`pvik-m(st**jtc1)P3VZLVc(b=V)PDXOz8GchpN^d z#Vy`y709z{q6C7)snWEK1hITTHSVJs!D(<~#H0GH8I`(Qmo& zjOXodwFdfsruM;0-IErV-RF4tV({tRhnRSU z$3+jfe{veD6d{}9;Wf6bh7-65i#j#URNjI7ePUk%pX`s(_7$K3H?&n|(#0!fDGxAC zm{?pf+Z~yiPZb=O9L7El9rJu)aU@~a$5Sx9$cjAe%PpV$*qx`|+cZqB_fyK&*kHui ziQ{ghi#s8c#M;I0FKNK|Ol|uPv@|{p zJLX1zK_)EW>C?|0f*(H`+vx9IZkrK%xTg7;Gd1kx&0?_FSK5#m!x1GLm4QK6EIDWT5vG`RpKr)~9j>5Rb4H#DOm8MSE3ZNyTj^S!%!V??a0!hy8kr?&owK$!Ir3;dwXVBubI>4?PRP4m}% z1!K4cTWCO0PGf-KdIZrbljDt5ZsDhi@6LytmRDB1Pqofl=(Ufri{mKv_56~sqE{KDd}_UK9!Elb z!*Q$M@W=a4w1+3uj3|}v*&Dx4)ZJixQc`H*FW1vk)6H{}C`Zm!Rzq&AVsF-oe z(5)f>{o~5}oN%;qUUT->Bn3sq;XIwFX7Q!_f??tiOhv#8xll|gtQr%oshTi88nU?9 zP;{?d*zmFEPCc0>)@S{y7Y}`2rC-k*{fwz{sIq;ndt2sdT+6%Oi~t+YXl~`!>EkAE zLR#RVfdb@1pOj3(Y~)Iblx&`OBopx|zvZgU`uAL^0S)!0jMN?K9=NDe@9SHg;oaGF z_rsMtAnyU2 z3p_$}lsxjOG13s_wIRNz)9N{u8GE=s$0S4Vg6^xV%=Ez?N?xy$GaK7io`J=D`QR3# zV9Q5S8oMRWzAIecxuYOOb5~&BC}W_S&kI{dAHK!;Z}%)UMp|<1=qlWcav8*8&%42d z9%;l}hvQ9c@4)dv)Hc7DNiK3p&wSpZRv*6dUC#QL1cMLDXYQi`X!Du(OCeK~yHrtX zJA99zW}|biqp9pG_qfrM3+oHi?0xS{A&Vj1VVUWwPJz$SVD8c6Akz^TlEO6i8TkvY zQJz&qSX%NXeUK$Y->rpt2uU}9Dl>(9jUjs`GOd5P1DGKLM6d%Dz* zd9wtb_58-o{oO;<_aKLj{be4}6liq3f8S&_4FHQENDi6vlRAARf46Q}#jNi7`PaA4 zUz~MTfM4zZ;wjPoJ@9*LUozLNA&N9nhGdD}NBlnFU%=!;=iINve?+e*q?2TM;vK>n8=~rXv+H1;jev9gy9+KKj_!({7_lz0U)*SF6o_^1 ztdPUM9*?`VH1!G1HNFd8`@Q@4Mc-A5=17;>orelX3Hu9Af5>{S(gb@|(AeJJ-4Muu zmst2paK1n&ExVHbS)=dHKFg@Nh=q(prO!f5)uVIYgo?7v06$La%fFB7$ME~$PT*m; zZ-6guzx|plR-9zcC^j-78D{B3NQnK^K;|WOkHy7OIk62kkoXBBwUI?bab<aH%nqblCLiYokc?@YmTw2Fba+KowGmnF{Qx@vFv&%~>6KLLY#ZiTw z(sVmRQWgserI+2PI}5SIW0mk3DrO0@de%$FNxATQV;wxoI}l3T<4uUv;5NEE(PvbO zBhj;-EyT7ojg~|lA1#0ESo2&Z58IzKd1Gk-f`7{vp%xZQ6?(LIwNyUoP@`Nz+4n-# zvts-=RRs>cYHl!s^P!LTRPG6EnVgWl=ghS*9scCJk$~bNhpN=>OQm<+!{(ik{bweP z3kjp+EO2(T_9f?RE2ZJq!mvK?_~))#uo=-V_f3GZ4`SGbzgPJ zco`iXzhv4~Y?d6oBCn;#5Mc%A@h(bN5E>UoNhUr@3uw1ov}n|EybQToCN`5{z)CM> zMN*mn6UQ(_`r&n7VR_DtV-G35yp7^u2do;)6(~s7?pX`qulA#q76zohiKI&IO)T^H z7^JC16}{B>q(F>d-mt{;4F{}ZqG$lLLViTL=SFd9qM+(`?&C@OFH$ei7a!bN%6dpc znlQSqLo6fA)MyG58p`nDRdbV;@7?pC_MFYilWE%+W?JqK-0+^mzSBPr?yBPGSCmG~O0z^)FRmV6XkSbNwYh}A>$ zd?lq2<7^UcW|pQ&ti^{h!=b~5?7lK}h1S&q6>^U%9S}VCD)Hp5{zO$Gl$AL=z_{#U`(nm%W*GZW@)P+7rR`oKNr~l3*X^md;RsBAl$&qz|)348|C=#r-;gUqsh*E zQLo+K3Qa_de080JxDBt!+|@oS7jiFNN7Hl~X_`Au1NKsTEk?tTP7Cn$8{xIa;W;N3xV#*qz)MR$-DQ5l*%p$BVuxjV^nD5c{CmP=ubf zLm!N&8!NhNtY;e+2p*Y#FeXdU0BbKCMI6!92Ofky$P!@n&CW=FvU!XO9?iEu(yAL7^Hpo=7(>uVk=ZwS380;U*K*8I=c89UMfyQh) zE3~?w2HbY-mZ17=9z;KM5a?4J0~?Js%8qF`khh&N)`vIydDYODC_%7mzIm+7sZ7QCcLP4BUSAET7LJ&;ea_rWuzY;cVl zD2Ts*hvHVZq*by>U}f`Zjo6CrIB?g8dh;T?c6tPrU1ZLx2X{ z*la79?h^P8F6L?UWb!Gn^8$S;3ASF0Hrt98QedYp3v3gj(&_Gcr8~Tr`U;vq3i=j2 z691m=0Cok6+5F@;kFEiEfAjIA1~X!&9GUa#q8!kgGe}GiNJWJAG`P^EN|~tcT4A?! zgU_)EOyE2jwWj*&ZA(n%@9;GG4=XlSGWR0K_Y$UWzbNg)9%?+)K(#+=9{2rTg;9b) z#15;wXUXrJ4n7i@K}KQ0qT9_aW`skNiLka5>WXMD*7cx2*r_KLbwuwHkAMQ~bKe^> zh6vC1iCPya#S0Y|4*NW=R+9-GKpcPjIRC(~*~QyImgu8rZ6IRdg1PO*1tH*pz$pzs zXCvnNQEo)5h~Dc&MX})&N3otLUV4r%12xO6@*kF;j;Hpw1|}ZVS-8#PfA!!y`;F`Q z_d=HAeKa2-2I=GK3ibqc#07wdYLlI4^|B)+x$k=Ld2f&^!A+D*k-_uYVs1* zC|QOloZWqIsH5t6?;FG~)-s-M`~ZVXHY$OVfj0ITeA{s4%odDHd9m?O7=e063yrHP59^9WiqI$;nvYw*&4pC^6<}gVx@Pnp2 zBRXDR#Pz0jCf$9Yp!YKV+2uU>50=qfEZ$`BD<(Xa)Sa-g zr5$mjj#y?d#~aS~zwKU}q6`20W#N2))W)rE;DP1SIJr=yT|1ADcTH}QjI@67>pOlf z#gXSP-*x{AI1vrk4N56RuA0n_3;4reeJ4+Jbam|!`!+pm<$Jq%CUBvA%4KHJrJ&!% z`9v34H**xlH(X@%b>RLX%OP#y4~CpC&UJ=Ap^qfLDWt^VoZvx$!aYwZNfj8UG{K&s zBNv=}RybGrC*yZKxq9l>^vgz(nBt68ev4GxJor#EwIhv9WQx)Q5d(h;Z8*9En?6t;Thn{UG-N*-^> z$;loZ(cdr0DJm&+Hm0qES>V}5>f2c7T8tySf6qkO9ZjWyzzz(sMzWT!dva-|Ya#kt zf;P|HqPh{Qi0UU0A!V!Uz843zv+MKYqHwO@@ph-*;gB(k^W;+0Dh*)Q==rYi;E-b} zdg?-H(}2kcQ-;Iva;)~(-pVhOva5+R>*8f^Qp@xYpkhDzzuw!7O!Z}}H~H)dZhhO+ zFU*Z{P(?pGrzr%FiQX)FB(wV1QvAdrUmKm$lOL(JdZsQgPX*?eHIH z1u=Ks6mMJC-ak1WIuoj?;XGhe8kb}B!jV<>o?V#gg^}|E7eu0tWF5JysOX^!Ky3Bc zqDFTR+GC1M$h?*q!iVs3L%UAc`#?n}UO3*2z8#4Jwe!?kL9ZZk!n@YRVqH+#fN(Xz z-}`#?{xE)$ujow;x{PvB62vwY!;i6x8{nzO6hAE_%#17FS+(A27*^seB`fj9@YzW- z1q=Ohv##?E3k7ix$yr|NFb+ME`T9dbTO59A3>j0Qoi}vw$m+XX!;7CkmU*8-$e+lp zv<`Fc)RqWx26yCQxKKAQ+-)H-g3*x4Zs+8(B*;arcxk-{hB?nx1p7qZ-Ww>N#a*pPiur-8PV(($`Mjzv1kz8OyCD90M55=5DTUUO< zIkXRUQ@N3$h<0Ca3mX~e!X5}Alw@Fy7&c_xne zz)#~%g4JPSC5;0&P{||yVP)6_6|JL-htysGp~1lUQa!FAP#R9JfZu94QmaY>oP|SG`IxN``j`vxFUm}hZ;Bus6H#v4IDM9`Vx~~-)?6AtJ+=1jj-S!X(il z*wXibz1BzNLW#3MELqBK5juO+ef-qNSH{PpeS2fMtv12>0X_)k3FK#C)0-R6GWs`Nx+rND?1bPIeKzyPltc*BIEYHjXJU>mzK zQ-|zNpmJVWm_PhBZe0_5Ro2|Irp(XhPG`|sg_x`ex|?AGCcG1GT`8Wgc8QjRx*O-NR!doyMdt=$%Dh@4;KMqUlGIZqQIVN(# zRwaCz2AJgteOefW z5Gza^b{@|G+jWCa!7sn7u|)*Rqfd4(tQv8T8Oem+mTdJ)=kocYcd_i~%dGK7gLapK z?RTPwj}EgYtY=elZ{R{kB*4C44JOQDr%zO4r6h+^@tJs;@e5O^OG8}u-&I6vSvLXv zj|y+u?w}3`aZp~kRI)qTNN%!C7X03JfDO}l=jx#SDK~e~J@-V6_n#0ub-`fq(#nlV zrFHni4U#Bva$+=we1s4_&RxRzU}Bg~<#NAA>+Z7tY{&FODK7Z6XY<-DhhH^{xV28< z*l~8Y0|k@eA!%;lK6&^24LSA;2wkJ@yL%B$%UX&01I$-GzEfrx+%ZF3kC!xqd3K{6 z5M^+@k)G&-D0zHXV!397uqhG%axr)#842SeN(!#nCBN&)dtI{%=Q~;(QwsCPhXS}x z0JEwZ`t5mh7`h7E=YdV9$UyLNolhv@h>?6-m%+h`S@rj=702yA-xPq$X{d1JlElz^ z2_{u;A$T;B5c(-~I+JUmwK~3jDqb*w{VZdm%3~1$GA_nwRj@=;5IiR?0Kcq(98dJ6 zEP}7n2h3+-_3xZLNpi#wJtuN^5|O6ow19KWIhEi>N1-v>$8DjEB=185))R{HXd%WP zL<={bG^$kHlbtMFGpn_yW%<%$mE|_i*w5{5&(UsU3nq0qScNwJN5lCur#^W}zds}; zc9s23P6>+Z(E~$zma2Hpl$#Qfav97{?YEI*iC_y4pG`i)fA}0)dmpsaD7(^RR=No{ z_87NpRe690^oi4eu7-kQF;{FOFpxkIVqZN~zQ^lb3v%~9ORranzG6om4pH%j^ah$|VtLES z-yRR`{=AV;f3H!F%w=MH!Bp+e{AWg9UTZ-;q^Xw3$oo<1&L~b35)$Q66igt-_c<;q zI(lQ<-da1apCg7mqYjkfiqU*wiIse}#;%q*I`DjsGU3i)5>^PH>MjK?Y4{WvP>R6Q zG2D7M(K3f9Qth31*u3{`1<&G-+wqMYk&+S-L6;-Vh98+jT@}*&=n7utXR3o`bTv0f zGt5|;pGlb8nu=Gp+m5eE(%xQFD-NYdo$G1NH;;W`kbnU*^tWuK(NM~BO%b0n>(>j3 z4w-0)KD74p{QTk4IE;hi!9-;#MA2nkX7Ho##g2SL;$!1oQi(t0gT-#EL=+WA>==GMHJoDpVU!Wg3+^cP;74n4y9UvkpAb!G zRi%uf&lba!dyD(Z-h6zPcO~N05dZblGpKz&4IR~s0iGl#G7G_>YANLVVH>r$`M0AH za%ogU>*|pZ z`_itW^jeuVLVp?KQxuzJ-eLP1e9(r_3-%e-`sQZ{D~82tHM$Bs+|Qni?yiRoXx*)8 zWo{sNWFNh&*Q{E#wlt8rj=(Q1>=G3^*rl`wJZ$0pd~{#flIyIj$cck@)CxZCjKA5G zbAWAM5FkDG%ufK^@VSu=$kpc#f~p8|^O4Q02a1pLUjBm$%gp=z8{Z*OzEQ!0>uIdO zOGWAqe}8`r-mtF7)=K~S+yrlsS)PopTUx_{6m;@_;l1Rl#+dExFA^WTCpS(H*HBuFZAG?K3e+E7# z^!Pj*nbUkcqr$1im#k^U&SbYZw)cV+qR1C@ zSKZ;j_42#+YmFA(Ju*)ogFI1=R7E-%D&Jtp=*xSgKYvo08QXO4xa%^eS}97k89e#X z2-l3vf)F7#og`Z_E5QLg5l)ta5DzbYXg7^JT~=|(w)x=o4R_JVS_NHc*OqAXo4M9Q z$XjVLCjIPWsPauD{x0Uy;iQ?3cuHKGFnN$%py+^0!aC{C9&z&NBkvK0kW`0%r22hF zexDwxbS4Jh-7MHlixK#?_Tuc36xR&{|I8t0!A*d|BoOc;pos(-{LIXH|r zty_taUpGo))eT97oV!D+7F*h9Y%yIP9q6}M@Oyeiu-nC02z5kL4sRAL;5e|~`S8t& z6JS5#>T&DV24k3bC0KBf;L_ zf-N$h1eQ`u?SrK8s#(BrG{Cbyzfgz5`H=f2lwfL&$$CQ69LvNpU zr(Mj^7jfx;`MB+T621<7k$8oqOJp$B_w{*e-+N)f-%V`w)y4IpZ=Bf01ltNNKBDcq z66KM)KS|vw4f&q%^2t+MiOB4t_&$>6g)B*#00g`D@pv`UaF0}_rZ-n7J}v= zh!>Jh`HYS^R4-iSg&PrVUEFfr47N-Bfm5m=|uBRn+9OdX~xNqhaD^vLERVf02lNapY>{1W}MAjXr@-T9BR? z9e_?GW@pLc8ubFzb`4<98*>MV2hUs`+L?UfebIv?=TkW&@gAe7B1(&*e-oEI z4Y*}KQwetNq%N8KSb|YA=WK1L^|r5P>TUbGYpEjmoVXGhY13{c>4yae!&23ki_hFw zXsz19CTy_51>w{fOrNV;DZ`&m*Hl-?~Sozb+e!?w7h0Cch)`|6uPeqoVG@ zc+o*hrBu3AK#&xq8+ic%0Ria}$pPtxp+o5gB~`km8>9uKW274-24={i=Di!=bIv+< z-TUEwy=$E>xO8M@{yU!M+0QRFTe`cM2(43;i(0z$#w5}d;48R72fizbXju?Hba266 z-yBJ;+(mn{Y+x3AGQ7{#H{12(ec*lmbsl?5<$hz?B(9(vm0EY2w*SG?z6>*|XPP2% zFE>6#?p^LxX#tLfqfZ%SD;MkcrcHc8Ga>f}q9MtvA#}O6 z-{WaX0~DDPjd=EYko7v(X_7c9D-|Cvq7wxZxlVIjT+9#q9q1bw%mlZY!Vimcvs1Uz zc)qwviTGPAMO>Wi)vro@Tq3a(XDx)i99^DFA)mYwBba}@@Pe9LFDSV?GDgTksSZ0h z{&-~13t6!mQ-G@I%dT1K9>O!Z17Sd%xElKaGvM9*+LKa!YD?K45n4w&e=g3MFQQ>odN0(O%AV`Y^!lB!)-9|F?@ zLCX^EFe~^Im2eP-;cF+|li4Bi1xHa_LXbc$97vlBpNM4&BxUs>;<*);UzXdhW$%@s zNLC#Xa-ikYM4Xe?@fB@!9M%}NsW*=32|b$c-*_yI%E*B<=Ni@<65B3H*_ebZGNBic zYWir073FkgBQ=ebv9*Mg^%%&*h8XSnu|6ik;<#r8*{Zcdzf0^Lm%bacWLG%^{EE^I zos?JmGHEY;6;KiGznsL0SBYX56gI<4a$3SFcj>FMJq|v#KpU-+1T$h^2_PqWC@Jdo z^P{-C$Mu?9h)yFL3yh*4%l8pH#Ag>7Q6GX43GJC&D#Pf|1!bOnrxqy7Y}345wTD_w zZ&@Pl81}CZl6_Ypi=_UcKy4O<&`cAHn4^L1| zUBY{b`$X;%m!}-a-c%&(sfrps`U|aTk&N4_!5Kj+#W#u3&_8My;x<40y<$Y+sc6PE zf6yA!Q7POt!AX3H_F0~zO|4bFJf_sq5XPzt*k-RTu1dhM$q`=cZU0AHeVHD+hmk}; zD$J$VuEtkcB~^Kj0xROq<7d&If8tWoy3F6b>Ow;=nf+$7Z>MV_DZ9X)H;!JRmtV@8E0XE= zr2-{vWm7$C@2$(ZbU$vs>U%7j^pH;&!w^M_wWMw!dz@b1iY>gAmXl9_D$obHW$)4< zDa${_Im>>m9s1DRepr~!y`&4Ep_aNp(zrV$d@*|Ku5nEg5RE|aTd~0lWUp`Cr~&Dm zp+Nh0Bz>0;UtcCpBo8t+0#xGq)&OK>IDQQgeHQ@{UcXTZ;eG=G^}Es}(E zKI2(Q5P;Esu>u(9h&P_ofi!{iSq=`M)PBLnuYHI^$WTKV2xP|)pCcg8VW`Q;C((y2 z!|1co3S7@eE=GXbXF}&WsD{rexupIRf-0z5V?n#fGBjS}|2qZ1*B*cbYnueN|1YPd z5{;@WqW=wum8Q46z2j`%YyHLC;|mTXa0md(DRE8!NEYeSSI>Am4xSg!3CQ1pq9G3e z$blI^$qaewnIClq)SN)t#tm&*4WJZ%_ZN%?qas!d5s>NQCyZ$U7_I()3ZNUwDFAYn zfx68FJPFy&80xDl0&yIGiGjpYgK#yl&lm{$Z^jYeyVz0SVdqD&q(`O$c>DC(AYTBy z3?N>hUIL#Q=bG0+?**iA2SZ%{iBelWbY$bS&@ab+T;xI*N4Z^jQ)oTIlO(wpF^ly4>(LbnEVcF&*yZO0Z~ zAH^p@;jh#O`LUzjT3Au|$|`0`Pz^KemiSmcam4%HfqC<@iaU(oxAw|nVAysyg;e&H zUt7fuhF##Kd*0r;RK>Hmy_|ht*R#44Lm1ShJ-$I|pjuD)$}Had841VJE18Pi|AsQT z?=8cz#%B638_ccvhz?P&ucr=wJY?+E=gQCm8G+d=j7YV*Xq|Fw<4S z(LSuc`d;v*eSlv`6cBdpPfn03Y!T;NGF-W!`<=`jE?>d)?Oa||##>^3jdPjR#^@Mh zaaVRXLgJ{pe_;tfZkMScB2YxOg`2Ll;!SbZEO6gOlzoZa?;%toC$nM=fNW*!Pg7%; z-K=+_pcj8v17IWQ%R+=VBczGhfCLRv^DO~*5zx79MgVl3x(7R^1xO*mM1et-#;1~i zDgcpsV0IbAET-l)mNXDgj`&&sH+v7am4g6J;0A-&e+tGN;)UPm5X?Z2K0&Sm9@^5! z&4h7|6vZ!%A$SRkt4yEP?|Q!FljWh-ke6A9`K(mTjGWFP1rzauT9{CE9pWMuR6#T4 zoDU2>eO0d|7yoTxf#rGkNoNd2`^&=oPic}0*V$6$3#7~K9|fd7wlTx)0qb!Hs%vCZ zZ)V@|XyIWvfe7wTtk0hkAlTMFr3}^Op7h4OjVX8%WqBi0hDhHA%HRuwiqyW1&4RW*0)ia!}Z)X0>t<(tX5kKG)z;Lq8fQ4pRKKS4#gi7t^qEK#*w(6t2 zUTQ=DoaP%lv57JkII zrKJ_^;Vi4O_z=vwmJ6?QDEPYA66fD?-~GRD;8e0CAfg4|KS(qq4KxOUAR7zv5I8c3 z2vn#Z&NN(qQ4UqJW+ z!p5vCFx}RWHT*KRGk(bH=O@673B*1&>=g7uc%FQ~K4!LGP z|NKUVIk5}^;NNidS8&hx@AQ9y&6*kr5Voc+8^+$90Y|S96YzgaCMpC{mrY{{bhrM8 z_P!K50wVF$Wj1kJ8t*9t5Ob7o#J@KpOS0fRh%hi9S!?D0D_MRC_Kx9SI?xb5c1%qT zc$qy$3QXzJmbyyPpAyk${K`$K|liv zB$2_wXL;X%cLfOcl9R9bfFlKXhP-c4W>K3!{VXR0qJIF^O|Zo3y_1JcN1ApsOcSkt zdRV}~-7L(%JR1=2m6fH5fZF*dU7B$waOE)*2jfi!@z&U?WgQP7)PMkzO=<=Zo}Phl z43Z(l=`=U5&l@Do8(tF#^$mYjscF{u^kBm&L9{@Y1#AEB+u!_&olgR?!RZhS_QSX1 zSiR4s?}9x0>a7{b9biXTgCBqjgtHkynQ`-b2RBk}upX3r3%{7V-KeO+zfko7X*Zch zgo7}kMBR*29Cj3`5}xplI$Ywsz-KJt5C}<{7{gD(Dn;>5Uk0B?RDC=b?hxU~IvI+r zV0WEgqhy_-_v93}nR>G!Kz7j@s|G$a%XU)@n5Z*>xqa%7WRXHQ;RpFKdqkLA-BH7E zxA-H9myStz`mSRAuT8%G3z1R3lKOSw3|1vP)Kj)|QDovu7V1$H&Qu#!|4HA?QAUHe zhId35VSHemk5A4%>YPLe`@Fg-Ec`kxiv8(!6Xw%c*6~f1unkgn@ADz|EKjT+2t!gM zLJ?TfkOhFHk+(&c@t5rtAu}F8=UepDX)*04Zh#>=7~;MSUc9xLhSjI94_+o$paYID zV19u`!_7QPZwC*NL0YR7rGHZK2RKEI-irD3Es^TR@!=+igC9WT+W{tR>*n`-Zfp~7 zHdnR{*utO`0X5E;JTI_Wz$N>ZJ}j~;4Y}FTkTsm~z?eJ0sR4w^#baQSa2FI#|1DT9 z5K{Bv;u#OXn#fYI^rYMBJpG@(9E59G|I3&o`vRPra%ciX2k={YK=BEx zsHb3nLEhJgsH=cX4AN^B?@bwVvzi0$>RbPq5#BJYoPoF@BTNFKtSAW__38HbD$vR=sgNDObm#< z!5~lum{st9aZ<1WK19^?UIjo;?lQlAvjVWNvq0fG!tWBJ3YJM3gzKh8zHv_Q11?U$ z*vKUf?DJhEu*B~EpC|c0bMpUfb>UccpHVFa2ZuchbD+sS| zg7XA0ycxY->1Dgnu;PCW0cDh)AO63Pcw=CO4rr`pHw4ta(D&7U4^qG-LF0yh3UZ%S zE66f_Hz^*%V{;P=jqk7e&(d}<(7_L20#+8qo)KrSfD!pv?o92te%7kaK!5^_G`OP! zM~9n(Rw?97i6ZmxFA5VAQdXLK&bF2NKg@z$^z;{7zj@lKCTgRIzv6wipukpfVeg@0 z{W9N8nAKHO8jFKo>oU35B1IA|{?wC)GmkT5Ij8UO^?E!luC51u-R++bvm^ajVCu+& z$TmJtcDNC8v)bdEwbSB!zbSY6BcJqyW~MlU?2{tu*Rrm3r>v5GR+*Y#SXV9@GX5zX=5l_rkzd^3tasvGd72#xb6Z}bmE~M~FpUs!!V~YXJEl^#$>n-)c_1w+Q?}A8hfX*RnRW3rH|Hq&RcH)n%L}RaIdY&KNCrekH z6(neT6;kyq%1SdQ>-Pn`9ARSL){3wR|6w}CLn&1?jyA`ba?kM#`7ke_DVeYtgYv!!Zewk0}wQ3Zxbfs{k zdTwqG*?fg<$rHp(y)A3O*HfL)7T@IxTF$;s{f(@0JJcpu%fNmBl7LMtKq8#(v<#uF zJzisA-Zbs-wlaKOH+d`mop$^%xy(DPcizPLEfYWV9Rgq~K8)Sb1YLZ8(vs8%-_mUU z`aS%Pg=$lhPZ&!#YVDy@+EHwDmKa*mh4t!@Ke@P6iSdl?qQTF^vfM#ydumCQ`rChK zH=J{=Ce0q#7kyUBqMdt_WPj>m!&5I0Uv<$KFw>#jdn)g4P{PWxIGYRD`#XVXl5*>o^Vq)DK_OPGYGeXWu6ig@=#&7ga8^UU>X_?Zl%A?kFsRa})llj>+HLHXC= zvj)-Ht-U`z{pSp&&!ZS@VhW14y}8nNXnNKKDJtV=Y{MzoCM$p7W(4r5;!D_$T;ZZT zI+pI0?;mGAQQSN6fzYg}wFj2juW9#E^oxk}nNAO`9qi(j> zkC|`|M{rA*EVP-aP)@YKgK3;kMEJl^H)A@*w38dIcJnlbW38k}m^{KObe4nUTxob?l^(J06U_dD`O9tnS6p0_Zr}FA6kIFv} z)QC>g2I}2);%Zx`8exbfT6f{)AceIauLFd@mp=(GT9kd|lWhC5U7Jr0&%SzmO!m?~stj@@cD&PzRY9KEo5hG^M6j;4 z+ID;338%lD_!c2NmY3$0ZU6gQG?tZnAB4mu;0}W}t7sn8MA02qN)9348i#|4BE(y? zcg==&@=F2XzRw&E2P{YW$C3kQxfRSv{NC6*SF%$FFcAv}-&|Y4+bjuPn)QP9PaZpM zvOjQci8_jpj~_BZ!FwtQJn=9S|FlP(X-@Ox@O2ez*NwQjqOgN5DFYg*`k7b@yb*d? z^y8-kwyEve4kk<^Uh+@1FPihu&+1doj_ZwLZ%U)ubqp?gXoZB)rk;VmI_B)U zaF+)gO2zdmX10Z%7o^3{|EjEhqO=dm`i$d8-g_m8uFmu9RaPT0fa)*Q{?d*>I2Hcx z$8p#Q)%StKh8MFN{Bf9iQeDDY$_nGm z{$;`#XL(zq*vQT=#28}17B7?-V4o2u3`jVYbMv?fTGUim z2kvv1--Vc%LuX{DnFDp`dV}+KpO$OEk53Mtr)Y(1Y2m9JNodsVHXR=QK7REsHqR3t zj1fW0_AEX!xjhIi^7J=Mrg{0k)QK)BVji_c7ueCKv$pinj~?w%plg%G z0)4Qg-Wt*QoF=S$0?%S0;aP)h;_OFDeioJKC^-AV>B$~BU+F9M;s^QXS%d8bE{$Mr zzqP?kW_5n>6v~kaigq`>Ix4I2?t~uJr5!fEe_c&%!1d#iqmQ|c*BWC)8xaz(HqY9R zquEwOz_UC*!qQ~^n_l1MOQY((!82)`#2=g0VRm6jhEsAD<_nj9+Av|&{k-vMAw^;} zv)XVuk29|6KgZsnGbFH*MR)yGoM2UoOb`8P&c<71xTxW`a^XyCsnA!i>n@T0pW6+D zNPOnIXG%*1(RoH9>IqQj*P-aa-IetmV`8q|LU0VYX;>SY#9$tx4OZGBuNzmJ+aFUp zk@0>@P4hC??<2Dv*?Q00J{ zbUiDSWNI1OUF+^>N0F{TOFi3gp)o|aB$ShY=tto2Ghzup9$M=J^rruZ;9o|@1mu&B zgCA!Xau4e{tP&W5cnw{j8%iixs|XdW`S_k%xidZK(aeVgLad_x< zb`430{Yg3zt={-aHc_Ur40&{K+1t~)Gd`h(m+QoQt;IRbrk?B57nwGi&QS-pwkwW@fKRHr3K8 zQ5(cw%yh*0&e3Ta8fsS5W5>ROuaxFr&0WR0bjdNaP!s6*5(hh@DyW4NjKfy(uBdis zo}q18&!l^rZ?2`DReZodGQ)}(bYo{B|LEEnw zxrVqW1L)i;vL;TGko@l!&(tT}%EV+_++MmxmWjTs*;;h`FeLR%drv9woX)!Hc-n?UcH(j6Y5jv+?Xtj>3-Ck zEgF~niIm8p)04ybJu;_$&|ooC`zZO?Q@Km>uER%~dh?wGoUVuo-lQ&6x~2p$=8P~&-+=axWCWm571%eUs^eN#Fj zD_fbJbDatYBlVdW(IaIOf-AP`9Qs;D>IKh~k#5lz=Ut&&JrBPpxYg8Xz#8;4Kd!&_ zvk*V-7dWGP;5Dl74CfBnGs0)`@{S5-hM$cc6ex+k)i#d&Gh^nrm?{E`_I7!)eMzsl zZ;txb<)o|l-6ljZ-m$Wi0}5qkZ5>K=ef(%4gqQ3+U2{`u4aKc(qmz;s8FV{hEY0Hc zk_>K^MdLb&GfXWY(`P@v&V$`yIhl^;&MckVL3vrusimsL^YA1XCi_^sG^RolOziPl zTfI%XK(o!55u}~9RSE3j2&&D5MNfOpfvh{Lm#BfphtGbBKX~R1f5IrxrIpGe8s2_5 zkLWRPXnJ;<46Uh|UN{wV0!K9yDHFf(Y?5~^jluGDQ*M{al)Upx(3A&8dYx}9>R&bB zkZAopUTwAQT;xT3O@O&c_9Oiln#r;w7As%AD`iP6nAZUdRIB-sS7sq zIqAu>hmh@RI;k~@n#RM^oHpCuH(nRD(ROhj8R+et%}SJpJ$3W3#ZOdwL41dAi<#K# zu1m~w6B(#}erDNr-E3Q;b83oCeChWg>iL)$zT1+kS(JQTaC?5sT)5RWbd%FwGrTL0 z{dit;zn#Q{Yj6!9D1m(vwp(u3`55{IedaW-@}e>mMb0tqY*R}(W%}Dk#+o#B6cT+{ zpTkrgg#~u}uu=P=D1sMuKC+be<;cbjhR7$m9QdEhMgQ4Z=#_VTiO2}TSvfKkfI4dw zrXTW1?t(&6%0`M-|t;?lZv5sf7CDA-8({2mw9W&elDq?D0dyk)<^`}34 zX1pOoe-vXoD=t-GjbR%r6PNj7ly{m93wmk!3gIU1(SgHo`{6z3+e6y4ykGTI`Sa6_ z+RIvgq4bQOguMAXbXVfpiGofMl1ZP39z7(?f|O4LsDr@_+uS4&^K#%`*Zmquqeve^ zt|hOV_y=F$XakPmUG`+yJcKJv*Am?t(2nr6neND&P>(~HZQ%{FN$98@l%O{ zE_8#w7POYOqz!%ztCw*32hQ*R0P1M#jl1q~O8F>Cv$JE(vGfgsDLCl%6~}9!Acct_ zmDeglVAROSP7K{8aY6YGg?p8v2Sc(o+{9cV>qVK^t>VvgYs_m2BGea49SkDKzHT*= zj#d9V#sq`Z;fBj4xslO7cf7O>qk{M3WpH}6GS#~~DH67vRPz2*ocYnKJTDDuBhFLq zIDE>VZm@$m%X3@$It}{o6*w)|Utyf}XVbQxY!l6~Ga1(@0V zODFaI@qOU~yl+eqHWM~dTeX)azxFO~dVg1FBxL{w*K^`Es^DWAyP8c}=2nE8E~0I$ zNFtrqaYsvw!n(^e(j~~pD&mr7<*ch6=SpPUa^i{rda6vjjXOvpsn=Ec`fIS#zT+F> z59B?`n{JhoXUuQ>dptJW2I^15j-tMWJwR?HW@kKz&CDB~t4FIM2_03Te#yu_gut?$ zJ6x?Al~TJph)^VMv?4FhDvr}L=;90sFq3|jzy|S(a`syLewktJG35I8=#EyIM%35A zLHnY`Geq42W-XmD0aJkRP?k}mxAY|U{Pi+&!cr;er)h{NFz>KrgQs}0emhOu(NiD(g>HTe?;<_}}YCp>pz6kDU#nF->Hs+1} z8NHfTxKFcSz4Ffsvkc(mEq(4K4j`z$C#{J;K(qZOL-$_z+WfRjzmYu8<>2-nk@@ZA zT)v~D3{%kaI623OU&y1Py&V3H@~rgh1s{fF!yNLv# zvglK^(mFM97s`IA;11QaTqX3*7cH+9?heazg_*t3l8CZwZ1#3IbK8@9RY7{!c=63_ ztdGacg=B2e;+V0>8e=qu6q)*wwp065KlS6u&p2g)n43?Ojt-)EeK+*y!{v-Me7dOq z<&hW$T+~uHX{SCZAP3v{Mjx<*c@nDFGo|)>=4*>3XxSGlytvJLU;Vl)>@;g|mfLm~ zzB@MwzoewxYi+NB)wLof+6$~EPc|*&IJ?>C75}i&i*B|_p|lgfz+|t#abMj*Dd{nf zbMZH@j@6ufUaoIWk^Q`JPEDBb6!*nCGIuRl-ya+C2nmGVjGiPNXPQQyLW&0R;mpbT zF~b*f*L(NHQkdV;cdf&W}(=pnt%Rd{MPH2_zRp^z22*J zy*}_k6I`I8CTLz$_1`*D210sGof_-x2jT*&YUDe9B%?G2m(f%tb%s3*c8}uzg$$Ak zI-1i@xx~1Pt`%T?E>4;(=1))hmg=V`aL!Sk_MVq^;=2;ebP01Mh7fxvSr`;!mPbvV9SvpPZ)VgA%=mm+ErLcTwtWBn@inXVMA{fm)NFL(Y|bsG5T3KxS}!t}7I73`XOYiX{m^c~ zz--orA4B&r3oksJ;1m-J@P;0N$QgD<<=|fM}^$H3@WUr_(RXU|??> zjGVB&sMar4~`JhDmyt)dyKY8V!B#aTDCtzYWoOu!jflPK@n60Vp} zKo@MS$m_3Lrx!n1otjJLqHOYJ6C$ItQZ;N%EkvTLk|;`!0E6vL%G zOhRsAO`{~oI{S8FiYaRuWQmt=e?>+G6-&N9GWNfGT+J9@OlKwbsr6J5uyh*>F{7!| z)6|{)`4dgk!yFEmK4q=gmwb#Gp0yeXMY-M)*lJq%WOr$H9P=0|K%$fe_ki< z<)G90E)cblAPJ~SI3kTj4kewJ@fF-iK%ottm(#B#w!(Mdsn)T5IhTdAVA5h zH>3X#oYF(JVg7Nrm+;SiW~@G-l1PzcyV65$#hy5P(3dMK!$g-tF}!0N>1tD4Y17;9 zEZrX9Fo(TD#iaYb%ZN~JrcRsQAOGEt+wx&uJH@2w6RY^?9}=Zu^ixy{yn7ZH zdI|U!>?Yd4-Fc}G_DgdO^q%O7&oCnJH`v$dKYaa(T?XYmAg(ld<9t?j-psONLCnxS zh<`H0=zg%Tgt%Z-ynz_XS>K=oSFUM$FQfw87zMc$jKcKn=nD4D53#t@kIsHj_ln*h z;r|yxi$0S<;$~%fm7kXi82oX)bPW}1o;S35x5gBkv<`QzOiD<~NY2SQJh>d(Qozz1 zxc}NCq3K8;aVFKU*XuJ5PeG5YkpyBGck31zQAU#~iAMS=-c{!o-|ow11j(ze%9Bg8 zghQNd(~)ol!LqM^w|g6TYwhHfRG|$kg>8vm?Y9Mn&L_-4YHyuDn;)_wK1W28HK6q% z8DV=O3SIxyIB`0bJ8$PRSKyy&+}G?5Fh+rX^h6&6HQN81VEbx%rLA4-J880(M5UQI zx|fYs?j1j{=a+BU29CCgp*6KpMA=~icF%Q?L#O#lGeqxt5D&J)By~Tf;5IG`lGx{B z=e_R@6)AQ}Q#n4FJJ_Bd2-68KQJE@GX(-vXpSwwAlQ&@ftvl=_Oohioo@#v+P30wa>+^3* zOl_1G9vTh5x!QIvd4y%oN_?xLfLdg|?6cZj6aU-HGM{FLG3$Lqg=*R<;%m~xMN_v< zdh~&L&QQHhaaCKShwJg^g(v7AjEk{9^?0NuuP6?|^>g=}>%BPZ_Kz|lX;FW-DTS*K zu2PYDB_3NhL{Kry9kBno^U+p-H+^T+l2FV8UOS_@1QY)BW1Y*rrZ9#05n2%zZrvWi zrT&`K^!oYqKYrmyF{&zTj6AGqPXVD2Uhet!FFU2oMUoF?pSUbVXo9Xc=JO3!kv=?E zZbkkgGj)F6PruuDaAQLZ(N9TL`#voYDdwus-17aq7vlXHn{)NRZLGNaE!>EL;Gd5h zob<97f;UR|vrjOuj=y_Ja@=R|=ATc28{g zh}8rmslC$?CMBkS9NUbsKP3LrlABBD!zb@m01ilZ4LAydUrk#~aWa^?<#B}zosRv$ zR^%VspN%gxJ$fGF+3U=~-Z=c=fn(-BX-9|c>-)9}4|^vgGM)-%r>2_(t4Q{Z)#yCU zMbCAK_Jwd=dD+3@|0K@&mLNzOximN}BKI$w9c8V)lCSlhenmTy?c5sCwY&GAeRfUd z)Ap6%{CUGd)Y+WA&d{^TBqOD86RiqWOqLJz%Hew>E4GQ;xyb4Xt2y|f%b($!{y@3= zS&u$TYRG>Y-;CQyq(%9 z82%o?oU)Q48vx6|`_aeBaLut+5xgjcwAim)lX?IshWh!9CAPDOfzmZ4pPrymJKeIG zPmG*jnEBkPMr4fbEwWx@Y>yX!8rGw>tSx=%>+XiQi0EjEekqGJ)WWvjw{~{H^$^H2 zCJy%8l-b2R$U#8_w4oju!5?OvV*|A^nYQqHGpLbDpIZ$iRv0 zEh#d*Vn9`+6_lr0pj#7Zy&OLufBr$l(%{Zu>~tpG_kI~7F78WH**($of^PdM2ZxdI z@D74I#rs4ulCwhIQ>j5TnT#%v`pHCfRYG-@L)mn+wJD4%hKmFA_VT{{ksTbj2pZV& z9DT>%;wt}+w|A50k>vStSvL8)-xLGxh!l!tQtH&^gfSEpxHS%8G%8cI8HYCd`9;qS zcSNIm#V;FwCpKOz6K0lV@u>k@*bIqw4Lw z-;11(_b79g(eXc=ZW~i9*~>UDJ2Yxj_-Y{ZmD6xDZ~u}N2Z0h$OO{GMm~fep2Ncfp z)IM;Nf{O_O{m68(e34O7WgqxA-IsvOhK8~EE!M^tXuVH-=c2N*?_ z_Uq6+spID+>hK}p`gR8xajr(C3X)z)8_?tKKW+@_x_t8V70+#2f4?UR{^_jQhVG@p zL7DoWl#&WN1~jzC2RHYp!>)L>qdDC~r*_zSxg<3GSq8G&osT3FsTocVQ}?DHPQqkR zvfVWKjNJ~a$|VLteXjAdU4b}nvtD4gDwx%H6Z(t<9;;DXd%r(j6hN_~vk)6Rbh{Sx zv8fEB;;r%3cE*{dM7Ly`t1n)Y4c-D8N zWgdKl*{yL@klA=ta*B!P)l`KEhR4%uqAMKW)F~Y9;cl!K{qOAl$RF&-DJa7R zns8lVZW}jj8C2nNUu!${sHD1W%A{RfB-}r@;E)~PT(a1&Y*$|9dtowsCE~P}&NKV1 zL)-x`-l53GnEtZ`=BL+Z=I|%;(L5Q&a~-dUCI9UH_Fh6f^Iz@pY{#@}g1h_j%sthX zxwbxF*bHEAreIo<^+nfV%C?mf4VvQP&;3w$+kWS~VJ~^`6~gUdo>q-gUJ>e63nwVC zMxw95pznm!(2IdJJiv~-fQhqEB;L|ksLXIB-TH14aqVMD=zMlPYYehOD&~~t^u&|$F$Uz=RAuFz)XZf${S^Nw6_Mn;Yp6F&v zFN!%mS61iRj5N zAvn3r4N{Ox#t~z_`;xHHFM3j^UYVgt{X>0ZCvaSNtQc*w!$u*|=2 z05CO{!lXCbG|{dPr=*}G;q^_YjBm#6YIc1k4A*i&scy}7uSGesp~I0@mG_0R{M-g_8Qltq-^i>0~V zYUWWZNke2M&|u}OP0|rMuUVq*X1*8QTu+|$Hj%H{tSde<;;t9tyRao-ui@2DKaHzO zu+ctCgr{7H>CG8P#JmfzLv%a2XLrg1;<8AD_?WC$Xo8XkEXE1Y}*gv=7MRk6)u;uPF z@wTt1;A4*F{XNTV=UETv?5VoV{!kfWs?yYUyz|nNeSUs&UQNr5(j-{ZAkNTQKPgdeMImFLbHu)$WrqqjAgK&+) zXY~{WX$0Kw4)6U4o6RW@Uk)7K%_f$rYhy!Ot*V{=4cds-h4vX(=Ay@JT=~Is&$)V$ z-id5=!_<#{4?Mq-FxW8O_oG37=BHQC4`Xwq^+}iUuycI-y0U>bK0aAqc-Wsb%jVO2 zEh(FAR_L3yly)9=9i8#3!ScG++R7S^(P?;&->MZnl@Y{K#$|-uX~N;36K^qK|1?-0{X)JF-03EnJOGmh)H#yw>{Fr^ z(01Z6hORJ%F_OAwswt=U_&+GG%@k7TON8Pk3HrHDv9&|vD53Hs?A~#-VvS|(9|89w ze2tp97734rccrNe) zl`K9;?VSv2k>I{2wtO~IR8hl%u@J0ycu25i3Q;^gm$mcDaq@^$PD@>h{LLPMgWJl@ zb=*(2|Icw-U#cx5Nt-y3Cbtoycp~t_%9xOKU+TG7Lf~0Gb3fmxQ5`+*m=6w&yVI-6 zF?QvF;sc!;l<$YGC7eHzt0WXg`mLKY78Iwa?&)>ZmZ^k}x^EvxWr?&E)7iEv!ss8>2*OF}(0V-^Fbs?)7xut_TDfqCWE3WpoZXB90-#U99avq;8 za?eXOXRFV#MaSXN6pFN2#6y-RyJbH)>z=;XCM;$i;U@L9ByDT&Y_)>MCJhiG-PS%# zx*3tY4%%ml>3>)A@|_)m(6Ka!hzWOrgF^VV;95n|l{VVij}_|uB=b7wI{iAfM1Lp! zDqHN=-f8G<;j{^IY5a*?lHSh??lqse0j|2`J3FEsDbLZ2UdxSRG2tfnx$RFWEa1v zr#PRoWLqZZak%>{Bp6N{0G}4vWMDx%{0q55*`0kGCW{%nl)8Ex32V1J^tjGJ#Cw_S z3Pq=jUQdsxio!>?s6@2C6?F=oeZeRxws$<=IOKG-)=5?zSX>|X-d|yEr>GULKK&AI z_8sk0^sr~BBC@7~Hl~>B)I%}F8-;oVNsoQ{)sNiMUVBw`=n$_ zJF3yn!Z)!kVsD|M%(T$-PxG+&nM$7dY+d;vO2WNO7u5hd&$Fp3n9LH>J%nrXZpl9` z$wJw}@5oS%>RN%jKO{ZpoJ#C28uDuiyUT92F`QfW6a-k!KdVzs=NYTOY z*GhU1ODjriRk#n;_H&)C^Y5b@1a2544qg7#x>th!WLFBPkSwQMt{!(OBBP=84&+RrkK!?eX5~RblPcm#;?3cF^3_ zQ+l5z7a<{+OQGgr=J23n^IW=~QScY`5uM|6;}MkFvU?(m`=C?vy84n&ieSFv+?Mha zYb_D2K;t>wT2f;rX3yUhZi?bqsf$F3$;2v5S)>fFj+-}Cwba=q&d&FpF-NyH$6)$! zgpruX{KaDi+{QY-2yQPPj#h!IAeee+vW>uEbmLKpbp^xAb}=s3s}VeO@-ok?(CUW} z?iR+fo)r#HdxobJo~-Rh-#iV4&AIMFkZ&?vpJZRYf->oh`%zg-J|uADfvM#i7zHN8 zZgMbxF~<&0oNiWj2=k}5U$6a+V(q!ZI*!PVUfR=l)%Hle#Ye?4OkWoqn{Is`CbOJ| z@MDGvN!_zVmiDP%;$Ja|4waY1vJM?#ekmTdUA$k&Zap%)%x@~Oa4C_mXyk^nRe1F* zu=+)pabK&lQ}>7pM^F|KzZ8*jQq-%pFOqm)r_bPasr!2t+_KUwJNTM&DDedZA}=?| zCK$8ylqsgSEp(hE!S~64m+Dr+?U~pj_B-Vqt0MvaE7mZnsTKmzNZ%Vl$lShENZxvX zXsOt?xjyV}c0gz)Iq5m^Lr9Vf8D0N@vv#hPc&BGMjbQKvc}U%FHX{<^&(v%$`?0e) z;K?-Kd$cNU8wtyCX*8?q`wlsLUFo|opHCda;8Lxb?k!|K`M}S+;M?SR_NspXrytfJ zexKA`zWU%jc~k~ZeIgxfK&V1TTemg}1 z6ML5$R#Z-Scg?rFCMf2PRLoqiv}SOTmRERvm32E!F~!dLVR+|sqow+B(K|lRWh?R%lWsL|Ye=Wha;8;B zyX_^(X>GI2*14ZUYBRdd-e^J8o;>5 z#os>)P2u#!@-A7n$y#Ni#c5T^0^~dC?wC|f!*tP{4+V0Q9!`bp@Wcq6lcs~wLk9{^ zm(y6(hfo2j+cw&MfJCS|t86*Lg?TNCAPnf1HW#*73&Qmg3V`tzTKg08o8WtX@9=f9 zFr`l(?lce6`seWH`IPcgW8ycKE%}V0j5LqFn*h0Y=Ty?46<9kzw36r_*@?G}i++I1 zwC3SgyTc%iw69*K_M8Tt&})GPe@gmZbeY&EJaGrwTv0U$R3n#_3}85f&?v8oeoT zQ2X)n_j}TWXTo?gDN~&;nuUp~!@C8lMn(w6>hXoFEmF%#r#mUx~{^6yO z!-Cz>$X;E+CF?3qj0=-zmlnA23)W@w8pu@J>>JUK)>N3fXHP4UbA45w@4C9-TwY$* z)Attb8vIWZUK~FAZL;0hh9^OwGkH=GCP^}mc+bC})2H){tjf?c@ z`)U3m2L6FVCI+%R2><*cc05lQ!+uDNR@>a;bsxk#$*>mh>3<=b;}{&6d*hh0f@9m$ zX(n9ayFE;Exw-cz?pNzfC&I*z{CH$gI+wBxs!^`7mFGq-BH2Sn9+!x@>Lm}GqGhSj z6u)~D;s&`kI?AQ`=xUwz$&Kih!zr^iE$x8lgBC~wSFg*Noa0pg?y$3>{(gQFbcifb zf=2I@AyZiJJVT310G^^!J7=c3BAERR>Q8vZ3tgF#`;Hl|^BgcEHjjEP(x+|7TzT@O zrTX+UxK1y`>eLVQN|K^pnJlSqv#be+*4{pG!%;-6%?jw6wIuFwzYo($WYD5<@o( zIn;Rf(eLkffA`t_Kl?wp59Eb9=gc`f)?Rz=leHu1P;(6-kS;3#Qj|wq4 zZ{v6sI%Yi!jYkH1_g4!PE)`<7aMiQS24lR=yaq}_F-;9laYvSu$K&o^B$-Q;O=NJ@ z>knUx!UKjf9;o_vM{_WrjW-T-yItuK4OG4TeL7y&;@G~{J})b;RZE>AS%UA|rdtqD zpWE+bTTxXZ`^*?rs}{rex?T_~w$k$Du3b2faT|v%=GD%7vs!NIOLS*o<89}>38&ME zEE&m81fFB0FB_buwY3TryIYubi>Ix%NBi>L;dsifn{OPaG&rX-GqOCnKcL@0fkxF@ zQvXt8h^MU(b8g7WzrAnOX*4mx=RP@RWLMnY^BY#u8*XMXZRGnng5}se17%cHI=jF@ z+8>-Q#}dR)WIZY+{%dmCm#E&%IV+NL8o6`Qx&4mLRK8<_JSM4TD-@G~dxQy%B=xlf zDIM+yRa9wqadWeMOl#u}M`p$DyJ*FR(u(1tyrGKo@iSv9LLrZTJvSD2&hUjZvRKWLD z360M;l46yppA4RhwB7GrY}aLNV4EgunIbsADFf$Dr>YCrlfcaT`NfJZ7CU zto42U=Zo^1sT8v;U4gv76x#gTYq7DlKRdf0?2Ytn?3FCdiY$CwsWj6Z;)Czpd}DO~ zw!tta#v|^RWSgV$d9aoaJL9xxuH$5X!&*mDrYO7=g#9E8W(=mzuFyU>qKHwYX#8-U z>OodCG2ovj%j0npdWEMYfzei*Ac3G#&ztPVF);;EYW%X7rbpHDvJE#tC=QzB@Sh$A zD~&XP6=HsDd;(eN@wSNhm|E1%_ID533`8fpJaD&}0S#ijt?OHOE<42T?jX`-s2=NvO|N;B~E>`+ZD9}M4? zk5W_$e-wP~aZF0|OH~iMW=XqN#;X#5eNTo9)HkvvQ5o;y?ABL&B=?KKxS4nOrU}ya z!}RVihor-zcx~krF4lQ=gjj1Ki%-)2_x;VLQHQe}Ra!Yjv1ULvLsfabJgs12*XZ~~ zdMRZ>E-0rx<0M{qPF-9hdc`_HgHR&VHO%%vM^1B%OL`j4|Kmgt*PgTN#~MeOkL0$j zzx7KwI@zzV-|I8sE~7u3?B`B<6Ir<>96GYN$~wqBAMW#$zBNowarDN$PRVMQZwA=Z z5|G?FGCNFCRMRh2gJX-=y621DSrqT+<|#Iwcd$Nv_0G zrYyfaW+|*K^Qy1ZwORG7%c;`t>)E*^;MP^X>UeCCSzerQH*R7Of#~Yl?8hEU2ToBo zUQzdD`$ALXrGNhc4cp3 zts(UDs^K-3N7wGVr_`8!u>0j33X3LHpjbT8Yid1e2*s7qo2sh4;6 zyVUHX%Q&8hpMlZ8^@wbd4j5s)Ty9;*jyC@;cv*KrW#Zn^>=}dPM2DIVI`-MO^pZ-q zQ@6iKx;+XBqq}M0^b28BLu+f|9aqboEYkY4KqEF!OW@-e@Z4wwM^oWhRn~QTmWxi2 zKNqL>-zSBtVCsz2Y_yQ(Y1VTFiLT16LGt#>F!dYu-fHf}H};*7aC+Yvww4?EbJYW0 z9Qkb^ZGu1ay=kqU8O*HnYV1j;@Xko(~qWT?7Dhv^o%O>r=#;edx^jq0V!n-~2!1eCSZD~W#OV|~q+wTp@ z8%vZmGS-IeTys)Ky2dIzCAG$@v59Y_3K4?*Qs4U$;ygz~)ec;B)kY(0Cr=~UPm7x7 zQjsS6^g~U%Lg+l`plOFMRGO5ym5dZ&$v7tcGyQo}Wy6j{y147lPH~~lhqir&*d^_2Wc%>s#em_~?E_Q(^DH7z(p0`K!K?l;B4L zRi2}t62)BnRN2AT9)UKG(FiA#;F)plkza?=-I=n^x9-<*T$C6R8rFRE_97{f!kGro zX5$+m1JL2)B*<}4Xp_xkw0H~u!F}mPB9H%d`gt6oB>$$d?bXpVwy_!ZPQ;2>>&N0F z<@F|2p^d4+@Ce$p>pz}aw@&tg8L3|RNmDZtS23}xG8_O}25~rwr?UIc*g;jZ$u%#P znbc=UL>>rRkbO3DJ$|$l+0`{mvRxu7I@7?Z-+H0*9uM%`TukQN1xvYAMMOKf_%`M6 zYKtTVCxW<`X*88JC}|SR*EejPxggh;k+ov=g@c#sW*SY_VN`P34MNdQW|*_@#YUEv zSsoMPEqoPbajQMF-2cKb$}l_4w01^R>lxuUqYp538TkumX%|hVL1qV;)lv9nOJq>Y zawRR`3To?H2|28g&<*ihSf>8em%W|Za16Gzb^%i)K#{B>{hGDMw@GF!Ul6T;r!U<4 z{IfR{5h9p-x^NATGmh;~VB4SEJhsq`7R2W#YFKAxQF{@^N>xi$wUYHNT$d%_4JgH4 z7iAs}j%6FR%uprX^$9M?zC@q%O4x#B35y~eko@ds*$lFKyOxw)WxA$L1C#fPeZD=Q z_(Pyg)sw2lF0hFljt-^W{Ax!h6n+-5kWOFzX<5rqXf=pNqJ~H?L2U38u6%;+Q38f1B^8b$8v6mov)PZ|wVGFnR?0enEDA z?I#UeYGF;~(X*M7-5@y5*XHu`qSwLCw+0*9em(16xJTqL)qMe{XNNwaz_O%=3G&gN zuwy-O%8N#vW8r0E3hvetS$G3ZQG(kmN{hGCr<_WFB*=oWogxEyH;=z!&AYc8buW^P+$KBk**Onk;J2=05erhIF&<#1%=Y$XZ(+(9YF6gVxd zm4D;)F`I~qe>tQdCrp~L6*FU;vvpBFMq^;|>#B!QUwp8ARDGji)Gu&x5(I3pm#|aTJflyn^LZQ)dOOyY?$jdRO$2V-tA0tU&7fo+n+b5*y~&S4J*JAKEj#h3b)ifH8X?p|1!9qm{ALJ zFBDxFyxu!|0M4eLTuwiG0_^x3@ix35Hfp%>hu2fVqDh6A`P1~ZuGR44ai8^5mty|O z68NV7wwv8dPTyA(;+0vI(vj=5mGMt@Zq~&<3w{?&)$NtCXzf^c{8hXAWsy8>Cfe4z zhBdWecwx-pWDf1RQp7zj_GM@C}hZiR%^L&oUa5Zo#FTM9yAj=ilcN z^uBNG01i=mB3l_RC}EOUYq(kT&1x!ZgU;+9TuI6HqqCr43X8bVOq{fMLNiSkOlw?Y zhr+GSTy77F?iG&UQhb3asWP3lJ$&;#EKDpyerPCcG3`pB3XLeIO`BJ3Zh6H}!2K zOpZWe#AK6m*}8EyAi|8)_*~OjHRV+D(o%KLp+@%z&<)Yk#@-pmaBa)x)(MMBccLQS zONq95k<6N$JF%_0!8Eu*6jvD4zkHBqRp%*p^R{pKmm-|9G)}F_(vBS0f!|TvdY+A- zPh81eF_|(1Ni*e%+kGVz!(QfCg_b&WfHD8z-YMeyOvs2Q@~OUNt^q1JGhADA=YA}w zq(j;+$gN%ce86x9r=7K28&bNGfI)ADp6)m}xxPDGd|5m!wQ8)=vSUv74%wpAgynOY zl4LB>U+30i)m!Ie1$%Y()(TcUqJ2SOf+sv_}u$Uc)_bPmCD+-r|?X92TO@E|Q#OasgatS^O^ zHdZ~bVioAWfc)%L-TnD{#)zQ$L{}zLJx9W3arJ7!U>j^!o1;4UH7{=LSrz7UykF=G zF0`rl^=nGZym>Y~UW;FA>fo3YNt9J(u|T&n;uTw}oN7*si%q}FFP3hVR;E#J5&n9` z#V$!bpAg})03g_7=IyG$DC|t{f9%TAV7OiJ)_~CSo_X+li~NhtK8XitLtN@sxR<+c zj$78fk>^{ZA?1=YeOj*t6Rvh1WW@_HDF(8^((Zb^mfb1b{5jRiz&0WwJu^k`wxDWa zvW3sGb|(qF9rB&2%9sB`#Q6yr(vSnG#CDPBmtNoOmPmqHF$27&CObRF+1g0bcGKCK zho>@cXuc-m>Z6R1R9j<3k?JwMhNQu+5!bqr<0V%x7@s<9-EU(NJAK=9t2=$LPCejb z)QQMekwbc%Q}Pw5*JoR8-C|^h>K|cFRay|W@ulB+<}2TGw7iH@j?PQDn&SUs^bJmZ zy6nJT9CO9|qsN&J2}4gtobMEq-}H+9nl#_uIZ zJ7^A|Xl+(q%13kde{6mF&i?50sj)LT3yr8%axIdFJFV8abo6EWIvZyF-eNQ5q)5%MM*$U2v5~WUemv#zJLS@3%S~G`>PzM|1Uy$8 z=*jvbACC5qO@-9oc*0lSW;x-)t$b(90u;fyo%^ zJ^M+WIPtcvKywRwSK9fGO=EwXEtNJYW-L)c!HF?811lHtR_xc<NfEs)xoQ4KR^vavoK(8{ob2lA zj#{`B)ukx3xnfffKv_iq!Y7b2on=?IYSg8F#g4tqxSCzS~(b|XV=ImHW;t_P%ZmjD_^~xwC&BMJIH#h zmilpQTHX52hQLAWw5_qgs}3gGx$?e1X{usDE=sKH*L*(sgw(V5El=}W&3FdF0v;tj zviL&xG)kV@-nx(?PRH3=-XS(iGjF3hrxY>Sr(%oFOKkP;&F6%(UAQNZn{<%Hif7%t z!CKK!w2qEy_8Cf}ufmcAH&TtQ=xP029z}eO7pPaCuQ^7rU!?OM#OMob>YLCt(&3~r z>Th7}++gr^!w zQhb!k`g(>r*k*%yo+a!kLZSM$Ymd8p>Xt%VQe$Cu57wZaY^H}xCehZ0JHCg9(je$J z?0dI;$JY++PokYG?oS@}aikQC*yns(K@1K}AQ4A(h5CZlQ%|}CYVEUR)v7pidU`H< z?~c3Z>blm2E?9MI<*oD2@KKB}(Yn4q5zSdi5SqLdl{LHx?^dXd9&5cAta;{=dvw59 zT@dx<)u?IEQMcSY=(RCw$scgiUumV%G2XhY5tFN`ZpM#if1HuY368%X7Bw6b{nC2( z87Q8Q2MAt$Hh9WzdPnDyS|(s z?0%5sA%0XfwQ&c7XKK;ZS=P&Y&fx0VV3;|Q&o)mn?z8Cm!D<=3ROvD7GD>OfPTO&_ zHmpVBX;Xk=A7@K7%40{J0k>nc<=vW1tMEpU?>bMHfcTH|`(JhZ2<@JB_R4d;V!YvH z6M4uYbyNBcIE(S55J14w%RWww0Au}N0Hf7uJrY@-L(#QgpLJ9YGNKgrlR9u$XEuJr zmIlBd+-ez4J!5%{5g;5$miF&t8ssLv(7)$LpB)4bQ+T=7nmin%BVmN;Mu#z8v^|k@ zi*d8{_L(5e`D&_f6Xfh&9MW~+(4mF2ViM=6EJo>{yA)s|vAo|uewggCx&4lHy+)*0 zAS|gXo~D}?D|BFZO?_e9==g?iYn!OY7e~*o>|~SuH7(b1&pjz`T>e=tFCQHL^$+Z* z;&=R?Ijm{4aaYb4`+nZ9T%2idPaM1R>?*S32Hm=+s0vvIlf|;})QoSS?^d*z%$7w& zQ@KXN7AO1*x2OXG0j>Fn8ExII)CWOnbshK9)J%y?G4FC zS7J{CyS5+PR7jY&885R_t#fOCs26J~eQx)LwExHyv4o27es4NC-}P_I+ccrdgN2zJ z-eF<#q_Z6-T(8SPEsIZ!8UuM5uFR>C{?Yck8|RZwnUSV}_hsfz z3Tn`?o~hiS)pPhcDlJC{IMC zFFUQI_R3{H{j>%xt{J^_Lc&;N)utjlPlQVs2Lki5GJ1e zDIxH5#vs1FzTEn}ozO4oqt2V|)Gv6wtM^2xyYJ>{>gE`y(AC&Ep}cjE%A2lx<)WzW zH8K4G~Ugz}mo5mfkhfQ#Bm*2MNfAW&dR3L$FMM zD<=e~*>=E)UGS6sJ$O2u37E%4O@tQstJd70ZUQ2|=J$$9jOe|p-l~Xq{|(dsi)h=t<^W_&6c7DJ7*If0ff$K@ssaG57Vgq> z5kP@qw6dT|7@#6SwW1$o0{wE#GOW@Tas{pYL0Xm@vrgH}O061pS z*cIhF0I&TF% z?B(=8eA_>mQlRN9ZvK`SV3z;zmhDCVkfljfB6gu_RZ9Y5HBhn&LWj;Ua7mIOoih-| z4zqy6KzsX!4659dBOE%@>Kx#x&{INMJD z62N0Y1uWDi5C;&cUcm%f9LDzox)4F!03*LbM*I$_V@nhqK>9TI_9{r4h>_vq^xx!4 z*-bs;G|2I{8kp6Fir}H_P7}OiT;eLu<8*g2+iCeS3;bY57wfS!(IE>{3=d+ zrN&O?f=00loh)6OoQiSN|7g!zQcWZbQ0nco8UVm{wTIU|8=%ksAd$(zC>n{67zRYm z|F_1nrovxsAjV%~#HpMMGZ|;JIA$B$#~*Y|G<8p$PW+l3sf6ON&JRE2SZ89iG@H08 z+QMdIRQw~#LHT0X!biXV?YLs_b=XSBMV;lpU<(1ti}LrIE1!Vv(^R- zY)=64%?CVDAQWy26~g?5+X70w1>L|uZ7V}Z6)Z)0B8Cmfp)8q6C;e&V+V6LE=w8vh zUS`pX|co^`=_Ve6A<>$YhW%!iBljqyK zmR4y@L*$pMP@-vYXDyc;L3qyx10;KH zG-PQ0&YpI7L;eRwKw%~PS3|@8vqdclU;ht`W|rh(v2`;jgW*M(;gg_RrdTlJ{HQ89 z{&1Hkf!!NA#Yl1xH!#hj;kkYgtM1#D@JYzOnYCdDMNMbs+z{Q;yF;LmKoI7fH~7ve z&vOvq$)So(@LJ9Q{p*+{xeQHPNPQhv{)dT64)Ygj28IN@`-dk04t)*+3+3c|hQngP z*aOsa@+Y%!t;Mj5Fv~s<1p8tQu*lMXL>B-CoE<{V0@NnJ$w#>703&5Q9IKTCQzn9x z>>-1t0SMHZGQD>!VIRm4>_7q@P_F-4$Fx}g$#>vAv}MkNqAoD)J%|(#FzsbUG3|zB z0JVoe*Wm5N035WYhhZ0bR!BPn7)S_P90HR~c>;KSz)Wh~0hsPH?*LC>aB!LyrVJ`I zArkK$ha?WiwsXKfHSmx-{#x)ONJlYsKowl)$U)!v0mKzxQnyPC0{%Fkl`@AajD-*m zh~$t^0y|(Ivwr{(UYG_kid^Fyy|&h;0jKc>Go3`6wW6od?(=qabebF%W)aP@tW7s1 zK<&mkKI}WJMEcfSoPjzBns|twm-RCDtjQwYw2OG4*zG)CvcKDj9#N-pvzlu2Asdxo z^yIovRQJ^R>FtoaT?v}-ho2ofE=QU4>4h{s>l-8df@hCuVcZI^qRqHQ9s!SQ0<{0Ep^H>x*-_5g0n)@c^yt^g&{v6efe-oK@+-UtNTaFDpFq(~ z+O6@TmvO(8h?isu_8@T9`cbt-z|7AMi^`hN&9-l_%foj+5pv3YzWr*IaI(?tG+K&e z1?)6@uE=5{uhynoXx1fYHiT1)-RJIz3i3a5-zMVrFNGUrVPV&IG3erMPThEhl#+`> zNR#2?l(Zao3{ssOrJ3Wa32}_U@-5S`YSf<1v71fLF}WKH(;@%}YOn~Tf&ueuA<_ft zOE8Ndvk!5T0WrMyjw%mi{Mo4iBO8E^!x)fA`^zvEF!8`-J4Y?jLtZm`#*bEDGCo8U z5Vl}_BVbG+fdT-K{8*Kto=ySu^jUyY&h(?IU?I=-yY?yxjJpspl*nNOt>hX4VYC1z zKk`R60hlvrfraJeFdaY!J{Bz6VBUjjzgJNse&k15q_LYG$}kQXfHs52R09KM!%?9D zdRBnRhgMz)+79t+safOSfhik%2?&V*L54OmczHFE-!({AAm_uGFgo)PL+mSv&aNUy zcK{iXRbW>jc(fekE&v1zRHoPmFV~bI8H*0lmboAUDGSJfrV4?76^Dge}zoIK>9!!f-nXQ|ENu9p+L$&W`Ovi zE94#W0MIsU72>(Xo*_0Ikb!DL$srF3SnYloe_aNrK3_1%84d_-N1PgP9KXc95&-5cm+XX%eKy5>x z^Z-nK$RC*81vYd6cyF3RX^6v zBSQi;ozqIBCf?HdpE^It=30ZLc=a|pq^hs}a7YJjm5lRJV*Oazw8KhFvn(Xg5L1QZPZ z%oF&~vq>7H&?^klFN5)<0yKGGV!+Gl%z?cc{Y#LRgVsFA*M@-)BLL&x9XtlGE}5@@ zc25j&cLklmat~Rm?U-}tng>OfF|zIKm85e3b?5Bsx;It46oCT(38PX$ddzn_Keak^V&keo;a{>&z4`=Bi z0hT`+8KA{wM6yzz?0`o7gJFk(0WCvJUfy6yki|R+Z9xVB^AipueZL8*M<9qi`$6$C z-b)nZ!`MJuAJE&ZHE7~962ca}f6JMZ{0P8I;ef_Jv}lq++Co6C2IlY@=>ZLC3%U`D zSO2qix&CrGPOixD?`}$i_%T?N^PDr2M4(QTCzVv?qt0He0B(SXGw|B?au$IX1KN}W zNemgSMer|~e&?p2?5Ip8_j1UCAp>KFMdZQDW1-<4|HsxrmjL89^hCf&0IvtMW-L&C zpu6OpLr~KB;ViVBVqXTf7NAM|&ad_`bfn5S70}W7U zivO=)k+83k)3@Vofp3R3{xK*lZTJnlslL~V2Suh9zz%vAmJClSuHAlSTEyQmy+_gR zr2`Bm3Hb4D^yCxx)fIm)eKqD3WV8Xz$^*Tpg}xn9CO9(u8?M^C<8tX%zAiR8=}9&` zHU8zfp-*>g^Q5<=@fCdgzS7to=~JK}K2mrx6AN0d`1+?_=q7oIzhRqA_zFT*X4LQ^ z9@ys*6sIz*ht{E87`PNuzNc0uYnw;z>G4R*wOH1 zkykzcz8IJd8JZR~$&J6?%|4_iS_7^|UH#mS(6|A1b3On0y-Y(vjM%-ukA}KAl*CRb zf^4V$*H;rIDH!DUoq#n5g&<^d&Y{2K4kt)GO$EB)0{?nA%x{?VKY!^5;w1!0@LBLL z+UYn(P`aPuv>`pa;Cs9Y?w}V)4s6~fOdg%t`aiFxnRkVM|6gCAzpey5IMcuXSGw#s zZ26yy5nMw+jsonTLCdsV{^w;sAln;;f5Q%dav7rhSX8~3{@*5N{5iGzPp6<@q<8*`URVwV(M_oThJiD^y?o!Yrhk7wup|q90s$#9 z3oDMmIRc%b-Bh;o8^$N_w;={{BfxS)sJ5#RgY!vm>Bn$l0?3T`g>uZNQ^ix90%mj*4n zSEw8U>opQ5B>LB*X@TJko{jOJXKVUrD*xYzv--ag2aW9i8*%?%+qgFW`<0c!PhTrB z%k@$yYTR}Fd?uS#%QemJ7)0q&h>83x$t6PJV@~PAO}^7Tuw>+hbMSxL@tra`wHl@) zbnUzwgA@y3bbX0AYjUn=W9kt1NqM|$YIFz0|0y|<_d?& zG=Dhv$JK*eOyDs1thO&wnJ zEmq%SJeEEcm@o=JrLP_OPD>wMDPBq6=m)YxP3$T5zhQ3aIJ3i!etO_tDZ5g9ak&6^ zH3E~3FK~<4vNmZFK_db6tx^94EV*941p2Ofon~C++UYMlpgAVl>M)Dl{WP3v)i+_R zidXP4h#zSk(P8K6JV4qktVL$GDX;#+9MjQ+gcoKi7FHa-=vP09z4e@G=xrveR<2Fn@`bA z;R=#B^G5yzg$PB^i9S4T5pCl$=7E@X&#yo^eYbLuk2h#=Qj>f1jGNXSUFtb=0*RIb zw>`8OV|WBZdU=9VRprKb*_ZCSho)A??E7}+`+7eaRR?#B&w0=ENHlNlQ%EbhXZejby zd0_kMQBp`Fl(}bc=zJ1M>108k!}4r@XJpLlyMLasWdUt*7HL$@q}3?weNjfVD`D}E z;gZvHK1QqQ*CZ@}J49^BNXK{L;0wzA^nx~XlE&}r3*?1M(IA8r{6%yQ-E8xi3GI0$ z3|FP{URO!jW!c$WLz);(jFql<6Unwb5+K=D;?Y6~yTiv2FN%`wp7|7LnSMkHW39%F z((;KOxV~$@YjIr4{`81;dn(dLUJJ3C(z?q9FwDTh5+3aM2KvK7FreC_Uk|nk<7m_IjJxV$dQkbanz1T0{l_3qoX4 z8I|#KdHc}X(VJ9uN8{>(9W$FGc}!}d|IQvR6F&WGx$)Vap25o<<732BMBwmKoS3r} z%fw^TsdG_8DTmWmunG2D*cMrJI$gK3_D-mp_PzzIQpgDo zSF^R*I(fcvD?QMQA1mDIT00f8$5I|59&qq|bo)a+-B2y% zuUT|{Ejm^Xgq27tZCI7AbvYO4fRj9qnD6 z>jY;Fb0ciQ5^`BM1=$8HV-T+#Dj=OETAxgW<}-{`g`bI?E?R~z2?Li7r9=c&RV z1m@#jdOtNPKN!voX0Bzg3*iXS7gDO?neU+ZBxVt^UKo~h;iH5fNmAY^{)0*(ia{?;qfJEV z;+X^{b{Ip&=Q3xH()oj@AT^7vqRAvKQfcmG1$92JM=}q0jE&Xik@Msc!y3VcC7KW? zis9iLBeA<8m$^#Inn7P6C$K_gs`(LKP&8Dl;2>6Zrk-+4eg@XAYDD$bg*Ls7tKsgZ z`iN7H@sv`GsbM$OzSDlx#JQc>5RUBASy4>lqe^?3WyIjO8*f-des+$d5Bb_R@1Wvx zw6Zz6RxWPZ4XUfdr;;R7pm>khd*m1%zu>+~Yr=Y$P=#3$ z+fMg1$f$SrHbz#I!a@d{IJ#@2pcyGQY{Lg{ixz#s`&6-*R!5deaoqjw!E2$?4!x*& zy4k|j*ux3(9qgSjdYP?;F*xkE%LL@`YG- zQ1{r6Ql^pO3Nc(a4r7j06dQEvt8UJPifWgqzI}X!`gLB$bXdrow1xg+4UP?SBQ%j# z-?)#vSA}1EJYQlc;_hCSD$ng?wNupmVr~ZaxsS?)Hp+K3k;N-UcBiAxW@CTXS}v$r z#i3?JoU3>{SygOxe!J!Alc2#v{&%$cvuZCrk@Ve^Xun8U9u=MFd@b|Zhcq$0i#_?C zCgxKQhW2a>x*yqi)Y*Ed-$t)_7Z`}L*1h~#w^;3V-JEkmQ(uxjq*Cin!Yt@jZt2Mfs;XA)ZNW%&D?J;g}v)~|Lcy} zt3_kRt&J#*$ryvTxNI=Wrnvs@sFuP)$I6i4mKM(C1>GeFP41S7JP`#YaqBF-?W#lC z`&cLFK}FZS>PGgRpuc_HNslvlJH< zpU9pCcwUzT@w}b_d;@&#CD@*h>|5pWM(&NPt{q7xPLRe{%zG%bUUf$QaC7}?OgDTY zEQM}byKBNt#1#n^y4ek5+Ux{AWPO_-85f`OH1Hg^UFc^2D!R?SYQ2H79y@42 zc~9(0+CpJ1G{*>@X9-W!Uk(UIgB4stqZ>W3x~MZ1!>%`;-DH5f^v=@g{4j8M-cHP{ z&-z#i$1?SSNfHXbg#ir(f!uY&g2f}(GU5r77I%kF@<@d< zc_HU+3~sN_JTMg1klBeXIg3qs_L?h=ChmelqN>5sZy2kyI>*;G!{}TPZMclL-jpN+ zeh`{_#D~4*d4Uze+#h7qehs`oCV1fmB^mET$OVZ{=J5jD8Rt!-h?uUBUO1Cnd}6xg z69}9mQnwWEOb|c9J{)L%wjbqQ+{dlXNyu=2zVgLI0-L!rV$<=@WsSGTPu6$J2F5L_ z6*#^&8pzhIADGNL7c{P#eb47NcAnl0ipR5$2$EekK~jwoq4VS@S`18zWV0(%O0z%q zciy7tijx+Sr52nw7Ooza;F$5gW61z-`d+W?TQE+xrk6w&T*R7iEubQTe_nz7c2$b~ z%NlN_Rtpn>H^_n#J-sSBF8!ZV-HtD$)U}+AXr%NJr=y+|Mu-U)sbpiZltEgq7;IDM z?i38Ct%3C^qbt*2{)oQyg}meb9W}zc5vX4gT(zjjUw9o|RCL{AExWS6i*Qx0HRyl$ zgLY%iX!Zdg?&YJr;POpU!=1PL7#oUj5kx~p3e35`@ALC zdtCJDT0?MpLM`mg{;mIRp1#}h90nrd;bsjeQjojQbjRuKr>#+V4=RAzp|!UQ9OjXAdV z7T@eSnmrkNp?QD)Cr7F9)w0gpoBN;`;XW+<=0&kH_CB#7w4QY?A}=U3PJC(XyI`61 z?r+%LolBLuw8|7VE-?dnwd#53Ln9KEwC`hvQf~sZXbEmF$8hE9@~98}cW#8F77qh<{_!Oj*M!TJXI+$%-MsQAD7P|R zncfu;+~Fi&TwCOlUOhS#?k^J34nnDq4ZR)hK3d;F+74{O7#2dd*R5yXq5>LRF{y<2 zhwD{8A`8Mj;Jbm((eH(C)yH&SOQi965~5!+d?_Sd3$*wt*Ow!UV2jEI*O6>R?zMAu zJSU^1ira7Q+O+TvK8_|QNd3{wvq0{>>1l&UNE`fH$$PGR@}9L}ALrn~viT%zU8Lt@qil(Ts4a^ZEMv0-`K)NbwJ zQeD{(c_;{AJ)a)TAQ?ZdgEO0HP*713Nw$;!4=(27grFfn}HSGB9u!%r?@?{D_wrD24mc>4hwhvcu{CC_foSbU#&35 zQ#`wBWLY{PYpr)pUI{^Nh8R^9+8-Qr*eo8a$gGvT^~bBBD#LX%cjl05|^BT^EATp=3!1b%9Fb@;Z?CJ zf}hjK!zcnoY5dghoK8wwSuc`jsgKD`7`gKpM{KINuj$^ng%&G_{hpeZo00HYWG}_tjzFc8xK*G|8@0!=lPcRom!-_w`!!{P%*F= z`L-vq1?|;^JMW^@&Sxe$>wM`%lIQ4(e12fmYY#h`+cDqWJ8iSs#`yyP(YtLG+bXAECy9HPoZDHc83oNs`BAVFF4H4Tj&2 z{jXQAjS<1thrM{Pr0AKZKq)#bYZFz>#}(Z>J$FdTDV>S!jhpdX^sfqX!fer{u?UZf z%6xIBac^l%2g*+~vrrycn0+u%;ZwJ^`ST=s6X=^0d5(r5yq-sDhgTRef2nQ#4GSDx zP%&9&XMX*_B1D^{c09OoaG>xt$@#Ko5tQ}J@?dH5)RCQ9XyK3U%dZRdg;k9wL%ri0 zOyue$Q=D>&iN8)t2FiY4uNTNgx_aPi97zmA#rZf8;?6dWFG|&KYi7nD>hZ*jUL+;u z`01s)mDBF;EPcZSKK8sg^!6u{mY46VE34o5_1E5fVi<3fp+fYlPO0Og+YzT4F9L-( zSglO;q@t^=d!6)lsG@E)bI99x%({QK0Lqc5M=*xy z>1M0S#$Z=|#l$Vi0>{r6+*~!ni_Q{zW4#?IvAkU#qT|p4u>ERrt9UwWW86#wNSNu* zOj;RwXveV!E8G`lBPP3~Ugs3fh>g?o!`8XTIX&H>Rki<|mZAl(CRiy32>y8rT zyB>Ia)vfFH*I>SE?EP1L5?cY}82>LjaVSynUBe41_Gx)w%w0Vf!p}RXa7?vF$HBeV*Qn87St6K7;XXT3MKe5ZU z*7LbJ)YWa_fC-JUbEzWGo98dezq@Y!O)Wf*joAMkzka1hWe%44vxiR|hHJTA^t9}n zI^EE?g5VdKLuM-~%kS@>6w*%<9w;rF82B46ppB?v%BK2Mm5#z6{D!%2paea+3Q@&- zh-JqCDkzXs??26(UubP|(aA``umPm*M4>5Fg2$aA=O>gy9v^eQ!s!yF;O`DeNf<1P zjRxfmuL)khcC^ex$x>D6*Y0?Rby(}$H;N$MwXcg|q+F@yN`pC%&5N?j88w7AgM16N-BMY%jyX6P7$TYn{3we7ifmL?-@uaTi>L zu0$dx|M4vMeOvagQQJ|*_xsq3&4)^(BYF>9maS2J4)*J=T`_tNFDDRPr*;-qhkH^X zkb7ciTr%s)NS02@TV~<-LPEijK1twwM`d-=U2MB3sr}=88EyNiS1DJlEU@uOX}mRC zPePD#tjZ6^X z?V?fZ(-}#RA@%AsmegzdXX1l{FU}5M%Of5u$w<^FKwl_C&Nq(ldiP*O)mUTZQ|R}0 zzT1}nEG$IrDL`8HFc8$=%LVDAP;Lc?`~OGUpw(acxnAkteCv;dg&?<@0Lt>DX=hC! zrU57Xf3j$xEGdxb5({#62{z@RBodJ4U`YTnXqYC#%;R6B{mHP&$x2&f^*awvnW1DW z4UrW`86vQW&jj=MPv!?mY#xwV?qda+flziJ_#;EgVYv?F4kAGmY~j_QAGrlx%3xG0 z3lYfJ`6t5wN~9o>0Vf1dJ`_k@0QZLRK#!m@7VQTE83ud^C?`rRR*nS%2Qq z+;5oXZ&(>fEtmZM!>X>SY@PQM z%E?fcXvX7rHf(I@+wHKXtd$zmI#@$!~J^BE(Se(a&!o9aV+6c@{ zW!*nMHdoSK7wSKZ-%r!Bov=RyVK$5!shV{`X2^aP4ePj1nnGW%rWFU5U{hMtpn!9# zkffPn;~l`Nb(U;xt;dockNgsbm0=j8WJt?`!nC?2QkJpHQs z{D-=fPgJh@Io5qx+$)Xvj#<<{>ku85QZ#67z07mUXfaf>>m~hl=(D`N=7MUQ!T95q z20iDIZtYGl=UI0AB^-%*L!@H;XjZwk!MT7QS4ElNRf(Lu9rBS%h{-SR#ZM^B9$c79#*1p za2vD9=CZLjuFhp$FlxfQgZ|8T*^6SvB#eP}#D@~6pw;h0M%Kx)k>UNZ2zyvYa2#|~ zKeem`zb`ts=qC`4vP5oW&5VXuNMEx|N-J^rsO0L6dAvT2Tm#yg#GdaOSngMdU4yZm zT@ySql0}2Cj1I>~mJ2K_yw)EJZR`SVgpEDY5!DNYAAilv6Ehfy3iE8b^6Y(=l%j`A zm`z)}-s6LL(e``s6y_!P+Szc&lR8D?)I8G~zW%ditf z&D{~VQ3)tZW#hl*cKGFrK--!{2u2X4tpoT0j6^?>QjEj9$=<{|k$O!v!#2kK_xCL) zbSLl59-x_(UYm8T6Q@}(VbXR%epL7M_<|V6g;JN=&8~6dkNCO zes!{~R;GE61V&9ElR{5i1_hleVt*)LR+kv2<=ZIYHc*KgjY0vO*>8Il&+ent!(6Dw z(!WGfA;)a$6B1m-jD9)Xzep9WVELZ2pjd@p0w(%j)V+69Q(Y7A8$dvjVgUuDDJWHn z6zKtd=+XtGqkuF4>77s>K#?L{5eXgX7K(HW5E1Dm0#ZZop#})??gRRi_kF*$?z(H; zweI~-$w|&Rd(WOdGkfMYqVuK&yapi}bQcjZpyAw`C^rV4#6sios*a8svoPA%wyFNQ z+=gdH*}i^0b}E!Dv@3FV4wuu2!fn0DSVRtD6dl=qAgs)e2lhGZp*p7>*SDi~pAW{k znco7*h+q>Cx+8I0OFOCB$j|)lIJP^VCqB{nTk7aQV{1A&Xin)*g|(Kw)S5GDzsHiF zbLIKN?Nb%)&54y8bzf#^+n(i6%X3i=n4}*sQ|ZFnrVfM!Op0|^NS!nmz8E9*!a%L) z5k{_gvXbG&jD()P+>+ACFP#nwv=8I+CB)klhbw9tWk(!w!*XRYVaf^`EbA&}2j^1) zQod6#cGpR+|vPTk$Q0#fJ2x9A{&ao)nrc z>@;v-kFiLY(_YAJh`-8+QoNo#;5t}XmI!+9ln=7(a6G464|*vGD=U~QRY+_bRD6-% zUSICjuy7!dKh-h&5;d$^nD3BN)o(N`W#{LBZfBLiU+i1;Cs`BttTq|Vu6I7PGi>?p zo9dd{sJdRs-8jW)Hq#9MZyByK3Tpe33q5!I1-t96me{4n#06vyB668;v`ltf-u?da z?YGG2AlHW)qyaRcQGL4dbl=Vxn<~Cj;7N3U$Y6NKB82AH>5pVZG|8$7c^H|CET&b% zCHS~>-s@qu{W@VURW_~%G+e#vyeeBCW9vQpQGB*Z(u+X)8Jzc`6L84^N*~AQF5;fm zI1W@#99SyFY9yOVvU`c;G#a>Iv%LQ!n z+YTil%|LjzF;X%)7khp@HZnzNjgU{9KpH-D@`ZlP%(TAp_biIwB@pmW+h;;CyPpzm~AwrvxTA@Cc|ru=Pz{^mpfM)8@AmqY|uT( z>$URxG4;hJ@z{J2srq>A2^>=EAjx{oqdjpub!0T5z&N+!Yq{%VYoEGu7q(+IyltKm zWv^Z_tT?1I@XRJo^Ny(a45`bQdpPhX&$;(ap70a@bgCkk!8vy_wcA}|C&0OHy)!&r z*!h%QrR2>yqsr%^Nl|8gV}mMoXHe>vSrXzF03Ze@PeTxa9QR2LCRf^pvmy~K;>b??+J4?E?{@O_Qq zxt*Gylb}BXnq`CJ!mx?pe7tT#i_sF})X=$tJb7Cu5sB4v#>P+VI3B3eJvm{-X?X2| z1F3~u^k5P8`{*3&Nol1bJV#H3pQ(XPjp4h3-FN2g$&uVnI#N|$d>4vec zo)Op{4gyQI8yec4GYyqH34Y}upRJ>1Y=C!r6&3=l(ISZ%#U9q0;MAHiF zTyo}JZa(aO{dkbY?Gk%jdD7Pv7&b1we+ict1#)`tu6vwtGhcSIEp9l^ij%o{9;uq+ zBjG+h-9Lg?H8U-Ft1pe13`}~pV0IDbBlX z$W%^o@7tdFIka(3=i|*M>R!!e$8n#%9COF1^FkL6qZ0jIRAu$+~` z&WyQnpn_TI+eXD=W2MQedbS8Gu`+nOeSw5^)pkNhN`H6ntOy2`1cZn^rigc-``Co09%uj!d zqQEcICUPGOHjT+1$88?KCuCnv1f1JOtM`hAz4!C!&r9v#tf+qQL(ooa*$WomL@d{- zx~CY^->=D@e<9bo+I&){VsDw?Wy5U7Uc|58TVu4ay!L*CKNSw=Wk>bi>We^C4Nw&~ zTF5dOBg%Ijr=Eu`!c!VAc!fy}r(x2QX2wp7!f~1QIA0=Lub%9`&mHz>(Cs4u3J1rG*g_sW_+HDEQv-D|_&_JdY$YGEIR+Fh8Oh!`03Eqv&v%^7DZvH9T4=3W7PeU+fY%Zw5s2@3w7dg<_-77 zPb)5Viw>qD2b$uz$GHmz>DO>>Uer{qb#==$)QTp6=7 z<)F7(b2?va%Q*t~Y5SqHp`_@J2-&=WcPLldU3KOslM->k<}VeeDdgD$gC|WNvXMUVdnrhsf%H*9x zTtLWB#}=deTebRjj+D;l#5wZMo|;U*4lRL?SB4hhuN$w1Nxm$uI#=W3(8g?N!Am!F zp4xE0hC$oyLi}+~z87~WNz67B?3hZ0iWs{+UGz;$hDLqiy!91ebnOUlpDQ;$Qr;C0p} zOj8uw!((*bmbcP|?0xHBzWfw)OV>o{LXe`O-ZPe@8sX9EmueqnsRo>eK$UUV1A&z^ zP=D+|`}R~e?3^{n_(O@J+^`}Q6Q;leCx4{6Sbf5Ljg%_4@mk+;)YDSo3?kx;+z3$mY<9^5d#;^- zz7Di%H_}`34kq~BX;0oo$J{fGPGRL5a`hJ7oQa+2r1AE-tWKg<(R;Y4+TqUKW7mwC7Ss+w-w)o=9=z#-y_ z^}*}= z2iIVP+YYSyBz)e-h#ic_2NGV{Y2*49&bbc0=%ih!654%J9{51#+Jhu7qH21Yq=;kG zRDPm#?ymg|g)s(Rj=NRX>u(jkqXs+9*wlUJbyAi`!o|wh`sCsxsha{3g5FY}LDO2%A};4mV;K!> zUb3mLoxac1(3{69HE}swALX?6?_gFi3|kAG{UXtv#^qv+{fXzdo5F=m6yIjDXAX@= z#q`szS7_uEenNZMFB&(>EFYK+G4^+u8VP0VZ0l=us$H!gDZ|-SY+WnCTTh&Po?TIi zesDka{_#gCLc+pAAM}Xcf{vHGQ;rr3z9~=Y{O%qzyOi?DH$8J|L?P2i-p$$FvNDo4 z_u3bK+?0)ejP@B!6iMxs25^|*H_prZsO$k=G*8}zq&TjXIgMv-!L_*fEL+I6<5P^& zGdQK3lf(NZtcLxY$`ukh)zwvR>V)JrRNXj&4ezvc_*gQSr2C@ztwWj&qHdRD&*4)S zdrh?m6KCj6q-QVzdKGFl0h06Cb3Vuz4+Ksb^=XyG`@C`ux;#L3&mmvU_D(C^+u_y= zXQHDdnZj;{bcmIz(DiuR`>itLjWk&X=e=q?pJdyewVS`XRXf<`;pTQt`QxQT-e;IQ zyZ2$|9#D>_Zab35_tXhw4S6GO1{r<7({d?XP-Q5)?+r)14}gkJr44+TZ+F(@eUT!g z;?!$*l|SA{W;qvR)CbyEH_qinV_^H24 zI-e5miv3h}EqwoD<#H#pukV%PAe05X$6{GS9#xZf{li%x!ai&|re6|pycx7mn9SEy z2rS(jR$H}p_K+Gj?PS7T&snHUZs0^$Fw2@=Vm;$}Lc>FJd+N9&La%lboVJ{-|;R}|g`pHx&P3bMEz7xz@jXq$q}FLgB8 zoSw%H9#*yJj$UvZXum9>=ZWhvdn|&U@~;3bAI^_*U>`J)q1~B1;pFqpEZC*ZqFUs# zo%mhiR>}uO4;t6IX5AyE=}i(=#RB=)e)!6SXZi?sjo03s`us}ZyY)$iA*Cd-7892i z@ngPd!%nw^J2j|qf2qmS$FbBYb_EAWuaxaq>@V))ordd}Y)HfxgS9nJ=%?jtt6b+_ zTrpTmN#0u-%zY5R_%vVTV~fzzYOlflDkrqputL^NK56ZVdL8GtcD@z4%O6Sqe5lD3 zBe&pLbk_We!?m`CSv{Q1#OfJuHB^djQL9CAFEh`i=R@X|JK=9_zgs=mPKq^|jod3! zTeAt>RC|NSn$tgbo{jFz@K7^t5KFczm$=MgE|Z7Et7Z$HdzX?_IQqBeXVRlZ$JXvY zREGJvUI$yk)#?yX!Ye9tJ{#4czdk3Y-&0sm}-*w7dlZMH4u`oQ1eiQ+xfPj0nSc=kkR!-`gmk!|lD|{wO%tzaf?u zq)bBAkPlL?seOub<9ZWbZYS;WmO6q%(PgT;O?sA$o}*0l=DgEG*!>!OS;4wew6q6j zRGrOQli{3EI?}

|(m9V)xQlje%aruya}`)5!T-6X{unuH+7sxDV5u#Z8V)dWBZ7 zo^7b&`3bxK!}yNOrnALGZRTQ);IO*5FU&7oZoQ#N4{wOUl81>jo*A)4`=&gfu{9_{ zIKQckm9X?~S)n#~5D_EM><-#}rK0v(a;7D2zZ~`&G@h95M|{MKNW!ZMZk&qd5&@@k zK(>T>$r>OGph7td*r#d!OkahVZlM@#JZ#&}<> z0nPEA4cxKTiHRZM<)u9JVNIUtltWgGI-?$)k*ojERo|TWcl%`lD{l9McL#Fj`XZ;j zJWVeoc$rq&4;y#1yP2=0;W!E5%u;|#4f0ba7dr!mjm^z$3Foj^qV2>=$ihm$+q)^< z5BD+a*Z;%o@%bw^js?H2=Ah;UDP3U>YqNwfQ{JC2m<9MQ@9U7`S1n?OhD0OjZ(q6u zD=f^0aWTQjVH9%(5k>lG1^sUuopGF(`Scj+EcGLuhVva8l+w%{Ix*%EqadTMr{bjg zcFIF}=INs0b_qcyk%x_|Vs9L?T?;svC@o$zQtUFm8Gk)R7@=4-e9EG&& z)f#dG-H_r7Y@uId92&DL94UGA=nSQ5jj`2Xx9+*Wq*v^A0?!(oND#Lj+D1B1=4NA~%<|6!lc!fD6BU<9wN>b_gM z+Fx1rYEa$Ahe@y_=52j$lNa$7wx^UK*Un4ZFc&rS)twz*zCT~K^^)upVg3V^$o3nx zixh%H_jNSLNm6UGz1=<&m<@I1d#}3pr-iar1Ye{!$XgDwVGwVpo6{@IRrQ>*D-j4l z5KX)hv3;+fALw1aRZ)r@vz(RYrmx{UcqJz1oZ_dku%)m_I_ySFWCsxn#=UP|3DR`KFE)5dYF$%rdd7X7{L zJu%T@ROC)7?@Mb}dkx*X7drJdMaCLK_U?5vJX+fMwk_A9%W9HV%L3ect2)2=TW7Um z;S;&jiRaLaddr)a{6uTC>6$BkT%9qleRZ|`F`WuShqz4hLnJA`2WX&dV2Ud%84ds_ zqvok=O^E5Fl0>sVR66fXs=&uR7DoBjOj!7h-K+aOm?iESo*wu#U((dw7TtD*sB>gI zN^tg=)c^sRZ8xMJ?VMrP^~B-AopYmH2|;m}mh@QV)yJ**REw_0R~5}I_n_lQrKu*asDUFX%~40$Js0yQ?f7i9e=G?TDU;H)PeDKrC770$>23pCv8`> z1-%}fxf_|q$8e#p0dth*paUvh7Auxe;O(?NWbD9vsi(QUDeS_xs?if9HDOVq^je)7Vu2K=HvqJF5P8br`RMzS3^R)aBN4v7N;04U%i0_tFiCRk)~aTrqUNXn;fij!?63dp5o{me>k>qilG%x{8d7>BY52@MplHy69T(^$CXf+{B7sal# z8tyrEqdAZP5y?W=C4#^yK2wn?eeSvenoXD*wP=_qM|?GSn3I(xg^$X@(zvX8Zvsx-)&q9&V(~NSNlY(!fv99@70XnF?z$tL&Z$A9BIUo zlc0NPV{_-DhAX(LBD@r~G2F3EELqZ`G3N67^KST_EdJeWXTA{nG}2?f{pN$j66o>p zm%yToQ}YSzdTpA4+KynL5NEJJk>tu}B3j4269~<=@XaiqQav9wDd*_1Q+0;pYH-RS z?<)fxiw4*$Z^)iAj_s!3c$`GIchCB6_{oG$hViu94X2RX;fPa1WiiYCqAZX6J7vmu z`G=;Mqu*I1KG@&)vcb8<*gHQlrxcBSl)^$^+wC!tX@Yabj~0*JH;y;HtH)FN(TjWj z{pkhqj~jL#tB;Y#375TfKacWYIq{bX7qC&QR`Djr3eFxj?lA7~KVb_~!Cx?GmP{b7 zXdJxd2xD>s5RKv>4O?bFyP_AR9J9rI8D6c0_XJ(Z0TpD04hLD1&l<4{=D25wUID!C z%b;qxFOvnquP_Pc5aYKIO(?>11g7uKNlP|>x@r`?g|NjWoF~Z05xQ6I!FT%45w8Hq zg0iHZpy=toCd6tX`Sv5`qYuUbLxLg;UhdNR%kM8R>F8ZN$o>JcdSbQU<#JQdKeR3Q zA(Oj^fQ<1k!}}%>uCQ*lf%{~AFg{#csd?~nzh|)rTdIh1A#el=-jREI7=luPf^ehI@^Hf483mc|>p@S^)D`0E@g!%AZ zPl&wIEYbk}$G2Zl6eb}AqK3pdB-$(Ye!@_%UZ1z~E@ZQ34}k_j0DlX9^lsH^Ej%f| z{-^ZU8&3xQU8RorQVZ&O{#~TrFHh)5kv^{f_ zVS`>!LvB zf($6CrvcAq$D{-9OXK(JS=suI|Gvi@i7F%;G|Q)Baob0O(gXF`=>^Jl2lW$X79;}F z7f8fsz<~P=YaqtKK)LjRo{F*Hz~Jb;p@PSEikT?9-(QdY`$|}M2FW70aELjq_}A}+ zzQV@7JGyX=L=M7LbVw9Wykc^51TzF)B$qK5CHB{h%+f<3sPu|-QA%n@P@3Ke^6~A- z>;HbZ0`KDPSC*lU58uFu$Ig^Y7IP1mKWA<{%Eu=`4W=j6*)N zQY>!kulJtsG~yMpXF{O<@#w!V1u_qltw0`RemCkoT>W#BvGn^Qe+e$RhA*UBJN_Y2 zihWHkDewSV>TtH18lVvqNI=5Ao@ z^Zbt0sprR7{&dr2$bZLwd#XzDr2t*3DCNUnqb>0FD=j<{;r4F6{}cAlH5B{z*mjXc zLz|h+zt&jF{>{IX6;Y-F7|ocMuRVMRGG0fbYcZIq82dp8`

|0iOV})62-O_Yi)m&axy{T3Pj1|MSSC%EeM>gdjDznM9>za76l=G=O zBOt6-IiFW09(UH^8WV<+=!*pN!r0$0JkQNp$fIpHm@Q{TT@KJHKu`Qr&RBlG$nz`z zAJ%^Zw+{z4P=bLzR*1%ciyEdjy-UPYSoP_m|PtMK8&sN43eOC z4(V0JfVd?L=wUD>-yWFUYzY^%zyTbLlr2pDw%VEUjId<2k_>8j#%4!JS7&B)_-lR7YL=U&bV*2$k_e(fpFE5_$^#{VDc1}~o~g6MEVH71!stsk=Ps>v zKHIyWlKb3DNIHW=q~|LUAB>)aJ-GMjm}=@RMu`vHfHn0oD}$7swi~40iKhVop&VeV zfn>f6($*I4^MIlh3ixkBL1!7}<>)abB{gd`yGKZD8D zP6A~F(@w{CL%b?EHk%R{EoJbc0yj${tQ%w$_V5F!DM;$W_fP}u0K|XlB2u8$f@R03 zgS`+?f`W*iatF+VBHJ$66&fQLASy;dLPgF}4md`Os@KQ?iwQ8}++u*=6?zB~0~4D; zoCu(27V7{L0LK-=Cqw!=@fwZO5%?!#>$E>%CddW&He%1Frk)s_%S`6u9~or;Tk!}J z;O>H z4|)?o4x7TzJ^@Au$Y9i1mOBx(nL!Eo25~4OA-Ur~{DJ{&Erp(58tCMb!vse_Ishas zku>mME`XB(&ZMZ>*J%OAOLdQ?t+hU5a|0g88aO1|G643SjtuuBTflp;gQlY#MiXdv znDYrBIkqr}uKUl$N87L&#N=wr>O~GlK2*E~5Y=ELu|{&o0WOMxq=ZDK!Iy8_Ah?UQUUOz^mcr`%?z_I91HD8GX-8he}nn%8uR; zLi>*InJ2+p9vp$vy|U+j9Atmo?ta5DT=xdKKll28u`Z3$ho{$cyzwOB z`SOhLcA|YlKPv{YE<*yKZ~&4AJ#B`CN4sBsdl&>c`#9AtN1^1a3UI+0#7RnR?PLjL zK@@CBNsg?pv<7o%+(btnB#!(M%U}Nf((7lUh$XBK(c`rN}Xt|2r{oIfc|Fk$flc5mxF|l0VQMS5_|;9 z1H>;N!wk7VCR6}Bk2q>)1|kot5|PLN$fy;K5x4RtVCVtvbDS##0b!yEMNFBLcfuDB zF>7F-Ht7s?1(>z;Bg|U*VJb2NmYxvL$)+C%UjYfpV1ofTR$fEI@F8B>Fp!LXtt185 z#l}Z$>91sRIkP|!0rV+_%p3rp8mWSZ0744L#y!-D<*-Z*$=-)b#{dN{0os=Uyp+uY zkO=|B6h*`t1 zzZoF`BM01&s*F_xb>(VR&S)gEV0LvI`B8gUh zT`)3$|C8I9fneA$=vablS-Bbz^GKmY_V_Qan>JK-w9dihgiV}3w^i1zMMp)y{QON9 zskLlMR)2kPQ%2F>=4<8iH-v?=W&vyAQ*9HAPk=4X=X8N-+gi7GF?GX!ppxg*3UZ#H zRH8)VPpA4amlcig)c@y@mnpCR@&C%8lTx@0ya*A$k_hOETrl8?1IaH1svRWg1Jd{* z4AU0;IRVn&#Cj&<5Apv-a?Evy)OQfXXBceS7tq!@9 z&`e{Zx>JQ!fa3HNREJoL*tEe9&MJ>eLHgG`kE!AD1cM_yhzpsJwIsEm! zA)z*o0IQS6TJxFZW4a)5~d)DN!QW&58@@Zp4J<@qWu*y;rEV zt7BetKx2M-#bO?{`yyo|>p+R-w^(m>vaKT@VU~O&B0S;|n4vi`*`^{(*0r-{Wo8ao z{l3l#v6QmGyco?+Zs3C$_~e>gsK3^;T85-_M#FvN7snRO?z4_G^~bs+XBuy!oBh74 zUR#=KA$T5O|vdlq#OW03BpcU%&RE?C6qx5L{cjN zz8s(*fw9+ql@QH%DZo--K4ezg*#gT{Bs&-srJMo8XBaR;X(25x zkwF|Br$90a*!AERz>tCT4#dE82L$mW@+2|PMd{_`TEsB9ndB(2Nda9AObepfM%s00 z(n6bk;Ijgs@jt1NnFdJz+bEzU^YP(_24&;F4N8C(7yTbP5)2Vk5R>7DKJlGahiKst zuYWi=hwM&ha{w4_07wRGYcTTViGXAc?I57Y2H3G(k~&@F+?dDG$m0vqY4^`m13Xpm zfkS%(wQ5&q;hnb6k~eum=R4t#?waN|4c;=Ns=a|2s8a)BhHyesE6(U_4sPM3WzWW~ zAR=^Pv?PX!u=Ep_V17M1;m(4qDPO~W_QE;%pVBAT-r~-`n6uO6% z>oS0V13jP01=Zz%*&?|F!FaiIz)OKS2eDc3)K>vW4EFy+aVX=(W$uqHNoh2AeBTXl z2}Wbf^vbxgoqI}=c;5Ry!w;?QqS<{}l<>#UFF;KOQR%k1pHf-e{Nt$FD}<>|{W8vH zqOr}7m|vVk*efcJPbh!DYZ(+9B;xuGLR-uj`lY19&&fQzPJ~bEX&3p^W=I}omAcKG)2*Comq+#EgC02Y>6Ku&~egD zv7UAEU~mSxRVw$-JM!LoCiSo_D54kchfkeunXHe#u7e)d)c2gm!JYMV>Z{NdNu8!p zC_SY}w?gPBngNK|lWi=){d}8K-e-@|5a8gMMyU$d} zE3mQTC07bcXMyuod%Rv%mGg6vBF83A%Fhz;ouSR`oicr}k2!pmCPwLEIUiMEcFeSC z18ZG<*ZoMh%kD5slnL|hWbz=}ul$55;Mma}0Ygu7@o&^-9U7a%{#xxoYtG;;9k&}# z+CRl`e(@q1A)DBX%w8j`E(f0KMb_C`N?RTn)3IjI_5Uy~X)mu_Uw>P45%Z4H;DtZe z^TyZw53D!?a@z_R=8lsG52vA@>y9*k+<9W|c57?i?>l$dbUeb>c8gJ!tL)Y~V9YZ? z_;}#k*fMiN1UzlvT>xwRkX3xh$PNB5o1hAs*#OWSP(S1Tj5K(5&*$)sUWm!>-v13J z1z(FXIx;GIK{W;->;fN(mc|!$$m@o9)}xUyUfvJD`uxOejofp9%*_f#B{K)%LxXhi z@Vnr=5ayyPzLUGB1KexCu0Gu7JA~w5z@CeF=>x%;`(D=1Wd)s0^MDiwP>H^Ps|7H8 z;HT_haKOZKQ8}a~cM)~o>H-z)%oixnqV0j_hj*@3U~ z1KO|dfh*z_Xg~?Dvgr$(r;ab0_QOXq3EY9&;u8ccd=0hnv^*PZib1R;fYx_1QWdJZ zP7b2>y&oh95w}6PAcd|nz&?`z4WdrS zFx6q=Ibc77LFNyGzc3~5h6q~uJ+lT~tOh<<&;O4nxw!q$*X5$55%)K@tC(`+{0lrf6rD$A8sHqm|PcNjb4p!9%@!9oz92MlirZVon7?WaP(Xg&^csG;cw zm`GqGf*uSo5t2x4RJ}k?d=LffsK^rpMF4CZY$@)G?cPPW?otB#kYO|1IkA=PjH!u{ zmJ^%%<3!6_rMJw)XnlNY%43$!5qs=b%YJnPp5u0n#^|QDsm*Z^lpujTA11c}z=W1N z2~D^%(O$P4?UqziZ~0=0JhpO(^XcrtQw^SPP8zd0Ez88&;I#X}D=9yu?-MtnW&S6>`Q)^TY!Y2QSCK<3qD~wj{t- z27>QUJn{c@;>h0;8aLJodAHz zMB@wkR{$#mQV&KjTXMQ-Gq#^FW!&?Lea+q3lWNHPXNYapEkv%*A4q_$${N`SNbbTk z{3uwQ-)S`3aFek-zLSWMN^j82DO4-JylcEAV^mi*`Bqv|5m}Ib&B(NXJQG$R6IjUG z2R7P=APsQ2p?iFBqCac>2jYW?&1Soq9rXigbjox=y1H%(oqEH3z#enC=FA^&siTM( zcJc?#hx5z!PqbefPfM@hw(wTf$xdlX>RE%$o zADqfNysgRC%4@s$x_c7MsM5;u%}tqR=b*rfDMkX)XFS>#SEcYI5Gg^tPWpv}bCoE4 z3VYDIWX~%xctD4hT1tilmjJ4U2d&eLDbwZHW0TEjy2*s z^b`->0PgoP9l!vmCjXcdOEpB@coZ%Hk&x6iTliY^kPUJR#M+BM+y!WPu;~H!8&c%t z5SCuHkL$y3G*E-Opxae35^Bw5EMPu*nCb$l(Rb1`|F-&B1vpco4_*!f&jX@cLk+a% zK~DlA5^#QW6qv}(flVIvBxpzs&6n<(iHB;!p+gaHV;0*Ma+6dTfP6qPPpB{)*t!|V zAnd<32zvpRJqYQ7Vsj{#2^vD^fNup6^`xX(bVGN5X+td-FbPT^hTz@|16wZCjlw0k zwe_$+(9z{gT&r>nIT@s!J)pHr`&H(ESD@(?=u-t6aEVWWfCUIzPz1cv4&H#8R^+F` zbk!M9^^leWaUK7<^g~7K^`J)!2D}px=cWaHA+E9=qlQZ9p*8}*mmoQ0<$lmnh_W0H|M`7N!m9 zE?Xi&xhZHQ8M!4;oWK**lDroQqxIbYI~(L{x#!N(1Aed^=-y>R54|yH@{|jrNi4b; zfz4P(oLN6%$(4Z|?TS1|)JHa7(0{)cwa%|OI2EHIBU}aYON-leud~vXKYRT3qy0BG zlz2)WoMNf&uERD!={!X~+oi^^yt*FSXAruyJMnnKF`!$i$IskG_rz4f{!}{^HD=lO zYIL__%N_Q6O1F}Z4dl8YXq*Cj#}6dlswmiYAim#&;oq1(>kM=l_|!L>%dn%26)Lah z)nI;M{s`*w)_&QY=~Kcy=pS=GIi{fx77F?>p@7V#xgK>>ApMuf0y@HI@Zok3lQjbq z3}#WP62#R&ehdh@M>75&h^QKaE*E-4LApXsLDI11og1YL!Nxgj{?) zK)Fl?%NmB5IKLL2;&Udv{_T4eYk$gcmoMZ@K_BHh5V?-tR`6(lU6;ux6<1K|vU*wT zL>8_WSTbCd3RsfnSomHOde=s?ZKA(*z2j5gQjy28e z6WClnGn}lPSABPZPK8zDbnQVHVrbAjkuVF6krrGK3S0fL@6l_58~479-j-R4|c zLFAAL3af#3E}9h-g0hxCaVCPL4SIy_9J$;unv{fgAB`v~RI~d1`&04{3WYzCpQgc< z87!x+$C4p(yoa05tL<1mRSc|shFHq0`w>{dh1wsBw<$}WSFLNTb*|6ukI-89N4tt#%~@Nln7n2*~}(#okg zYzX@tfqxqK0>!o77vI?B~39WjF_=u+V1dbU(paF6F6*h(E7a_9l|%QYY(9BZz7Fnj?d?d!_I9#nzsHK7HDtplH$2+a{IOE-<6>!0Y^JkYqfF=Wb3F3!%D4yU7=nc35FUAEZ~ z@f~D&pF}gG9x$KX4{1|T6Um61pDANAbD@b)iYLhx5dWC`&dll^hMkfzNt6%cSM1NYo7D!J<>!))b*U)L;EIHPc)M-r{KzCLH(?v-yAw4_hR2 z34yjKrtQh(pu_$%)Gp1S2b?wt1B+*+x(!FtsDkbPI8#qtx}=sH2UkOnLXO}F&NfE^1>#j% z36-`o;JlCYs>m#pP7JW0!9(QByw3$uHKZ!=LEE_(RpiL1B_TIPT}pg1K8O^=+;Z6B zng#4(T+oj&#Ra*@QPg?Ngn~mslo7S^B5;QV!Ljm8X!;MGE*S7R6fSm%gLYhz+_vDk zbv684vcQ*?Hi*92zGFz6opB%3du^0Pa?&j(C|G|2ZBWecDa9t0o{r;WHLQe!Y< z6CCOKs2A*OHrIZm!EKeZ76Lcuyji9Tv8i#}1AOtTFM&~l4sCUOjkdV3NvSsjg6y4> z?tlk|+SyuSI-^oveZ+qwWEUtKRN}Dh1#xL6JwoDO0g#sljGx@q>3a|S!F?FMwmM#R39bk(w2GhoKL44&br zJ20RU(chyeTLP$2$c>3r2@*_|?u+LPJ-x>afZaSPk~^xI>-%(Yp2TU%v@GQ1sM55B9W ztXDP|mwimnb6dR~?K~TakU>s{L6HD%223uwC^*!Fx#+E|oayLsXbqe?-thwy#vt7v z_o4LcqyqNsTG*U!#QR}@9RBd7(*@5qKwmi8k(NIw<~}~>XY@LMB*)61r_wEZWMN}N zEc4+v%~$51_nT{xv)cX=mxlGQSCV%8pJeIz5mlqv*Xf46jWs20!Z#Psh}GQ44Hw$D z{%x#4v}^EptJSPbC7oQgW0uglbfKBv z3CdWyMr57_bHr%$YYwxqYqM;=pw8CUWgtm1F70;KN`!u5CMQFY2yJ<98i;rm34?I9 zn<`L4+N*LWHZ!0Mq4e@Zm<6U3I!Wz)7Xy+FK=wl9jT1~d+hC>NOmmTMr6;F_6`{;* zC4gVxkueoh4w_RPLnrx*d_y;|Hy|2i20O=1K!frAg_w`0U&ateB`oo_HJo7=3-)ry zi=dqXUZ3_202!b*W1Qfjd$rw^opT(yO720SDTXvg}3&MIU#@iH+FeDJc&I7!HI@ z>#Y_Ckj~u#ODMyPI3*|5w7`naC;6SK4_BV|2N$&(on1oD=&hBYhy6wKe#~CZkki~( z6+`yv;|`;8Yb{ehyd{UEH=NcL5vy5atwmjr;Hqg<8oq)l?oYnix!&Ws`3~{y_r5^a z=W=@-*Zo?%|Z3Y5lmo+X!8(<%3EDikY66>S3Y+B@o3S7;XVHaof zX@M38VN+;3q^;8;=PmOd`j`ZitabeL>L+~|Ar*wNEC%W=e?l>~t`e=!!;{>>{D@4E%A>aiyq z(j`9noZl`OFVh^%M>8tZemblv;OV!*+-E_sR?ZB%=eZ8PV)f3K2f|(%F92JQq|B~j(gNEU}g@?nV&-4xNUPu>Q-1uXziE88o zs>B<)gs^eU_deNF6AFEYo$MCav?k^HVE-YRWuZ}`|K;e2{ zpBHZiw#Lz?mLRzoh3h$}K@kGk5C7e5aaCK7YAeOz846E@Xh~TffNx+~54PbgPl)V8 zdm{@4mJqX(@cku3G^^8_pD^Sjkiq?iAadvA?TB=6PlR>UNno!*e`@@X&-HEwSSg(s zip7CEUPjpwaBui&kTh?Jw|#vs#!t6aPqXR{NQZ1f?0g3o+zhZDngOS;B&--vec+}H zbYIE#BI_*!ygEgk;wIn3&?fGpRi=s{}JhD+%S*4uHS<@s~f|;<>@vx?(GuXueq8) zkZD1zHbOakB!53|t-ZkZzjO7BGG;6E_5=Rc!@n=~JcrV-9Q_B0a62_+(0V0cLh zUk43KBp+!)`VZYln9%my|M-5+^#Rk;zs7qbMm%Sh?)I3W<&#Gnggx-WWP4x0e6X$w zL86p)FV!uEr=a02>O7DDAR%^v{}z{5M>B0nAZx^^{XK6AK^5}z&3fmzf6Hv_KL?nP zwD4x922Vlb+ZNK_LUQnk;kr-|e3%uZ`oX8{bK_>TP18`Uw&=k5K3InfvdN$j9p}&Y zK6L*IVckcco8RgC#4~E*f=;KY?J}Kz%x&3qF7qX9zzSh|aB;SARoAq*VJdDW8L+oY zUi}fM*HA%Oy&|W9+-Wl?+Ah05LQTHICkqZo^g;D#>f#=OLAIvpnUgPOL# z^u+eQz^}CkMqNSg{4YJxh%sbbeYAOSUiE#SkyTuld7NG_-)tB?Il8P;T|woQ!dK%< z^SX&snRG?s&KE{CCz+jkKcxC4ok@ycPNBceDn`RBM-(B>5p0%eXGayCqpx3hM*KDp zU+lU}xxO2t#yfP~pF{)6svR5dj%ojGIe7oX`DrqL(dtJa8Huud7Me zOE1i9G52nj=|S@Eb>@1l)GJZV z$nyqeJdZ@$5d32I1n+<|#l3gFgo^2zC#k1%@$CZ1Ac3(g!LQ=7@$RLuPHWsdG)lwa zs}sO?Ht-o%DKwO`%e*CV8Qu>?S53?-?tvLUlpWFOQGuGuDr{MJ1^vtohHtlqgJDi2 zY3}>8$hG{``Qx4CSwCSH0y7gFbRH6HTLaG@&=~*PpZYcb`H+W-{cr>E^^^`71As*h z7d`c@WN}E)%5YuI#g8WcQvB5|-1*{@g)jcsz<>=Y@|AOOtw{@f+~L2r8@(AM-HH+G z$$;l4M!ck5#y!}q_X2JQiq2gk{I%JklX2TK!&+$k^;{H>$bHPW#Bh2F2EXm`>+@z# z4~9Os^4A2cpo-!Z0_>@lN(s}hqnT?Rooyi-a?j?6lHT87R#_iRVc>9;-*Z`@(tnF> zDa-hQxbMC4{gd~g$C(vHa_P$67#BP{dLb=Sb*$SzO+yBsPM~WL#%%bym z{x8zrJP^wEiys~&QfMJWRP@-gRAdQ59!tbnDrFg!O2{%~UxtK2BwHk835h8Y*>@rP zR+N3;cY`s#=T^`6>G>}2`~KedpO!IZ?)$#Zbm@ivP>>Pf)wxFn&M5(l*9vxc`G)kUvI$NbX>z$$X%& zvdq_8*XNcj$>Hw@XOvsqL|!}Fy}X?pBEKi;MTPZ+%?>V9gPGX?{TY@QRCjM5dHbn% zy0<->GOb@G8jfF-fR$VNX$Hxg?f@Jk zx51;5XBL1v9ES}w!dm|ZDUpT$6{k;77vpM}VtzUT948$5xBy1Auj=RB2T<4r>Oz)u zN#b9IB+~QdFPM6e>T@wm=-uww;O@rh)fz*BW}hvjrU4aempi%k*-sgZUlvTddg&3h zx?4sUd51+i`8TX#@wx6&>LOmyV)pLJQjx_bW>g#RqV$?kQy{hE?@JY@Z_Ier-BWKT zROx#3vO@)>O(Jh#SJJYc>C;fFPt8d(%Y%i?9-`)qsW_9pSxw*ToeSI@O|5X`2G``A zNhwT#=XeyJbtvv zijP9mjKLMB4)z$&KvlL5%$8MKEFQK=;-P#u)4;)x!8FhpTYmxHh6uUa_F~wopllc~ zrn9g5w7NVO(x*SC0NEYrNbZnRmu_y0HR)&?c7LP}ty_51DcUQbw~|6o^R;-*%cS3VgecwPhjjMLR|XwDbQO|;;BY%jQJL7*}I^YYb96YJwG zNsDTQRt3dBP<0B1qs149$pgk-(Z)VKWqms{PKYzI5Ni|PW*yAcaBEgPofd5v8K$^b z-d4F;L00~*d&8}P5=OK9L&Vz^8Xvq>RlASg2|40je|OecgkAM+OrTw8bnX0DMIQz6 z+^9t#9nlffYQ?)lCgMX&`N;WNqq_K+Pn$)use!Gi$jkG&6&Ky7y+tiyme1FAl{Qkl)o=1URg`Oua~^BC)pf$8U4*2OT5E09b#m9M^E+JiCF`wh zzDOR~i>q?KGzjw?^s1gzSr5KARDUJtdw_E8hn8;wCs~jG>0i44tbavY@j=q5mbvTg z%fYncs}dhRbMp7M%?w4HIo#1fLG$epQ+PC0p%gc&MVUcz5P_V}=saAL`Asb< zhmNTNKL zn6Rac<#7tN6ICn}J}j&#kBbE?@DXF2%Z0Ii@UAdJ8=4X2O7eJ$4hJyAhC82?aqy8+ z`HDQbe9(V98D1U?@U+Ct`fCJyp&;r!!Bt5Ics8VMwJ5if$CIVFgZS^{(IeN83>Ov= z0Y7^bm(RET+tU8RR1PBX@d>dNiz$0m2O|Q!*p#?E`PGDx>l%u|A-Sb6C6l2Iip2K? zJ^~O&*fLyZf!=1qlZ22`s`->)7hR>&+m*{qOosQ^HgtnrwUtVjO8ouN1C$+S9^c{m z_Fp>IoBjD&QF;_enfrgx?A}H>T$^J5I6<9N8|O(gq5#*27I2=Z(bvQo>Yd%eM_Wp@ zn0Nz!PxmeOL5RQ!{YRCiX+~7FRmW)tF!7t(HtadD2*}kZaoCut6v5TQgLn)N2$Pi1 zFIt@vPc@(XrQ|zTxU_c+PO0=dd;`sZg3>gK#J|iD8>4OW^P8`36sYVBb(LcO(?+#& z(DcMoFEmIcS3>t=OEp%Ud4(Qiz?C&m(utlyXK{ip?@ z_8P8>K6gVe;VvxJunYxDX#dtpntQj0wo-1BbH2dZGW!~F8p?_fmvE#`1-<@k0jeTzuwFt^*oCP z;l}A$Q(p;QLa&xfr=(-Oe>&%+$dd8PV&2}emPpg%>*)3pcq;s7M6S^cs43JiO)k$m zQ=LJqR~JkV@u>;@66W}OI$?0FZ$vB~FPW7=4e{M0hhfv}$L*%tFXe{|PD>ojWqWse zO^KT2Em(r~B^>>i;`{ggT^Z|Q6uABMg?HRy3PKV1h@KT!JMUT3%HziB{)nMy*$FKu z@?#thaw}6Rl7oeQWnUK<19RdIgyyqiLMXn61bKRqC&$`G=h~f}pO0$A7q^iP5=h0S z@4|#tWE3}dWAp$}qeG_7@wHGYdtH)8p+r&B)Mvp6bYh15b=#mxNb-C7RtwlbKLt^@ zrieeqhWk#U{momf*A(EGRD!SRv~>p2Mc(ns_x(vhHG(z9v!(=>74O)NkzEC<#MgD2 z+#NHEr`rTdhM=@CF8H~|HOK7YQf?odYg$8ybX`u@Su7z_f@d6i4SSd}EFr{`TG}6; z;f@2WFMUQr=|b**#>JpcS$vjbF(|5vP09S6=dL^)tEtJtyZ&w(9e%GEU2+KjAcv-r z^X8&n;8??1%s2+-G`u06)|58&$)3_V^YNi--`x47n61?usW)!aWx41x z(+Fu%eK-gI7^>UcD8{H`)Upo+Sl2!Gzn|UeUUB2VRXO)n3ED|KKrCd} zF?_=Zgwwv7WlfJ(-|7x|s&T^T?25zYvMg=FD%D~VU(!9Q`PYZUYr2;M*fWdo?L%)Z z$9IOTW)d|g#GK!I2gVpH6o7p^ezesBMzb@_wZkyv|IKE?>PbIV87!nt6nxeFRDPJU zbWvq1n=9F+$Qk3klX=sM0S z5sH#rqLivpT;al1h+hL$BJay87e^VdWEKvJb-ggsg+H_6&UG=Ldu|Z_rs6*I&Bi`k z#~3d_c}9*HcgL#msO?ne_n{|tu??D_b z@}lljNpPp0-@(nOX~Rx4yOX!)OVnnvBb>6YO#2@Pd=<~MH_+{Qd=8MqP*osa#oB~@O~zEvU)_@BLt{e~7rM)?|^-!nY8Xyy(F zZ$IUEN0->mUq5R+>T)aRjYH=<{diU%W{R3ytm)KDfhhiVy`=0GPF+Vw6FOm06vV?M z2$L8n8J}U#;3^@~Gac5kI$j1p>|KXvM}I}^kC4ZX;3lf8Tk;}JC#o#hqhBW-0r zCq`-p(qM~3Kdv47kR9wd3a~au=9Z1Udc~)6m6fqyhR_!%X0Q>=+LX$aKn+o$D8A5r zZObqw#!kBm^`%dY7cWs8F#3{NO62ASumObh&EOru{i{ilrR>DVubgdI&j+vJf=Mn# z-QB%{>UT1Zum&+*)E<8=idK8}6Ob?$GYTjAm1G&jwXA8;s4C_l+TDCTQ9PgK!J4ti3>;1q8L zNyd-mBE=aL(9A+YN1^Hm^gJKb{usG~=#MsvO2$)5aZ~x9gJsW_;11%Wpgrr93|fU! zvF%@iS{I|5zTzkN07urOB={PTI~2V?uY}sh^b<q3!Dc z`36n0(DqX}8>pkRtqh^VeuX96y9faBUcwXj%af`#5C3TYI+ng&)W*18oA~MDmf%Oi zNn(z06Wxx+J-Pb`!)ztVHlv{<6&F1&EN3~{-AEG=Uw`4}dsNiPLOmdH_Z7PpzFirU z>886COsNaFq9r>p?I?jxkZl-UG^wZN{4_|A1FY2}WzBBDPS`aWLUsQLkP0Pu^m3{U z6>gP4Yesld#m$H7F=4J57ECG3WpS|UmN=hy6Fq;_e~IA3A|&F=*-j!@oP9-GO;LeA zWpUqQR#>D^e={1NHle%r_Ox+54|%z3+{Fy6CRtXN9yBFThNY(I4#H6RH+Iamsll1J zEEARo(Cw-H4wFt(x>9_vzG``gY8$*SnD^>wkwtXQz*`V)S(63_D-JGF zbq#azlax=-MbDq|4-L%Fo!P8kF}4qW)x>|!rbuMn@eW!ucrO3!YeUDRwS?ja>o$p@ zrIOHeGD0$a{erzhuj}ELu#`x)!;ZC!-b7QhkM8`=Cvf1PCAZCGgTTgqPllVrrsH`W zOUo06C6zm+XG;>D8U3FtrIuodjz!fmc!gl1)OLaor#9c5ZFqdD(O%Nc4rC{=# zNvi1~CbrB?ub;B+!IxsjumnTn%EL0|ocl{IxHXH!q_owa4K^W3W$tNw9=obY>4j#dn_Gzy@RQ#W0?(<_b1*7Jz;>FRsC%O!BMIV{maG4DrmLe%zyvocM zXX<~IKZ(;znrPH=O);tSI9_1McP)s?&piVW}dqXOoGcp12gbD#h6`StD6Gm?tW?qL$+}vxYDGIB^?0A|U1}NE)77 zzoPq8R5DYqdUbTHTtDer$iu6|v01mi*8JBgX+2x%njdp^1WWU3N}*4M809^*(+zWd z*)Ygor(=DQ1t`oGjuQ z7JKPF8?T+uV|5M}icqZS ze8L}=40GDgw{DHpzpJ@`I_oeTJ?Um~?=Fcy?Q_Q4XKGZ?;3t772eM`hL+Lc&f&7thAfL!B}K_(Q-b>UGEL(Kp5o!Y^~25+YIa`+{b^xR19YE( z+~*W*}Ruwr^02%GR*}P)PzOSk_CeH>liWY9!T!% zCUM;<$m*QoQJKFX55-s2%#Fgt0kbpd8FwDo-EzQet%S|N%#EpzMpqk)0DuzC`LL zl}w3bvgJ+q3<3HQ>J2n5!iJ;u)9@~(n1vVUTtw$1^xq7PDt9$i|$}`&`SR^T_tu zHyhe&%!SeH>>^&xmN%XC8a~$_xiTbegto`o=`&qcd%*Nv-QwH#L33&INQwM>S}G)M^`BAvZA2(Ap=2Zs3xnsEr&vw!XK=N-1G)<@fV1Ow;)jvGy2H>;Fp~1KiP-4m zYNjaw436wy_~F%0-4DX+o#jYF+}Of)CMpWm>H%ArZ1~dlC9gc4xk8djwvY| zhZNU!nkhL}giosghf;pQaq|KzcMmiV7ADrH%y6E75gkVF*8`>ROQdK+r>*$t z=nvHFjAPbjx2#+6*669}=bNFjSby3k)S&-z@Daf}56;_dR~Cey%3ZoYWy|p8#LzlL zZ_$;lUg&u;yZWA(l5P+jE8aomtd#}tv@?ab;M)!tp{{Q-nfc*YbyJsPgFB|lOD(X&V0i@DEAq1}T!dm@ zB^`_wH9ow<+mFlp_J@a``UeXp8q@@|zs4HrTkW@r7*~vOZH*aXiw~Sf!e2LGRl?Ug zGi*==!!Oy?z+1nuCl-$l7k{0q&@i1N!l=}qb_n;qfWz8b!h$T+>84ZpED>$$`INTi zgDkVk)=FQ7%Pb7(@Vpl{9lZN$<XMj^{r$m z-hP?XXD-_seQsOl?tRmQuol*pcJ_@q_8jxYo1vvBNAZzEPSPq>>{D_&vicOqT;EbW zS{uO@a2+z|`s%^bZsk?stPE+d3+9Q*L`+wRXZrN~qRu9Zu_KA9=ks`eUTWSCRDGk~ z+8puQu-u_U^8-^JJfQ$PE-}jR9e>C+#wE(-m*dlnI^TpQlfwQ#!;m*2^cUM09 z*=*o~Ar3o;w?*jKyxg(YH`hMWFvUN3dnn5R8-Qts)I|oI3Zkf?k<1)wkVzlFZ%C|; zhPF*dng(AOalx0$3U;K|x$!8a1iPtU@4fNIkk{J#wyv=$>>Im4x}{T(`S?~X-5X+y zoimR%k)hUzC&O#WFd&Aq@YT2HTmh%!*)NmZIi)%Hq_`)2 zi>`nilYc>PgR7E@ev-cQnHbx+?Rjj;Y8%qu7d63($&MVeZeLk+zom0{NTd#${afAS zi#KT=*wfI^3()#WHa$BRN4ghX)IY_Y;$)(cbB&7j(x>H9Zet6zqj^pK*2mlRRs06- z%DuELWJub>Hy1?c%UdN}RPE&J%7(y!+UqdyysfrJj1cC>K=2XeqtFloD5U%Jq^hzDhZfc2 z<4CZA`5Z`ycL9Wf4i`clZ8E1tf)x{rZhK9}k=UlC>_h{hPSvhE44iH&Qt&`!1HMSy z5tNtA1gcILJyMU}gkhIZ&a;ETO@7b@5x4r6r1S6S#|UFb`T#UQeeMv#L3kd%ilu>+ z7+X&U85vC6J{^O*8oTq?EBNv1c!sx|NOtbX9r3Ebo$qmQBjCfO+vUNCJ zw)1%;^-)JcHp=iosbsPk+}X=47cEMJ72t|GU59n!7M zb_iIZl`I9YF$iq|%_#T`gtvldT7{B+x^1;6_cXYuv;K;0$ggX|GQ^|cA+{)OQzK7L zwg$CtLk$XG;+TG*h7~)|!{D|~;D?R=b1swkSP3(F_*^@@4|xyzzrE+b%x;@9Zy{BMt^CmR5x0%l9Q{Xy)A%c5`^bj=$-uM^BLkaKd=gpk|b@3jc#Pa7VC~J?qD$ez^gY2iTP)&X6N+NVkU1E63f0^bb&3$?q zZ+}3Q*yk!wqK1oi+70|b?Var1^WxJbymXIjdtmIAS~%K^U~pu}8>_oO$Di%GB~tBh zJ~9h0d>7AT*@k{DGAP)Vk;z?@XE#MKD!Jd^O^v2^ z^WBK-)`#JRcgS08w+n-7qPj>biB8HVPQ71W^2E@c?s|Jv(WHd?mGeqs3=XT?A2U_V z-&uHh{Z4psgOw(2w{(9HO&v?hTCi;W=OVHi*XNaG{gTB!$(wR+DvLt0*Z>k)C7qh$ z9HEYe>M&9Wo#kiLl!PV-No^I$(gaVcHK!@1pZuL_joMjlB>HI4gt^IA!+1)3WmNau zh7ncFOAjo4q9Q{ox?KDV=O(w}`fW$Q)AO)$X8$~ftDO?KR1sH|Jtm6kXs zLRt2&?U6IVDgSR9gpn5pm!l7ctA5{lZZWZD`>_^0^HuCy0B3bFjQMclLhOD7% zj~isln*?$t1UD8LLT!5VmP@%V*-V1ivn=a2ruHO-^v<6ovJr3}eC#WPusnlNgLx^C zpr|Vyg~liC1r3d?JNeC$XY@>9NP5L8RHW=DClx5dhB_%ex9i``t=9jZ_{80j z8vRnd)yZMJkBHYSXw)}wsz1=wkZrm`I2ByrpT;?XZs08D?k&@aT>L0f`vVn~qeSTs zU&i3dp#$JOLx@JQf6#NUcOa5&@Ew7UCdi+Y!FrW|c7(|ukXr%o>a3J$!i6+_Q-9*jK!f%8tjYcCPpI4S1^U|&c z;st$}0Obg_%6Ot3gK|dDOr;u*GSU*rpJb;rI^7w7m{C)>L4LKT@^qun+`u`c#$&*LD?!QFc{?6D=CMaDT! zH5F2~JMD`MDo+dV3gT0cb=fmCuTaxk^b7#To5d)*@=N%r+{LdR${)Wu3n=?7Dv89- zEphPkB@`)E?**R)I1Y&#QZw7r_5gh`?RCEsPZ@a_QTj!A(iygD%hTGrAd3Yl5`dNM z)Cq}>@ziB9RQ|e1U}5J?-cFbko-D_s!1E?EWd}<3!w6JLBu(auBdQ3A;>ho}$Im3~ zmz;vO!QYvAsTx2&4>+#rwC;s?%>_ef28NI*1w5h_?q>wmhX@Msayx())3DCxMBpd3 zTLmYBASt7j>~DqY3MwHq1|Z{JTv;+VcP1ZXlN72@6V?rF9*l%+G!Yu`I95~c#qW#> zDNB}>6hi_x$B?%?#e@O`Y-~HMGD;2^L%^i10L_rJkof@@fkZw?7%5aiLw&T7ACp6a z-~)((HB5+n0R}h9wQEr>zj^{M%xX}B$~0%4medV256VWpd) zsmL2LP6~O1T!2OrmcUkoh3BJCs5%ajJ;ekD1DGLH2V9XplHlvu0e4@su=ACe<-sLg z9x7|LTTSQ2e7FMk3Zy*9IoLsxxbD)Ful!K(sjYfS-Phqo)j31c%Bd&XJ1)5Q%LOMQ#7 zbJNRjIeTWXKTxZ)eNmJ59_$D5K$d~u&VQA0$5Iobb~nfWPqn+SD-P?rpogCKGJl33 zpZ_X}mka;T31jq9+}>&~P!NDcm8#hZ`MNA@Q*ZTdfH3j^bC7}xA1IwDplAS#-mCx} zmVib0L9Lb~RmHy=j}!sufJqp<53N_t6j z$nh>{ER1dO9E)33EFs9E2oi@g^iM98e2bbI>DDHoXOq`zIWt2ziKZ-FFLSq%B zUKB|$*Uaq%uYzxGtOWfxQ^A@s?6ZpIF5aQ%CiVk|HIFua$z5*R$l}`2J7|J=&M3Kl zy1DN4;tjL6;H0-|d)+Te+FCoSBtd z(LR;}vjj$~bcWOU94t983rA|xddFi~9>y)#R@8p6cZwOpC<_0OW4 zUf6`^6#jtyVfn)?X8UNhRw_cWgZgca+aN!D16fAiTh8Ul%d^u(2jgBeDV|>AZ+-dI zXrtb^uI!gF!d$-7Igh1f4b%(Tjt^Lw+}sh^A5unQpttFxw--zk>xpCRBjxJ<8LLf` zS+0pbM!RvNNT4uSUwgoq>#Ie>BbrBuARMj6CIiwk`PX>?m>Jc3D4)3^GH zl6Cj2c`U9Nk7hbx&vcZ$Bb{`=m$F~l`Fyp9nVfO+e&^-6)~J3<-c7EGgO{gI@}!TQ zCojiIpcTA`dV)hW5ps`L}$^5L`e41l-T9(@8 zP!X9d zu4Wp~mIscFC;st<%X~z}9;o-@uhkeN&Y4Watt9c18?Nf5)$YP%-R+}S@4gs#HoJH! ziiok&lg}#&$*p&^nXcJtrWuQgUZI3d_7Ow%6`J?;Z>pV%e5mFMijt*`uB(RYmckcg zxQBycx(RZBBo%&v2c1xWk->(w!tcDOjkh25JZ0a5$-NV~>VBbFH8I8(#t>v;2EtqN z=Y6tFM{}LIC&--x8{#gz85Oa*^YvSFmX3-t>}Rv&WjHu8acx8z%sk;tNr8Wg@C2cj z>__lhduwm@w*P|$XSfT7ZM7+DUs}w zwhBEaKTwVVbB=2tT-N6{+aoPRKJsjPYvA<21Bo{gKeS_=OfGt-ommv!i2sJxHlkOO zIp=iq!90&n(dl-RWYfzW&A3^Mb16JSa&QzpN#N}!ug~S-%tMQMp}d(ipU{kv+*5Pd zClJFvHH$4Jk4H%3&|-TNK9%;Z?(9e_hf&c=kauyG8AZ9-)h z2a-}6h((B61eM@Dql6NH*=!s*&EaNX#diTw$A87;5Fd&Isf8aMM!=Je93!N+R&Coe z@zY;%j|^ql{d(!}FF~vc691Htg@8GoFt-N?V8mBn^`NbVxXwsg-kz@?{<9(S*SLN8 z!?$1?Z`a--Gp1e{1|?zwV98Ck^3+L6I^aUJ$wvqbYkvjo_y1!=qL>x1GoN3Kyf7SO@Z`lBM% zwtxV}5CA{`;J|=UR6^!;LaYanWd|ApgcW*YywBW3xp~fBroW6$ux&1ug}~($JTEIC zprI^`xoIR|Wvdl5r`WiB*C}{ViJVc-Qou6^-zjRqBv$=-h@`2RI|*MJAI)aU;ZCK;UE%K1M?@u_!kxFCFn<`|5{qXl#gpbKnw2dr#F zgcoo;tP{VMHXBKj7%?s&zk#8`+=nM2xD=?Fuj#EhyDl7LIqmJuVfD39Zk(53-Y>_bG(yypJ2~3KeXrU@WT>^U zdt)$7xDK@v_KRt)Fm>)WmOFq_8 zL)kee!1(gS#}ygA46a8T-ug~=zAA+Zksi@`o>FZb3a;mR?DXXV=6xx2)5Q(EoHMa0 z?h?J%*=k055oIuNE5=*tW2fn;rJ92+u0tziWwR%}4Z|HIcv`8gIy_0=EUo72;GeRK zrEWC~QWEvHn=0mJ!` z_M?s(lu-$#wY{D(jBCCsLq<3tR^FtyTskr!6!n3#Fzw=F)!7$K%tRyRP6FO_(DKjq ztOOPFK!v8d_ye&~-7K=}vZtGJIAVR@-zySmQ^|ETIR2HrmM3p5B;cAW!~N`{sW{>C ze6cW|S)g98zB;LwIjMB@Zsb!s0Ehm9*-MpDHQSp`rxma^3=Ln{W|Li^q6N_(!k@hu zY^-?dmmfKB;0%Ft)l#TQM)H|i9PGQljEUj&AboCkAy?Xw$%mP;A0(}Z1=hT+Jh}!f zg>R^H9c2pb;$Ms#(OcQX=iJfvdCMO7VKMdaqSg>|9Z58AiFDNb!Eotf3R%uzUEDeN z!i}Y$3bjJ_>D`Mk47s1v^DiSs2=0G3)2YvLCoW74e7SoAPC5*eFZzUv8>Jb;0!9P^ zD##8$Sh8n)>*9smxc0qluI|hw(o1%x17?_!q)tA5M{md{|8czRqQ@%Ntbm@(G34u z07-m{@zORa3lDt&8#27B8;QbUV>#e&AzcK&SPH%yS{8OIfft1dDCx#(%~zYG-N%P+ zbItG7xcUZjKz-QB?&9;!4{auL1q^8Y#y20Yea)b?(>yubcy>Tcoy$i(c-;cW&6P@5 z(pq#X&)9c98@uQ2v~h!VqjJrJQHSj<+=oNaiz-$)xx1J2pzwAQ;RO*h$c9oUbFXv4 z|3~Ba|LtjzgbpcIHKTU@CCGwxX!AJ^wN=S zIU3U6%xKTO09&#H8Q}3Db9Dxiwu{t2Yg-APE?tMd6a9LwdKeLDwjJBeYZ;T z;js)+V6t_4-gESkY>>)1dV()jY#}@{IH$?GZ{OUFjvS*iokGeL+ClrQtnVgS24at| zGY{D^L7{ab)GersYMi@ClW&aUP($a6g%OQ%Ntej`RZHn!gD70rO+KS{M7njoicoS3 zr|kSqJae`&Q};wxU-g;;C)ef{ab=M!A5G(uZnuGV80D)p+kZp#ZajIO_R{ zHHd*82eT!gYdt$C*~sj~5oZ@yv9yB2YP90WFKBwQYf zt6YytDWYvLHtQKFATb*A8iqz3ouS6Z)-TAa{gQPWy({jVG7;3@7QN{ z-H8+i-P#ZOtZUL<-BSE|gO@rB?}sT}`hnVR-2dNH#sMHQE^bfp#alA2a4m}9|80h^ zz^XaEK#`4|z*=)ZX|0~FdbO4+Nb zTxoCcexlE0*i-I{8mf-<-3wHi@DCm2c*V7I@&d!Dhb3!muYaKQOZtg!!Mn}Q$Bg!# z0qU`O4#x8nn^eixWQwU7&y~30lKvNPpzcpyN*OmLFaT7LD%k_Q0_3b03|dTJeCMXw zIUP2m{_L{$xWTY2x?e+MK>$=LB!!lHCQFJpO71-2jB1?T*EdTb7-t9*o{RXqXq2w& z%zh6EPQPB0yNMoSils81!SA2y<*z+ido=gqqHVl)m*nQ_qSHi%60sb-9(i*!ve4ab zO|P>+*VuIpuk2DbQN@Iy^sd&pWB(^63}HsshZkf-@^pMVqwNBb60!?U1tK2sZ1FTY zhY)fIq#f|K_o@(&XwSMV0S17TfC^SNyj7lp!s{{{)IOYz%QKoMq9dYf=q((+aV84T zp#9@Ii!K+&l@50k-&bx}^;0}=(C%aJ*$^d2u)I1JSftkrVZL~`p*WCfK3k1_X;VxP z`I3`oAR5vGpB3gkwOv?H;6_MCtd3nO>FNskbM}UbExBY0-zeBIRaZz)jvFjFV7{f- zxcB8G?Vj&s|B@HZP|SC^_p(=SGZg0exn@sYoMCsL5m}PG90tP~y6-c{0^97R@iVW0 zd(&w~s>#A!f911r~=~u{fE7uKKDhn_?%t_Vc>twGxZ1`6`(Y@O9cJS{1HuFBsBz6S2hzd z>gUdcQs!>0EaCMh1$oas=o#0y&qGhnyoBjV`JR_^EPR87Bx|yiSwAjaiYdrlf2gnx z0&S9p0~rMEVzlMO+G1{i2|%$J)&xYErp)0Xwdw814s$O z%i*mkeOAse$m~E!4gOPIz1#DUT@XD#O()U~3c9JCAU+1F7#YpZ2P0i_AYlPr(a(y) zHrzp^U$DhI|FOYnD_UQ4L*W*-Ak4gl!+wOi{|fccGrBZTf8j6Qd>Qo^Fv7OvJDUPV z2lZbcH$x=jNSf1W>{T*+8Rma{Kw3sGK+EX!-y{DdOa}+kKxm*-(lCn#8dX>086!#b zupcYUA~eL5;duP?HetQzR7EY?wju@ zPbn1-@0fAm*<+g$zY-#JsL;5hcxpf1ly8XdQWvohtXq@iP> zWjLkb{P+i|4DGMq@t}BLS25prBci^o&Yh~j^QjtEHP1Mw&)BH!*|aTvqQ{{Vpq|t+OO4GgIkWlOw*md^s1d7pC2k8rR!l*B4S|9&u<4s!#sKwC|e>mC`1AxfUL` zN3iI7q}X=(@}sE#@aj8}Cls+^tEnuTILKJh$K(`xF>8VAe&Kb;0ZaYrKRl@)TfKSp z_&IKWkLtOICKZ|^F8wA^^EG9U?F-G!77h&h`LkE)SBC|2zSwVYJo!c3Gtt?1X_k#D z3mopd=+5c1U8ZSf(pI`5nrf=asjZ-@7rCOD`uc{p12lb;W?Ga-bu<`3Z9o>qp0baUwB{2Br0ch|+haZr1 z(wud006hm%nZef+JA$&cZTOw_exR1-j5xG9$x;057d7{123D8+j{^ zED-vSk4jI>pvhpNeB=-PF7SBHUhtY{5@a6rPcwOtt>Rj$Ot|i z^N&wV{QZCa>3+P)OonvIfB%L^I9v-Fx&0>O_MxRisWjhKJR;4Qc!(mJ~5(( zC)oJqgyCF8(XrK<3T?yn*H8Q2<=HMom=yTbemfnGB8#gJ{z*u!Pj=G$>KUo^;ws=u zB}p3IZ=t14Twcq>mqEQuxt@1}j?*Q!=s|z&(vhuK{Pju9#nyVnD=U*8Q#7nA&r53W zl(2`Be!J%1FEC>Xh`Fv5yH7H%3PDq&?c8 zjFvEQ7jROy2fnvpDEatlesCXvE989Zv%w_B;1$>OlHVWk)FWc*>3$(CQxG)#*UFc` z23_!0^!E8wK?We)KOb4dlAA-zp59Ff~3uPQ)IEWE*k%gaBrJ4l% z+7ABd9bSX!V{L^}sl9(zaY}M_+!2%;=ZQ6G-YmP>Sw-BfK!Tmmtl$ z(dz#+VVX2VHci(Rt|-?$m6=(XOjb#HmP09-w{2ptf4}%8JW*ce?@!>&bgb;=@^|A; zd6gB*onN$1&Fjhg)UsJSZpzbM_s_O>d^Rr^wo7QmT9E1z!&}mh)9PsyWk;@acKz-b zT3K`U;B0>G_eV{Iy?LW#RD5{6&}Jep7{EAGe-6q(jrNX`Fjyy z89iJKA^rLAb0eja$lv`|KzbO*D&h9neVvoWL;o_g z5E3%)lxD^}KM=SCE9S`Q=2xZ6mFb{J;Csd&Pgp03!qFgOJp=;d@4C zA{B0j1*3oSi~y@(C;{DV!A0~Cc-!~jZ8i_0g z6#}&W_Of~atc150gKOchu<^T-x3^}4S^O-NApu(4|9Sxb=G!4dulltw|K-qC{>{Dr z<(l(<2A8ggFE#e23o2SZx3sz<%HF@{K5}%T;L94pn84rJ^wooJJeSHR_Isk#xF*kG z{64aVb5ulZqft`HVYF^noxT0_-u(y%r~3Pir~b?0@(ZAKP_XUeFjJtu7l+}`Xv_KA zQ#VSI<;!D>#JE~minlfTKvTzNp-m#Xk03)dGIMf6i!au_UUOs^&f2euI&6D(G(V2h z5YDGK6!v_TC7_;l(yg3@E69wzGkcTkL3W;t)r0e0=&AGgt?0tq;=6`=+g&>woJD9y zDhX1n`m6>ywfFZcO>=`$rhf&dk4n&FegCCD#3I*}ZCt?n&u6Ei&!{gMHuU0q65og5Ad?5G~&Rso4f{kC;FWOD) zV}9X$(k`Y?2_3@^??=fxGAJ^9KFc@-xZ(F}V1{m{Ci@rFB`@P5ryZ|qx24^Zxi@~l z=tiDxRIZ=ThM;>lXOKNL$0hZ<-^p8eoe+Q0a-PBu)LjJGsQ>PJI_el)^%Cvs`B@1I zf8iY6qnm~;Ni9ao3!kV2X&Y%}yw|3l5WNZKUzsVYzolNG$E=+c7_!-8qbWV??G-c{ z?3*9C>i1w~h<7c*=cSb^f87t%P!eN#-eozFeJl1wfB;a-SOamG;?oVpi}%{NgXQlJ zFA4TCSq{QJ?DkAVky4Q9T!sG49TheWrWFKNJgHoK6Kh{5+@X^+a0Td&v1FIB9iRv! z^OBb~8JY+$9OR6#-h?5nl@)~hR|Vvhj1ZC4d=JeZDBF+YAqK`ubmt;zcbGGIj4xAh z^`*J}9>}1cC`nKPj=KHTjS_CM++KXoX#iaz~_J&McGu(;fXMq%6E?B9;xHOh`URcpC)?1poaM1TrD72EXI++qU_A z3n6%N8}Y>gnZ6BSRd2MP{A*1wP3{7$^}9v>+wSDc`@h|raMgeB5XFNp|GL1;C#q9` z0V4llo$4cfzx!jpynTPZ&vaX7UJA4{%G=UVR|6?hD+MFez~QO$QJ8@y7CEMmPfC^# z8DT^sTD5WD-yoG(BPBad2B_)UwrxwWGM_=dj3l(6&O+uEZ8QGcbK0wJ5!uN~t|C8$ z(%Fr!zqHQ3v|FHc9$rK0r^zq!^<#rk_o;O3bhFTh&0%MSD!uGDj^F^bBL4+ z<#G+En^G|(KBDw#KTZAYpGF~yDtHOR|c7`;1on zw7fxZUIT{>?!5Px!5A~zK~FlkQ226$2iWevOO8ju1SX`KjgpdGfTziCaiB8)S<5}a z?-w}4_C1Z8iwj3D4E`$LnyEdH3F9tk?qB90dJi5<$>hi>jwrePU|!Erzy?2;SVJ&2 z)&r%Lpcm0P`S98Dd5t4mD+{L!t_}Nce&i+8Q7n%w$j)b;$fDQLiCF9a7QCO+*IHY8 zc3-#Liq&Ys6z zdlO^!xK+AU%uA)&TQIk07Zk~E82$;y>%NCkZGg@tkdJ`8UJH^J+nLbv+aqem>9p!N}%ky8s&dm`+>96&^ecvgh;{UDL`l799-pE zLF0R{VOO8?sUuW*#?Q#{dcczp7N*pgY${e^54HGxU%HSfd-2qUU5i1S%hLPXWCCBt z_sHI9m#;i}ohH25R^$=8)@e(naG#>5*LFL#o(zgRLKR$>iRc~>+G_dO)5|L1{AXsX zz^0fRi61I9F%|lcpRk$wL@A$+z(nD-1QUQ**S;JbW5}g5-eD6R{rzkVrlKCkizwIF zjm(t!6cxVr;aY1H%$1ddku5;FKMVuw6oMCqUP?JD*&6LsDGiVMyW!hCJbiMYfa8@Zp&k&d9gRbJ0PNCG}Zo2chTF3J%4)J@ZT60sNps zwGuSSZmSjGUJY+}vhkQ5p$I$EO8cpicHAR-p=t-`H=Vs0IKxjBM6;nZ@2VtQ_f;W{ zfQWn=CSm{FfN&zipg+QsHKS}@2O%NWX|8w+2RYliaDZ|V8U&&6j_5*=#tYbfj)y1M zW5ZC|FfZc{N>)@d5&HX)eS{L)aTCN9P$Sbv!Ndf?I=Q1(AG$~!7ry47&H^HbmEMHW{AoJaKtTutgw zi0Px_fuzire$7i5kX*W`gf%Cbg05pM2Z|x)JDeS>qP{qJ^kD+6B4!7eRW{?}nk1Pg z96F~gESTSzKGfFYy2VNmUMQ1*LDap0s(iIabl@ ze?HS*)1*Gv^>s@ac1ZxX5Gg91?=E<*8%)h63ZpMVq)hzh0+}|~n3|&V- z$#Sjuppvwy(cZ;m-9pG(x2R#a`1?DX%dXHn~5)vi1=dGSN z_L^UiX-K+t!}ytZ6I->1=SJ6Mk^O19c70j18{ENEKK8F)y9yJsChM>)_b~^FnN!tz z+-Dq;D(<{toS$cp{lXKcEZ4#79WdT|s>@B%! zBe~9^1L6W!4-ik1n`W~RhV6Czu0^4s3y1VT{tvj z1q%vFiHZv7Mnpl1gn(?iQWX>g+%}rhqS6V1iXzelMIb0$glrY0HxU7qj-W^f2}ODf z5aOLHs9VqZopZl2?)}F1&-ce3V@u2XuC?Y|vpml;QNi77lQ&dVp|J{=3G*6a0wmC& z5rCU$DS@|QHr>>#(f?RO!L|{zR{ag4B*Cp=jp5P+;A%BGkT4a115PLwKV zk3T1{LV;HV2<>ZX0Cob!k$Nvy_z^@kl9Aj8$SRg<33P+D>Tkg;7rq8pH~5|?Z5U*# z(D*ngP#Ky!w!kBRRriB_rCfbfu%ABhD8eCOR|!gNiB12IOV8QbfkcbWL}s?lr4JpLgB~ ztC&7iu%I9}GfgpqzDqtdV(S&Q_8V8|3gzQ25Jxml#mIC4CGx`gkp;P;cROeCxIRE^ z6b%DyrsJ5XD0N!P{gUvymnK)aFp4Fkw7<;MTU%i4iU_#bZZ!%IZ-QR}Wl z>_YOK&}uFo(gsGB5X5s6x434=Wp2J$`v$SAK8>u3?5i)!1~D=RTYC^|f(!dOld{g{ zsm5fwrttHT3vXNn4>mstoPDC7L?z{rN(Bo|M(WcHGA=;_TC!+Q%v0bm@pt8Q(B8X}o)RN1Q4pLL5o5 zRX#qv0+e>(w8$iFh=dQHxDz-Y#|(oR=L7|tPnGs%X+ zX%$M&pQ*DL0s>E3wZkbw>4TP@l9%j@y^|Xlb63j;{6@Gm_YwQzRyb)*Y*HZabPf92 z(Z|kz`bq+HdF`!N<=y{dI6gNwOl7q)@c=)DXWm z90SPy*FCQcQe}Xl-_`G-W73?j&bG=eip{ z`1tWP1u^%YCcpb+y4oRz*Op;g4s>1*E?AE^Uscby;5%VU zn60+4d6h-79czC`j0*SO=SF`E151n^sI`?<*sqi|{LC_K?BA4wG>@cHgVI@r(jRzcXM3OSR+>J%gInOS=LdLn=7)g4#G;FJt-9nD3be0#)c zxCf5kFczhbi@6Wy)EHd7M}Q4<3j4Z1dO`V>(|(b~FON<}1iZEHpbZF*r88plUi!^6 z*~q>*6TO+&HFY(vKJVGFjhc>$^KsQB-2%IxJ;{}PU&ColXldCP5EqqECkmP&?yIF4 z@@p<4Qyjg~tY4ng(a(duIDfu$FXw^G+ZE+6mQnPdj%{2_;H?_Hv4wo+RJ(Y;Dhk)XlT0 z@kRMXt6n7St}~37#dX)?kA1fb;79*ou?$F4*wVmrXJjB!lU_9h6f&_uj6%i(DD;3e ztC?I{e-uGGao_Y}{l#}+G`TWG;)QB!S^fB8SWw�#|_H2W^V5sP7|j5Q)`b)}tSF zj1lC^w&6SE3yt^? z*qxK#=Xy6qMT)BI1XVF+B!Wn^)A6y8ykpoc0>XeF1p$qyNpKnbu>lD1(cKAzb|zpP z8vamCfN{lu{A0#H$Pc-c7l2SnL#PrKW`6^8I-PmL!f!Ht3uf>zSGovZMn+PXH30b* z;2J{+73Ee734B22!>1jj3syXlHx&<#<@?~@rfek+I;Y96mytmdi>?>xLiT>eQVr!i zMZyl{oryWm?iECf);zZ`WA)9Xtn_G+f7kWkp~kqt<5mm^Y{pMO!o1gYq3-}~ow=IP zE!vY3_XAd4M~|^4XLEM_T_IYZ-X zRE%|Bms^>7R58NYm<@S&)9`{9R286U*Q4?Rf;WOv+}3U{RxSwlqJU;B!Ycda#I_I@ z&P`H(xQ#74>Js?se!Ho-?uYZdZl4ngUKlFDQ0gP?|FT{i~5}BBnha_o`s;bE!qbg3+Ia4PJan zH$FHXm^}MMW=1Cc)D@hSs{KenYu>uAsjuRog3p<>W_;!J74e(W#@Ee1E1oL51g(kd zpaJJ|`B@ZHHKCC9EVn=@hu^5?h4r$9&VpX;mIi#B+B*0th$yl*#EAz_v14I$;9sfz z+X1dp#AVWQ+UnRy$CnL0Jiaq01{)W4&Q*v!_Gt*n3_i?HwuUSak-Ez;7i^q-bm~Sa zcP{Ny7V?R58eF)x+CIAX^0TGA)B_qwWF3$M>HGW!T_JPckY3X0;y#t`*fL3(&F0Mb+x?@9`rnyY<6$1?tbM*ufDrMxhPUFhK zHuhV3x%)~$>$T{n_Qm+7e7U)*yOn#$%3(c}*#pf5F84g%AKOWBIP?i` z8=znDXt0r4vjKh*_8v{dBkqlvXx_k=_uI?qE6N#_W>1u+%Up~~aUB9zryQuJF>Vf_ z#Y6PtgPDC_aK$=#cHUTETxMjtP1X;1qRGTmgg+ zRXLy|K(!3vVIhPE*qQ5636MF)kewgOX2VP?mIYp2<>N4YT}1ImX{#=aR-myar;fWW zM%8%$_#=qca4&}+_zQxN(xW)Ul7(}LADO{Sj2=gBliYu}ZHIo6|1@uk6>O=9rPXJH zcZD0&%S$6RRh5=p9#lJZ{ur9bnqm~@j$hVnJbv&&LuGDjVdsm$Uaaf1r^Be3tuEA( zyYsAO4+*9_7quidn-B_iZQG`FL53~lsB?R&+m&&lV~khzyybDmGByw9U9MkVH9stv zALC7N&N44*$~31b(GPwMH`|-Uf2#qvpZd|GXJ{h8CcjpakH9$)JP=N};(38M8(m|l zK%L^csfMmq@d8HyuTkaQ^)(h`fELaR8ms%*z5PL0iXZd#%#_lh%Kt8{PB#e zuz3SE=A^p7o_El7b-qmUJxU_HZ_@cgy}(IbyVoGVr;m>9GAlJdXz95*_d~u3@v&C* z!^E~q$<$z#<2kJ6cm}+sPIQYx64&=J<63 zMVwQ7ch<;Vve4T}nKqm>y}!0ZXDk1;${nFY_Z)I?-r*{0={S?EL z-OmTbwp9v-p=+zPPdw;|F7cbn~T|b#SMH;hpor@^ayk|RD z(*!k2b){jm71 zn;++YrV~1EVvk69?fs?3jwcFoQA@Ah?^(1!H+Mga zCCGTNfSgFzfCgNlkRby<3JHh&AVM3z9V5{FX3S@r5vv+=C5%5(1fCxmzK^ETu(}dc zyVEC%$?Fo=D_;h5BeyFMt}a91N5Lt9*QfQP>JTf;!%}uwbw$4wX$wYE0!D?%0ea7C zJ*wmdIl$Ggi*)rcB2eu|{~<;b2%7*5jTeX(wH_(UA+cb+DR2X=g5oO3joASMWm70n zd(2;p@L(%GY%R;wKyCpHL-U)swc)>1t}8mWFeY-}A>DPaFM}@Rd4;gXsyW@U zYOSHdQ}VVEH@~)+uE7Bj=;i7un+NqAP@z208~?eX0r@I38(q(p-)21 zf*``)%95?`lD&ZNk#UMI*UU#&^}RxZf}o%ht1L%K#;KZ887?C;ltj+%eS5wTZntEn z^%W!wNH1Juy=`~E*)95LyEOYlyNk(OjAEj>rsbA4%B{=EnOgEsDua$q8}hHUi4dc3 zJ=v`>EEhgCi|ZoRRgF!~H>B;%7tI=+YwkH~F{D&uy-!D3i`J}1U7-pyt<=n-^no+! zuDN|?vUGlvlSg|0CbKxq^-2MUNnBF_=%E;eSzBHq_+8yJw@q@vqnDI9$-m(7g`V29 z2Mqs%bxm~#Bo+sn-+Q!3iq4sjl6r$+3@z$W%X>YBypv%>@UY-1rTZG#5!xe7#cI?A`{$f|1|Gv2`@yR#L9-S$Y58nM@d+KiU z>d3i~B5v1b;L%hZM7-NIKF3=_=dxJUT2~qK){$$gU-|fsb~WE=(Q74IR=249*Jb(& zR)&7L6LI!^0;g5-tOeIt2-lv{^>1CP9EWKf#~G)algU5{+g#F1J6M&w+1Q9M`w3TJ zL^|biOxC`!IpomfgUK_4MF+X}eTBSH$Zu4>#FB}L^;!*4Yb;C#XU522iy(i8g<2fD z-~$-M3+Uv_p!{1dU8FULcu=reXRnQYz z4qQaodQoi{KfVPt4ESf_*rz0m?qYWG)@&{l0QrU`wx&9y6Y_CNxHB|zTFcsvw%gYG zI}x@q5;Nst9Q<9`?X?-=?Sr$gci$+HNMVSdakFThp$3fyq)6Y)ca2QQINzK7qUA~4 zq3Epb)Mj>0Z042rWn&rh=NxS!+s}WXXk4)P5@$ds7<7yY-V^@JzD08^V{6?6Q7){Ow4ySj~tX2?lLi7{VA9%H{)q>f8u0r&X+Tf2nXACT>RrFTnFcd{GQFMIYqopm6bhr7=uxpM+R8_5jY^ zcVnWh&WbgT?G>ERIjG$Fl9h1+SJOm6*O#_v#wX27yCU=~nVYa*ky#LFkj!X^X=JH! z&N{NC$X99=|M`~fm5h1!cQAN2;XGZwk(A*seD3v^;cM4WU*{X9+`$xXq#?7TokELu zyN^1(sy?_~gZf$qBy%;sXm;nnW%TWe0u>S&M(Q-)UwU-kJ<;hq`sSgFTt>mK9Ww6L zDG3XH`0@?2{6nB`dOE**gK17p@^W&sRKMK&JDUPYdoF9an1lxS9d+nf6(>PE+!jdB z2r_UBISE3;Fo@y8e83VP`@}$%18zFz ze@K$;|JfFZ2u`8Z#&qTj{Hl~G1$SKyUFd2|Q9iT3jAt4FcSaZsD1UUqhqdZ2TC#9u zo~QA{B%F&m-zVW1rJZ*cBCk4mmZ{I3%=T0$K&}q_1^CaVFmD9h2k9sLKDjz?z-Wua zLhDJi3`t?Z?JS1Ia6l0mU0_g?_ZErmWwxvQn5`$!m1avorFi`V^*&Lj0jU8$goLnV z;$j}eQw6MabX+lqW!_lAVhV;aLj%2|c+roN7~ez?A%}K27Ai}rT`61cjudonnk#H3 zX7c^57hYV0=k>hT>YO+=&@RwuxAA3$y+T9(*L{W26`=DnV}5Sz#)JN5jxTn!4QmPG zwR_&Djh;y*J)bsblSozyY&# zq{ZV-D$*Lq`sV+6e-(H8k?WHaA)FF5W8wXTgw08LydM~b}b~Kzk3#C!42Xw#~$$GgFVG%GoL@tk&imhacOjMqi5J zRQN%XbekG%UboLnmKt}dc;*#pH?*ao4FACF865ve9eR;+ zbM@sYQQ-TYDO$ydCARo+9pTJ49oz(bla(THtTY`!>D;|)T&UP7_e=-PyKdU)nbZ1x zW8-hqZ>q7aWcX@ny@yFOl3nhWwdm;v>l2yab6p9{O7}l9miyc4ipwMDKT5UzS zH|Y6vdZozY+(7xIW0Smhgd^=~v-m84qo(vCkABuma60GWE0s{%JU$5k1nE zr@^B0*fooL-ku!6(pbaT=YLqje24q58tyFAkU?}l`dcnu_QxLglI)gGKY z1|_BlRsrI>V+rxZESTq}l?If+J98v(H`GV_K!fiLI3U2Oz)P042Z4V^x_VF)L*Npm zgNkGX0z^Rs2wXP!bbTlZM(MJc9qra{!M$F?XKc2;2zMd0b+O(4kgdK#%!eUid^Y6} zjhrM!z0sRvXk!BPfsg%1{uU!tvJ?Td!QaGUt(nH6tA6A;A~jy5Ox8!F#6$qllUE}V~q-Y8Y$b@+f=ZNQ`LFC0_B8z zS7J6@hgl)JrT9p7k9Ob;6?Fz}WR7a_ye|r!B#o`vtyIeKP7KU;=Qj*{$G9Ir^iPTFW(W#ngE;fZ|?G5w-`$w*8QPG#G|J3l1cABFv2d;R@&p z0EkF#a;fO~3Fb!x(h|Op@NNJ(jSJ2=Xs6*=V-T5m&&;0vW4n}gIBd?hzA|M zEXG-cf_Kd_s}{}9%m(4408#px90D(`=ana>o-?m!ji%WCihYU`=O47??%vKi(l!&=U#{Upfy{k zxHGoEpk4?iphbi_uU;FUdn5};s30;URA}Mx;Y$6?y;cWP%e+e5^kbNHJ1i(hyT>k~b-t0Mt0kXhwiKl;^dVZ4QdBe_Epm1?KZyd^aWmtfKRoF02kSYoQ3#Kz& z$pG4n1(bc=o`~O$yqsl*+j94%%A1GV-zO+m2=G#q%dulmCvOo_4#zwB$1}e0F|gWd zdLFKq6EeEvW^+u&pY@|qh2U>!7MNU~I&d{N>clGImO_^5dNrXjdBb4#0GTJ(!$Lwn z-Ph*a?-1U>|X~d z^RHX{*eFsH%ga4kqrE5bu7bMis>jbNZ>0|J*0f6nSwGccKv2U1Lr58_Jn25KVSy99A$;%Xf9ebf@tj%w6V5GeHrqUE+_uNypJkkl6Vtqp}_+UXg0~P zoN`T!6*e{@ww_7BMQBP$&A2gXp~H#;en`;?l}wqaPqUA)-p`{ ztzpL@5qU|FyoazLx*~R2Ojr`A*K&jK_7cxtjVbKz#XU8P3JmB2GSKbIC8eeR{HT_8 zPQ#alEQt&Y2D^k2a$%w>QWuV$>uyl1ab!f)g|nq}A=A3}7~ z437AuNg$FQP4ge_`I=IF=xC0|AR+BdkY`vDJ^=OtdY4mbZ?COZ;|2v@n4ncswryZk zkp=N-HM~_1AW0#Ne(&YjNe#u7&hZJmt|@+Z{|`Z_2?2Cg*XvUOxEC+z9qxlv0GKkD zu;oy)d#RJjmMFp`v@VB&Bnq(H&Umlr+=4PAsPPzp#vv+t0$8?Ka;=BeatOs(pA7*Da-MgeSi{f3DYA-)qm>=yFtWjk(} z9U8mXfm7fcEW{Sw(QRz+J9R4U6JSB@)&Y_*stkzqQ`%sqfA`TU4HDw zy}N<0v!w^NlxAGe_}Vg9wHb4M1Kt10iN)3cb>Y|@&W2tLKH7_MomchreGMTE#f7ZW z(vGGjHZvwB(TC5SsOL4Zws?m-u+LntO~aw(tJ2Mh8S+c!Dy6n{UCa{_Gf`%ZLlcP6#Bkl0OdgJT?J#N2aD`7g#<0&9r8bSU36?cX&u=bXEVri#L-3A4(~QmJ`DoV zfb5kPHHsT1Jjj|Yn!vjcw-mi#9E+P3Bd!1W$Uz;(!LRSrxoAM<15_ITZ=Jtkd`JGO zV{9Ed$hN{RdFumCiEv(9EUw+QtuV1bx1;6kZP|I;80EltETeLq>^nWyZmiy6aW5B> z^G2tWI)T*e_s*JXn#MDz6p-Md6s8z5EGY$+V28m)nZv^H&{0yWBHPS3wI4go zK?3 zz9E&s&_J4+5(8jsr^thZ8PdE(Zl2bs%L_xYPm@m9Rr~v|4B1z@@vZwYVa8f99fq>` zL@7gMbZ#XnU1u7%Kv+D`X69phPv6;6m|jnR?5H1l;(=5PAu3RWW|OkYWK{>3na1tQ zpFSPy^wO*Bmmuf!GPW1|(b3YwYK~5tOZ1o>m%CJ>MK=8MmO*ZTrk!r}FID8`wv$QhGi3CTHRf%-)mp=q*}9Qu-n|lwve@hA|5g zO(zR8%jcQre_K&z`Fc13ySrdsPIICQhY|_GGp5DeC0V*IEv3m;rZ^>s&QDXr*=68& zfXN=OF5nb_`Vq^|T~mV7wu`{V*l3Sr-&?%ohq~pA2hF&1)f;g z?Nq{wRI5WDPFL_`YePuxaz}J+buNpP-E)hB5fXgX+u; zAUj6ITIBJT3`OL}4;+7ist$_3*b?86a&air4s&4Of!FSbL#Iej;YkgSE~d+bf)_Ay zw2WPOVZNDn5;^VIpXV98t&bAe-*48pn1o(;BN3GL4Wyg0K_MI-CjRG3p8DV11(=kX zg{2pW8=movXIgYl^CJi(S-nITf4q$JOhJ5kC+Sr>F0b79kS+(|*JJs^5xhk4W`+k|0LuS7h3J~; z6#gaiFogh=pZQx$he8Q9YCRDx9YoM77=84-Xhm;6x~rNzL~H0!H?ul(#6l_sb;$uHWam@oQAxO{M+f~04vKWv}SvnCb53hjx zONo6<<{tpw5B$9Zze7_FdAS7?jeZKZK*JgSMgex&At3Sv@I^6{UgE>q8jq1uHwo#i1GSbjR%OGa66@-~Vng%OV*wA>@mdK@YV?pLBiMUB2ENJ#Aauxbp{zSM z?jH7^uwS;@R>G9m^;B37N8O3OGQ%soHRA?9FT6$r%k+%z3p$M-E^c!+)CvRS!L#>V zo_U;H^i_l#SLLK=A0!{*|0KP8txo+ITtQz$FnYEnJi_U8$He%Cr0!C-Y{r#R`X7bT zW;`Zp6>g$tLgd3vfs3nfH(bce#OZo;1GsQpL30jUy`H|Ks7f%%Ro@4}rlI!(k<})Y z-`An?;GVa(ERBVGihY6;jn7@H;0T)_jdm{X`4Ff%LBW!eXdSOxQmAhoY~v?`cmuc+ zg>VTjIX9mta84ApKE%6I+HP1!KW=WX#~UxPPXNS)GF_xv3AkHS@t&_!T*Vw|qKp0C z4P1@=0p2HpEGr1El3V}1E|Xeolbf`V@!QyY`O$9MhdVobNf}}qCea5~qBpC(UuOSt z>Wwlk;p&!Mk0W0OTcrqV-TH8=LPn<7q2Kz;hYX|e7bmNwRpj=|OdIkFO;Dt077k*W zFXOr%#w`r*y=sQ#I}x+UIIUCLT%$GX96x{5a}NXC>%mWQ^Ram)QHIh<@8h1*Rm}G zP23o>j*7D#zO95bhcjdCb+63dj=1vjo4>fL4!jCuscN8x?gEqyzmXNy&m$RiFtCXZ zFu)wOCa1b1{B>DTr%fc5BRXVOy-01Xcl(2LKyprxKW{Xxh;drE{se3Yx|N zI#7*$>_Z0Iwn}3DC^HK;0R1v1UI4}h{2$x2QuRJjz`($`9}uboWFL3_8L#!wnFa8# z*iLp_0yl_B$2MpK0T&_yj(?oma){4S#{Ppj^&d_y3+ipaT(2ujP6u#AKwvOdu8%-k zp-gxYzkYCVGyDUJ3{bep1kL2LfQG2o8wK<+DoS~CGWTg|k#$+g)~I2kG!O-#r6y?e z0Gqjo8v{ZW-F_l$-eV;Cop93kflZn3Y4SLl>hPiO^kyj-tGrnfeGqtPAT&ew2dH|i zFm%jjPjTlr%qy|;pmqm^4a2`@p~O$BpqwW!^$4J(&-uMhg}f#wQ#d;!v;#G+ToHR2 zW<9{XXhc6o%vCgt0)Z+a7mH{KZ4W}>g@B(n1k|t!{$rc~cZFgK&^;AbGDj4hE#Sr^ zM!M>vmIItgFC;pRmK%Y};PcF&stEF~@mr^=Fk%4J_6ZR6L`tqUy@N&TFCX1U6!7D_ z*)x(QHoW+O3iAYkROUB9UP+lr3ZyGN#Q}oA*7tAO&*h!bYLo((2NC%L4Wm0DZB=f5 zTkyep zm#)8p=!h&#t=VW^e2{z=ZWBJ%Rd3GCF81)?9??_Zr;MKIED9G}M9O)tW+TfS8CE-I zcR|H^=oTC)WUWm6l_)S?p|0p>22fYv&Y_u_aFYU89$klAc-j*#`c~sa&dmR$Uws9} zSScy6{5Oef`R=?Y-(@2mN*B9HU4hhX>cUAIzibH>6rkO+O6K4BGLt8)H;PNbzM$}S zb7OPrLUR;dx4vbe^pYE9kwM=6K7Ms{0k4N_tLMGW3qhXmVh)v?=P@{H9#!@gWW5M) z&=#j=^_(I2HH&fGxRJ}B**IQ)t?^pKz_cE%JFiWy;j7%0G`~)dm>q%s!4o3<7BL@s zTKeMU4Lmv*>^)6Fgd^BCZ}s{ppn5PX)9s3dPh^T&!tqQ1nt#I>obr&?;?lg+K4CYS z)xT|XCqhU9ni!l>fk7EJbM9C<_|c=1Q>bl;92W-qxyw)ov>M-!Kq>jbpkFGCEH)B@ z{1uiTFdvvWCgA|8M+m6_l}E&?%*a6H+m#wZtN?_9`err|DFT2ue8zv=kMk&uYC*M0 z3^>0v+pyeSqiRi1mj2$TVuL=h66(=K=}));abFbl@-Rq&)gznfQ`kY~(hA1G0T+=2 zB6R%)eK(lm5T(oZOapQda5w9Kf&rLUm>-7_Tg(YKtDOML+n@ujUdY2>>f9RmBM2A* zcxxu#p^~3p&)b>%y5NIYj@SiMDHR8QghvQ!W+@ZKqxnEl7x1It51_yu9|9B8s4z)p z2)!{(J$WNi!wc{fgeygQ26*Ni2W`PA4@;JK2r-~w%S816paqRQ5f?6myC4XBE(Ym~ zeqL%Q!&-+UvIh|q{{V;~o+(_g4S125L)_7>{NTcW+cK~JS2m2F=lTi-z6`eSy2UwU zEER~VI!nFK+%qr?5qK!Durl#AaPJa;n%c27CaiOrEI{BO`Nm9jKc}xF(^Q^5gB*B~ z_rsGt;!xz7*z@TJMP(nb)Laej?!>U4vj+}b0^AA^tE<6Az>#5sQB}}If$+NyO1)GcWm4t-}M)iMlO1h##@_`u3sd)a6gdPBDnc4!_$Wj z(7(#m5Y5gleq}1>JiNNdWhU6SOlK}WqH+)8k^qi73}+2U$!m@hig5z{a8U{u?s?DH zeWH@!d$w^~Mfj_8LHqcIWq0mAYrCZX*~GO=!#nfMY2;F4Q5j|F zNJfuiPpf?zFyM_I+?;(Tb-v}@Ska&>t;T7Es=yq%=lmQSbbNdhWa^RYMxv*>u1w4s zm^Iv$U}O+EqlOT|R7{Tg;z5t~t6bV|lZU{s;tfV7@-*Rbu0@cQ*v z2JQE0{^fDo(P3I@>Jss#@#?Fd+Z(UECX6rg?+H{ra(Ox)Jhnb*a3!fR&r1o zR46crX=;U_bcm}CiSf@322oS+&gIz{tWg?{nifxQpS}*Y!w_<)|oQ^>>Pe zC5fRyQ*(0?oDFcPcsMIZA;3}t^&kxUhWaQ>xnvj`qX|md;3QzAnJxws+*Q90j-@09AuNG{ zpwz!c42uVOexWp&<%znUu^hvIEkr#(X7CGxIvs%0u7u5k;pp%*um*kJfFl7oJqCi@ zA*L#gC8j$B4QJV_D#CrXfOUym;MfrLJ-`*MKH$joMc}Xuk z!rz>+J&R?zHx}oVr8m9(@7K-8Vh5!z7g34_O1h%0{X(wV&C{>qc^BJ7cNavr^2`LA zwPc5EZruO($Qqq$m5Y(q{nRkyt7mGVaXfrP5#<-PN)_=XH8AA^zeElw5Epi)&|1@&a^y ztw-Smj-*ALd1@0+!(OX%%r2M+kQcqj`&Ni{|S_;;(3*~J5{dsG8< z8I7Q7ijHXv1N6(Z{0~);q`(M@$q0&%g6(}C&mu^Ls5cH z)Hrmq(UMaJf`(;g)-JzV3&CD@{VF}ph=|_+-qqOJ0r<~9vRYAne z2UDk_2K2e1pekVwP-tD*ax+URH!&t-5EvAiqOw(h%7#c|ubbZRQL3ab|6va? z`DY_F4|6I3k!I!31;xk!^$wisWJ_oPz=8j7SceFUPFr_K_`_;QxtBps+6ZO^wTrcm zqWg$T5>^oU#y}0gIX-p<`tjL`QGL-cJP<-@5O7QK*JVSfQBiE7l^XyrAP|K!HvA$IH1Rva zKh0D+fqaSzl|?gA5&ScNABiT(CS8E&9TjNsDUiOGLg1wd8G_kV7)!CUB1+mNzG03c zjKLA8r1a4RN!<@$;HDdxgC#S5ZcH@Hkc2_ZRLKl|v_LjXk>e7%`9sP{PoSax;8DeL zKKFyqXMrX4i9!LNY(>K3ux>9tZyjUfBkNtKVtrzOB&kn~Wa@q!6r?~}vy)dLJE00A z?3sVu?VY-KTEdHLzODx+1w52jZ+WIs?U5oG`C`kZLvfY)Yn&Xq7R^SBbE6(eNQ5-T z_+Pi(>Y-K@X!>Eubf`)}sDD=9*n}3U`L=yP*hMJimLq92rszns#Utx6o0j7*Q!c80 z3bWO#ZpOJl-RyyvZB)z_Qzrl3f&1b-N_KA9a`+GOq`6At;v*wv7YPw zw23f~Y8m2?9N`bJa_Om$&5gcvDKXd6klO;E{S&xM@^MIpdSvb!ryscG&Vc2w&rShV zX)jIBdMH*WS-mViKl|#A*_9ZE3q_A|NYp&rKW5~2)bcj?T$u%ZSfi=H9md+kl7u+& zE3XYv3;I3sRPOt--Tv!cYgs2_@{O=P-!NPnPX`Xqm+v7ZZTMrym938Nrh9~1-I}jj zQl$cRUAHc22!G%I>dbl7cx&a$M6(U!tIc-SJ!;&rYBxPpc=ntCiLKFXOULi8GodZd z(W6iqH)PSVu7Xe!xH#dFnl)YpMS+;4eKKt;0uxD1LF5G~`>yejW9NqzM9jWn_7+K{ z^mqR)ai!~P;fj_H=_hk|PMa>37vjFuqOkCgiLQ^K{jprs$mx||g6KV%98Id>Wn_U!v%LS&~p~Fqx4yRw+QPeU)CU#k8Xj6D7 zdrX1IRx0R_07gz{Zte7c5JFqyGFo;7N=`publ?f0X!#qCF10rG-@?z8;2lP-H9;m# zGVJtxcZ4J+dVX9}?NNL@6%f-&3K>qZ)4cKM^JN-bs`#lbKd)^cEe%DEU7Tu?`QrmB zfIU7tRWZxHJ@r{$qXV*mz7AiK4lRFI$-j8U>X=RVzZy<8vZ^nPNEhB z06@io0D$%X#FX@d`>jObBw<8r(ggbPs-nx0SCVWes=?Fz@I$fn^|~w?KEYMu@~(PN z48Jcs1k5I|xbl{e(5PD0dSc8PF@6R&CW_h;v6(6Z*FX2^&?YaQsOfG$Wi{DYwy63# zcqa%#5w9h}PZRUs_e+dBst^49()wZyG*r3LMTv?lIw;)nu3KP*h80&5htyoT^!yHU zLf~{)8aS06DUFygrpFMG%m~BipYA1U)xhik7L{F`uAqJdOX)BJTLVC~D5{T^8sP@U zp2>)L=}BH<#3&p9Ih1mr5=d4YYkr73eB1$ElZtna8&J2T!{j`elFJ=6BMHlJ-kG0N zCVsw#KFyq*mn~bhggpK!QF)TlcmD~8)IZ^uztpHI(6Vzgr3Bq-Ba_7|yBjHZUTH#w6BDT@ja&Ea0ITxCX=*x+w z=daYIZ3!|6c;EF}_nv4jv2D3h4UEX4j-?&>b5sqE0^Emkx-8Wyj)DsL&<|)l*XFZ* zxvz}y`)%sDYqB%xMNuFy;`dx*Z$#37#KES*+~WgLUdsoXQv~+-9!RP5eP~*Q=TZ|% zsJWfQ(=pLFn9p;5hHK@3J?DXy6>9kNb2a8y(;vs(+>v!tMrh$#vfdOME4f-}S*~S6 z@AcEUd-(7B71qZm&c7_h$_}f^CRD~#@iqqQ+EQev!Emu}Xjbo3E zW(T?hu`E4D#f@Gb`P|-GPEFs>_NY}yAlhiMCe2y)&6cawgqeco{&{+l!GI$SkzSDN zuxJl_(&i403D?D^t86ZW@=l76*I-#W6%BD@T1$VHm?PL3|2jvVWv7|dZ zG|JDQ5sa=qN)+T>EdfLbp@4O=&q}rmBU)`aA`;&R4)QR#*%1*Y z3k{q_eH}7U+VhWne}cqvppo ze1hCyRwlUIBnL&zFa-kM>|SCSctP~C{8XPZF=1Kmq~uuE z&;l8KT^1z)W^@i;x{uGEO3_fmWyn$|dvki9=<(I6+iGk)P`~ zTxj}PrhLuSnk8M?I6n->w#Q*-mbevhym1LQV+>UN<`j6IQQ2nEWE1@t@?21y$&6Ui zwo?=^-G~fWsXcq|*Wz5Xx42a;i;05*hcr!=TPTV}#V(wB z?z4+pUX{h?IVO-gIDGb4M(xzYHf#F5%)h<0ZmVzJE)$lwR>8Y))7UpmDc9|GnX#*x z`bT_(Q zwo4oQ>(bWD$_eASB%6!VH4!;l*YafMoGc=%$ML{xOg^s^FtVTGM zcJQ$KhiG^6_Kx&r7X@4`+R{ZkK*8PK(ido#WL&akU3|H8%as+TbwV1(7yqz6ar4Su zjfASucFeM<9UJ- z!J#JYYsZ}4+!cNeqcZo>d^rmeREnx!m~LNXu($U#xj^-gyc0J&iT+McBAZJ-69i~F z#WQm4DWrrpXR|{thFYTR2R0kFs(x}z2&3k92DnYgmmmMT*>kzO;TS&AryH+hI5z8V z8~=B1!}Ou>d4Y1psZ4vw-OZ0EgRgUz-V&AeK4%mNYIkBq(|j-0R5hh)?MR>hL$r zXxKMQx7C`oYEsYm_kUTkQe7tk*Y_CE?shF6xN2B;v_!yXb#}EJF39uV0Qt8+P7$k% zeP}=nO*LJ7RQ&g?KhthhJDWL|Uzml?-1~_ANjmuvT@+BjQ_CGZ&Nxw2FfbpJSFIwZ zzaRx4_~YX(4`%VNyZZeJ2V#`2`X$ZSkmnXi3ttGdNSzm_0AlrHNozT(AnG`zqb6L)@$0AKnCp>4SX#(yj*x}B z9Vh(u%I)pU}{10n{eQs=TVvR$31vYeo<}T7$Ga9<6 ziXn_5%_m4t&@2B~^5#q5g`BlzNclQ+33~6T$0-kOLY(v9=(E5I2Yo)(OG;SU3-lV={7ox=J~`ZsU|#<`&9bH*@9*c#Yi}KR12~DldV`;* z7?T5G#>^AaZ`Kq4yBV+!3rp#I%@R)dvGTv4O}%4zGs2t+kHY*6lHlKR*y-8aw?j-v-z9&qjjj zCSFR{=pHVobAVe={_V`z_eS-3bGm*|y7a3%R=;FREQ9*Q_f>es37+R*vn+1FE^rpP z#@JsDF1&(dLD{dC{cnEd?#|!+mX+sk&R>y7^ysf=3wHrJQ$O8s?5cD8(@74nhyVJ7 z;Jf}YjYD-cl#K?zzW2=*su0i5odoY^4Zi!=RaVN(?!5EE7GhUr{Hyo*)dKbsgEc*# z{B{zo6n;5Xm^_Utg9Wk5f$#2Evgo%18@np(xcG}xREEO;=v)2I{3pN3;urv-`>!tR zUr#7hUx)w2ezbnM1u$9FDp2G$IGBYjzxn&Ud{|ubUDql!} zE17wJpmX}m?FS3+i`6RfSmJ;GfhAk{?tcD@@57vu6_$nKVy3RiZ!W|nYTWsUbF=pe z);e}K6S@1IMM$=~Etd}DKdk9%zYu|!=HR7!rQGUWS04nK5u_eNnrh0CfanNtyBy0--5$1}r_t^u zBaS6iit{nb|M ziT<@k{2A4M3gh~zQmDl*-N}aKi|0C;N4jN8mcK1bIW!4eH&D1MXTQm(Q=;oWDVCHT zesFb8Guu2&N6sBBUVnAe zJiIV`YhP%j>_wS>>8> zc4H=Q%(gdWyc@j}b5KE~Aw#37@3W5gn?t-ivi%t&CU^bYFLaDt#ul!?_P4Hg#*4&7 z-w51E^BtQlQ`q_8V!lQvh1Zm3V)*QKEJiiU&_J~+`}U>yZCVy_K0$bUiTBRtop18n z55Az?9#<<{)179ndAFybE8vsZA7AMuzE|#Q**X$#^|Y?%4valiUspYRnpKu}so5m% zpwFTSN=Oy490MAyTR4dtu$3H?oef+wJgg@J?-`D-UzQgD9`F^jpPjBcn$!diUzHJ!rm9%=@x?ULv5eFq-v1*Q% zI}>=jVD?6pF#C%%zV%Y8TB|`qm3}yWfDC#1cxPxqYeQAfN3G$MibALUxYurZ3F@v_ zj)_dR2y&~>P=Ru7P9#`BU|O@Hq_?^WY%w*E^`_eGpe|khKI?j`efdChLv$h$mnN5a zR2-~(aQ9YcRf2Qk^jqt#V%$y^xqVeS*V&%8!&UbgOE@b*qjRmyk-w7Hq-FRWXPsO3 zwPnO6S^Is~FX3P4k6JUkRS(GJda}oF0zyq-W0+v9QI}@5MmSNtA>lv`jstSRx};EI zA>gY}%93VKQ_kvl~p^coS5{0ArW~J(do4Eb!LB0sq17mm5R3Z+Nwp4$e@$;b)Q`HVjlfX==E{FUuQ23^ z@B+F*yGpN8!eQz0IDQ-`GhvOXhTt|Zrst2THUe~99DYr#_?QWlu+&E7gsix#&=^a= zvjB4louMca4^IHd5*kE0fqF~!gk}M=riHc`ph|JUw*}~O$SkAxm?-ma26s7>hE_2wl~NJ;D<4@++X5eA|~+5I}jolSPi)u z9_)Q`fCRrwKDAU(N6}n4jz1pSZA;g}orCffqN zN{G*r^(pbAgk(!XfwOkt+r<2FLxRlm@O1Rk&fxkD&=;Uk z<`NHa17Zz;<9R-<)&ZsjWko1Zc70|f0e{SlG-ou?+vOiN5FKD2$Tdt`G{EW4gpL45 z{3zwUn7=51p0KW*2qo)j2j&fnORu^1^-R;SW^VoN-HX_#F<(_W=WclD5N6uk>Wxwf zzM5v-Bwce*|G=a9w<6bC$#m~>r?V8-_fnEM_XoOJzAF!BnC$Aj(=cs^f3S^ggRE_9 zz&n#i7|ySJ^N+jNqXh2#nR4X1_MLT?FP(}ADBRNAD~P2m(1F!I2&8|p_e)cle{$0O zITx?h2QofszU&#edUMb?WuB;4>Z!-`x@0_O>*d8BGEv7_lFov0bulv-_ns`+RKoL~ zS7X+bw5B!;W{T`f{%HoL{S~H$lWBj7AD$HZ9L);b6VSghv&iGY9>oPuzPpsSiG zZ!u^XntC8cgE*>~7lFipw9vhzWF}J=tO5`ISsz>|Y(5iogJAv=LeD@!!HjTP)C%6$ ztU!;*vEzizv_KUvdXCpA>v@ft z{pJ!W%pitx0n~R)I1TY^nBYy@G7Ej6t1xwiC6rj7n)ytFycW#vccoLl1l8Bd7hwai ztwsTQYG(RcAc<7zVW4TfLzfCf+aaFR*aRQ@J|BB;#&tz z`R*7Hf*1r5$a;PEE^t`PVB*EDLV<{lFW?CO0<8*DQ?SiIM}b-wS(*+8olN-qpIY!k z8^|=a^R8+~5LR@R4#&d=w<*TUs<0^8Gd=b)3w#y#6JG8((G_u$yNJjxWJGY;uK3UKE zo%Qhk?wdTa-a9nwC3}lHy;^Z{`vwUgR#N|&-TPuxYI0)QkL9M(&So}LT(7ur&dzuX z`osI*b?YTZ*>0tCR5R6e*?m`P8l&=gk*CatrIGML?J6|}(Sdod?@bYhv8|COT%{6t zFBZ6$-qFP5mJKOpn$w^q`b)+iqBgou1U5Waxle}12orE*TH;?2jbUv*mEmyj=h1+vco%)rd(HJR6Z)q)_*+L_#x!IgaVE0lISPx^js82xp>1q;k#4gLd1$&r~!jRtaX}xSx*iXCCn-}>C*cz8O7%Oii$S6^(o+!j((4bj) zi{UCjJX`?L9kwp5zNTqs`K9KC22%&U^7~jZPJsT`>J1KMCYX%s}w^yUYD{h5iYTJ z^eiF#6RO(f%O8J(?O45J(>O_%=xaoJI)3Qi-+r)39il0SL;e-j4878onx-V2rbHG1 zX$?$)g3pvH+0#Un|5^E9$xF{7kfFYk17C(}0`AXkLzuZsf;!U7MTG~1%=B0XsQmK7 zy{`{n&rrUPBkOu(4zc+19VDmq@;TKO-n#efzGw0r#RXZaN9#Z7)mnuxqu1kH9-%(j zX9sucef+g-TidJvhX);zhebYSuGn_L@9Xl?YgJt=^=);5`z8&(QnK8gw4}RuWO{Ej zYifS^&9ec)UY>$89?o8$OIsH2@6ni&rzTcG>sr=(1CuQ0CuQmT4ox|#5vx>j=K6C( z=I0AfkH)nv-zZ+W+*9H2wU1`|S>#rJ46&}T8F^{thkgF6^CO1R`y{^gd1WeP|Mk@K zVY@#Q`HbEcA6J#J+PAZEuk41lGod9RL6DN%50Hw-27!DPqoqKs8OQR+cYr)3LJdSE z;hB&~K(P)^P#94dJ7G0x6KtvkDaLHm6|}WgMYi0v8J^ZfycznoB1B#oGGF1yGFF~1 zfDX>UCLp>05Skb>gX{{AEaSt3Qi;UCw&ZX7kp4|P*>B^2Qss@k!$VQWh{;?IsPmqa zp@5t74QFeD3+O#$+POih}`iIr36GhLPANzWI*STDH6 z^WO7i6P7Fjm_R}Bk+=_{4aoV-5csJu5RskOJ1DW579YcYQpW)k>_3@`II4@3-Y0U< zj3`8|X2DCk6R}x3c4K`8ahz<%l{8N&=g}ck!nR(;B(+36!Z1>HKXA;(Jt@drB}@{8 zqgtSsKwChiPnlVDrT6sTI^B_w8N#mOk=%e3O_wRss%Z>`PA>&?5PNIviVh3&Wxtg@ zjZp#M&*5hDlu7dD0*TalXry}MBK-uEA+-ao&HfW)N(v&-3H(oOn$;+pkWk4NxFUqp zDP&|dIG4jG!WbDD2|%dp7(9;d=BWC>Go#dVUJro?iXE}WC`~b%4=Dd#cW^2-p<@#| zFzE+LgTTj_ME2w#6Rb2lZPlxX1g~Vc7PPuSfgTcEX2h$`4XoPvh%I+B@u!;t+{^#j z@T}a6C4fJgNn|LA&hg6N#}yC!luakUV;WUgLOcP&+1VBXNJ6{2CjK6w^&u-kI@kw4 z*)llwHjO6&@c_BkX#I8`%S@xA>W+&%hGv~cZ$2xNE`D)Yt0GVHxM|ZLZAuEPk%h;4 zWK{3h8&U7cx*~4zB>cA2t01{3R$Gkhmt9A9Er^m`Hv8j;0~uQr6T-HbKGVBs6W8M| zeh!1EYx}^6U3;udrj-}^Gv?}~)HSK3?*kWu;jzVZ2`#@4?Y3_nkP)$0x& zKtl4a&S!`e$5A2Dl1Qp3K2YMnH>ekOzoG0C%2#Ls1el}hBILa6!gC(90kV)Ukfm|U zUxaXiR+w@uA)KjJ9wcf}Xtn~CpK-NNxRm@ZUy*eHHM_VRaSW3Eu^f$>#UNdrpSqwf zg8jRMjy7Bi2koKW_=^PTCN+yYo2A-oN`mu{6Jx27prKme^UjAqL4G_6b{N4veIlUp zD26=f82lO!kQHIdB0c*j2sw&ZZ@jTMJJXm)*g=&!&1xY)hmyOD~CvV2Q@H#gG~L{%#S z`i<&cPCVRFC*ZlY+ofcQ9?MV^V)e z>l2mtU-`DBom>!7B_d(FQmuPqyubyUBWrXiz7ShS^&7O?rRmwOU%ij4b=9t$oaR(} zM&CZg%dtR1sb|>yZPLn3J92tA8tX}`HP|G$_X*n4%WI$=1OyFRQy{x93x^wiTCH~^ zB~BzBQ3~0f%0*Oy?@ehuHA9emDv#w|n@5#@&V@1GTG*pV-N!AM23>QFx_g&CP#aXi z3j6!Xgi-15)NazBe1Yb(;}QFih&)yWXN(&7QP{WFRZ9Ait=nf?+eD&`wQa)Bn} z{oMWP0%|i4n8WOa>GVRm{{8Z$AIsHNFZb-5KPi0uOZjgG^`(qL^%ga;g{ECirb?tYP1p;y&esA5vEL(iw{t^SQEkLGGtzuBe|_VBrv zW4p)n>4F+!6|ZVpr%i;00<0b1ec6zgm|4c{aSm@f3yN;$z|6={OjLGiRh{CI`{*~=ZM&7<#{`10-Sr{kP zwK%4S^|FZ!!MtAj@4aI)o1`E-mk!c{ewUZ1Xn{N)`WjEi&2M-}0EwEC;x>Z-lKB?c z>MPR#_!+7Iso-hVk&BPz%S;tU&IhQnCxrlUy^wEas07GAM4*fGI~0%aNldnqp8>oT zH9=fekWW^kb2e}V6i7%9HF;1i7Vg7Y7eME+0A0UWebOn5yF>lq`u2235aZVTQ~%dy zq8(Wh1aE>0$SEEZ4$5&JizbbA02(nQ{826VAAZW18zk%QBxd235AB(7t${v+$HeNP zxhP;(6l8iCi)cQ^ZGOA>1*%TFt~QaH)m423DuHuTe;^j7{(&1xADsrbX>q^HH) z)DL&B-(PLVOga7HY{RlNnX3yQS>c?EpR7fOHg!(=PJPag%Y249$Fev7e6S5u=pB99 zy-j`0_!9ob3g7ns5pAO2AM{j2!(vFce_O4P-@L7NtCpP9NiVH_mGaOK0)d)i z%eNb>ysVsg%EGNWr7-K|6^q#~?QV!^@Xt%J4;%dAztyH~lI-b=_3NH^crCc`%LS!* zsUxMc7fQQJZ;sw`>SWdW?%M@ldXxX`kLnE7?vJKlzy7i{Uvf)Pg4+%Io^L;Io0;NG zqk%($EIC`)l(ZJdAb3e6Wg92URQ-bTCX*FjQ#YSG=?>(Dy$t(W7rK@9=hey6jZBqNML58Q# z{TigTRq8q~##ZFb1Kxt*L4b#l*pcmo|Jg2|0p%ux=a72|urtmkgPdQ; z3Y&668Be2zaF1$|wcg$I+eh11D(oM;ea-nz{S0sAPqclzHGJsb4i`>qDQ7P7(+Ols zPhVv_#Zf>F2d<_WwiG~c+T>Sva`Nh%tx1))n@v_l-YR~o9-96TXNej!lx33?oTFV& z9uu@JJB|<{*Urx2fs(4A>wZsTxL7Yif~6w>dt43*U}qqyxL?$>Vhv_MzOM49Uo9ZD z5hWIi+NzY~Tgc-WS+Y0fRZAbBKR};wy~Y5{ZQolM18sD(&*8mZ({e_fA+%;SSP!)N zuG#I-K(g4+e=zLt1C&UfY2OpI1%ttyChQJoseV#E<(SD51J!Le4ZR0S%PIzE)=GI ztOKSx^3UPhfet2pGl=R0*%VW@RalTzULPoFMN&Q*>B z@FSln(-;p=kHr#nlxxJ7X-?Bs5rwZN_W>NuYNU)fTGYnxEj$PW%Ugp>Z73rT=sll5 zIz{3kOdlBq{b3{I)@G!cqCr$UgsBBfW$>WHD!qnenlgKj_e##1QahZ`hs+cGV&u$b zkehJ&i^PzO!vb*?28!ab&0rQu;0p@Y0%nQSacZWM)d5>VwDIB0x2*+J)E&0pjp<&H z32H*|<4&)C#dF?!&>Y~i5iY`|A?)ISjXm*CLk=KGmKmE#A zcF~>Hif6gTPNr5)QMXe+9JNy2p!cuOwL8B_&r*MU_GeHJ0>9={3fJ6|0^f(F?Oe}8zr2e=8UsMf>si5v03U`qduai|Pey3@~ozt})01%EbdSMIogFKY@7_e35vA$_XUyf(?mm5Oo2sI;T1Ti|mZXT%a*~LbDa= zBmp$bN8SHb7%03*LR^S0jtLIVNBq=-VYmoa^-$@u!65sX+46wwGZe{hYSDB-I)qSe zA=!)!5yGSeIG}u4qB+k7g8`t!ry>AI4H`&LYJuD!+Z#f6Lb>AIXN$p12s^4rXSbf% zt^ygSMY1!UMM#*TAAx2*E<>1SYhCe5tNG5-&kxp%7iR4E|MH2pc<5JlDLQIjK9?(! zbzQ9~Ne;^TfF4Ytj>ZNfmAkh%X=U;;2K|4`gS6|mmz73hk8hgongCW^JJJtuk)IVH zH5zhKM}71bBR}Pb<`~Fe-Nh>G(Qdi8$N<*_Iu+)UKz<_zWUV}aHFXl+Bs0PyM62gA z!G7++%EEd@5@IKR={GNO?>RJUI&F zBeF59A;^n;*DHx-0SE%RIV2g=ueV}Y!;MozHF~n+bF|7R&6UBn$Nmz`haxH?G?k#& z0)ChJNB2}p0YE4A!0J8?PAsRKB1=m8AY)K>jcrnHGt$F&^Yq8@w*n(3IB&000Thm)VVC6d+ja8x`{}$R5zdrUztFKpi-44M``=K_AD` z_J-2ReLl?)rt1AI1i>)SL9J6-1s>*Kf?ITjlG&KQL=L9|hKNdHCw+}qUW#@XQn*N= z-v!tQy0_TZF=bGV=flza3&xoAcvuojDQD7B%oga-oV>=p;#hxTpHFAgNwr_Exh~ca zva4*2i@%}hc=*9AC(&ugXD>C(6UbA4+_XN~BRRP3aEE`iaDPVE!t4#AoxiE%r=5~1 z^*r6~B#^Y_@}0Eq7qOL}4P+%(nXaz7IYs^P*J>WuPm+?hcRrmTSbOcV!o!?7^R6E_ zxYSX2?ML(cuy40xO!W?Q4{z)Ht5fGh+n}YbWzMzBEprxneAxRUFF#RhalLbkHt$B8 z#%1w`bI^7Dw0>1r;*)Ce1LXsr145^Ut=_c_8Aq7AG4w z(aqCuM?IJMbKP=zai~LXZp*j+TOH%G9%sy6n6~EPfNR3QH=6eQ;ytFQR zi#hoHT7c*Q-%knP)mS;I%VnWk5sEpVHw5@ZR}-5Qd(`7AgIXSSAP}dfR6>dK6mXhw zW>7$GI2M^atv_^7$MdIt>+36h>ZV;kb9US6_SDp)@lHE*w_JGl+nzm53%|5kwOX$j zc@xvEyI-<*HZa-Z)Wao?20j}SexI7~R}D{yiRSVq`Ao?3c=Tk_Zns1{8xcEJ|GC7j zc*newb6@``j9loT`F@^&u7iKWHv6tpi~GeRg049u&o9VsUh{EF_(_Y!!taIl{uJ4F zLugvRjS=f@rg3NS*$uhO+V~x`&fC!y%(SvQ^9qycb?0tA?%nK>eMF+O;$*v;^^NLW z`?$)DvL62J+6(vD?!Nbadyw$-^D;co=QfNi7`~*oylw__Q&F$DbyUgzCFi$FWP&k?aL}g3&PK6UtmO0$N6LTohS+`@MP z$5{`oItF*=_OXtAwd|qoZ}(yS>R~5U8+XBG`&0Q|o_zBw$-hgtw1bEflBMUxygDnB zwfnJ{L>#?a&~DZV44DD0S{2Yk{PjE{m?7^%2L-xoi%~wOjTTf3L#h{CoeyEH4bTsU zVQ-rxErqVqP5A;=Z&J>Oa{%MdgI+-+NH|1Du|G-J)C1}#P?pesl!N%dl(+~$h3uxc zasR}=j#jMDvh{7x+z5-H*y zsC5O@Ok!R4zD{;hefO)F>GfrIM!Jls?MI_xBWaG<9x7Duk&wO%nN4sojyUz4| z=b0_dQ`XO|T{E<+>+6%V3-!Y5G`w>z)H1eKy)bIsthKV?j>xPyli9J~3~~%_^Tn>v zpIq_GO}adjk$HUJ=cfEyX}%U^X3TE2m$ddb?qSPB+>a>j@~gYo94*ja2(8nC*6x18 zrcGr-N;l&*Y>mXN-hD3JJX!zC;-~BPtj+^7ameAb$m8VjM`}1~qfX$x-my5S{eI z(JIi;hi~3T7#Gq7%!LIus!0e9N#FJ4oRsjow1{`GM3j`36~ncK2MkFoA=0UdCoJ^c zNd8AvNs1X{Y>vpUaBY&WlR{Cv#i1Ef*dn{wkT_B^nga+cdZPMSN|+h3idi8nO3CH8 z703Q|jKBGaJtRNPcge~MCU;t(Ug(K}ENJ>%6#(}gBmpJJ?^;=0t z{oQ-D*9SjH=fnu5Hvf4k!VHHy9vqIdbDG!jLHOL%+P4npq!6+Hh4HNp@C-u_)x8R3n-~Ou6f4L|xM^`y(y_nwdhxT1_YOSAs z?Kg6%`C$D^mu|qF9W5?fov-ZP(7skCYjyhI)vF@*Gx8`K{-8`dP=aq>4;nWMDfBO| zS#V^5=jx(|3l3+sf4=m4`hL~$wmGUsQZ)a6-I+^E=glxe-4dN1Tei;S(A3q@Og|0L zUB3rG)5MS>>>EVvoG?%5eNz$&{Nd-xFVQ(p7<&K@5n5O!4j4=l38)F7r*b8W#|C<# z9GW}@Y8lyNLsX=by~QCMA*BgQN^l{B*rT(7-@T7yh7*UjCAnrk-G+|4A@!QTIaXGd z)$cMZFPAigyx4N|>*|1CR&1YD)Rue~JdIEAwkbQ_3yI$?S9r9xU;5b*iS#0;m>Jm~ z_6ngh_P%0-YH_cgiiau*eDD-t@dX4AQI$KC!JwS}~P)1<;vo^zL+PxyV> zL4%HiQ*0?&5urz$@8zm$Nr&#rJ=@zc`$*B<2-&A+*F90m@v%{A%P7cjxK^2(vbC@8 zvv|$fcf-F0ym{w;h)0^gZE#&=N$D1dSXXPkH}){gmoMD6!`lqv4iszH0GJ#Q*OidN z;6bX2c_mOKpjc#wWX>tbgxCSygA5dyOej(7@CLX8p~1ScVOA3KA%uhbR}{u_lj#V? zq|(ijO%jpm1qZHNQ^a=%<_HZJYijHf7Ekf(wogmFL|x44)p03ROJYKG!VJ- z)6Bx|z%WN$htQ+DJF=^3u0B7PsnNQmv$oE$TepGF<(P+@Tl&3Q_fBamo=;S5FkN%K ztYKhV3A6-Sd2yA_d=0PD%SYA^S#+LHoqY5fo3*`P~#!EYT|~iayc3ovrl%*vk)+kTRj6It`6?Sl(t>h=7$*UR1UMq+M3*{=S{+74*-NJ^ z_MR@0LNx6rViv}(xFfWqd%-bT3jKjc3t;NGlcG}bfW5o398Y2@A;;WN_68f*1qHaq zVb~iUBoPDWB1wT%3Ic;jFt4`DKe<>9y1>i_gFUEqzz6-zKG_PAmi}2l4w|Hjq3Oz6 ziZLWQ;KiYNhDHlo+NpAM0*Qd*5P%jtB@ByyOg~#S*Z?9<@>#?aVag>Tp)Qm^SaY(F zrJ#XCSz>^Eq8hspPv3xAJc)J}kERD(6~khLr>@IT6{bod!Zb#LjtHOVpbs%B4q?WF zSe!s*O+vqIu5cBD0$S!GT)Vjg?@2DJ5X#bp`6~r=xbo`R7*;_436o?^3jrL z=nAmN6)wf;J6P#ml0>kM0cRA1YsFY|&1puJ7ptQDDBCtNr3JEA0)2t_mdlFez4O`0 zOV+|IuGD_d0}n_3%M0dhJl@1}EY!cIw?|2`uRJh#BnH46%VNmx>7&WF4(`i-6+hGF zxX-(VS7L?qI*$-QYfK0NEjROs&;-1jt)R8lp<}pQF`9TFMuWtw#8DqpNRyaWb6?Yk z6*q~lj8n0E|MsWrQ?^U(*?w!kEQP7oEvvMrUT5yx@2qt{Enu&Tz2=*tY-;C7RneE2 zw&nJNLoxM6!x@*FMmBD({`yTL=kLthhwZ<0qNgp>U_?0PO~eLfpNK|B3F`!PAj;+E z!lB~_QQ+(j9kOMmrDzOH*|~4zfoy;Iw!W2t->D6&Sk>`}O4^qh-$ZWtj10^6n@&#d z>+=m_r6gr`bhFMX;)m~))*N@R>MP;fc)Q z0h7+y85(VELee8kx4r?)`+S66pUhBC9lbq`H}}^K_3uOzBY7x|+jP&S&oensNWI1i zW;RP7#2e+T#R^uQ?z@ri6c)pzZ{=Am#^^`QE;mF z;X5VzVkCR~S2q3_Stk4eaAB$oxK1WYh9Jaqf`)nf*X($Nx@JqpQ|FaG)WkR#UY4-N5`|{)24O zl6+SwIri`0zEfDgx*WD$Siq!@#OxV+v_$S(3~L=&kUO-AW1RcX&e28Nr~upC>CF)v z^adu4T8VvI+q98m90Lp;^w7pW;QY<+ti3;8uYBT!oAp)-$hh>|K6B1{qOUzH-&Kx3)S`+MwZQ%hqi^pemxWBUVhb#_NEzprWVtsSm@n=d zgb_RHu22{?D?6A~v(4DX3gmjmzgf5Ghn@~kW-lNP6WH7Q7QLTicgT8IDZw1K4mbie zkZgvd<^fl57S=!dqC*Gg=oK7!O5eeV4dqDKHaVb_Bi#!qdZOkb==e%NjVInp#u=AF#{!1&Nc9ijXD2sd z-QfDM%SU%B{_Me}&ar(x(UxD~!2*ZU$Gq#i045A#Oj!Gzl^iDGF(`godSrZ;Y1X29{YE&oc?9KF2n zl;)TZN`jTJibn0Q1dic{Z@*Fand>bRFEn=7rju`1q^$5izzNh7c55d0o8$IK{|Kb< zDt*4e7zi-xTUitAWTGJ>a0p3@avmif=0CpuZe()bplsjL0R8N-V9IJ6p8jKl6>~o& zqTPH4NqR!U)EtfOS$A`XolUw% z{lb!ifdijo>d7Y180trzfqX(<@N`_m-X}kHn}pU#$>)8=Z>P0 zO2gAx<8E~{{ox7#g+s18>qqX9Z17q|%9NlcRavwTiUe-B(VjIP?L*1po#7<9#;}$FFOK^% zog6da9xalaZ6v$aa4b{lJbg3#y%fjROOaDcHjgf^kaV^82otF=lJAPp&rg!8sRH!;K)bw889qxrQ`k|#*AEe z(WtYtp5W{_@n*UtNk&PSx@mU1KGB?if;nq+j|Qf(8y#Ymvad(7!XGxJRWTv+ zH7lk&GW$5L2>D%r$-K2=Wf3!{B?w8+J-haCf~R~nBAv;&Gowl;+@qn>@br4rx18EJ=QK*+<7}&sAQ|>BFUQ5w zU?NYV`}ZM-e4E+1tol23w&K@XuZ3CGf;E{j`maSle>nA3`lZ;7O*`+Nm~~aHUAZvL zur-rA?PyL(S~F$dc7;V}D=uv>o7Iowhdq^SmCpOx+OMztlyFKtZS@`v%PNe7B*)Q^ z5&2MK#3ji?nSoA~bja^Ij0$X30S|-m2a0Rl;{wbbQwi{C0qG3V#^#%`gaT3PZ_3nn zC!Yhyf!<8Q5wZIlY16^%XNxzZSAmhG)|4Q8#d~%DChJ=0Ibg2-qQSCy%_&9(hLFh= zFyhhsCIVtfDw=7iQUqvv6(}5ZC=vLtx6X%*KxDQV*9%XzcOZR=wtAKp<=%5zbz~jT zi%;?wX(I_Z# z1v)m>;`eA(Kjv0wNOTy4hq!{)uGxOtU$5nO&psa#hR*)TVmv9nD%emERR%gk)uE- z?DdJzE+d^(WS}^Ef+m3!6{z)l|H5qY4!r1Tx>K6e<%n8`iOCe0aY`A=3sohGQihg> wcw0-EkU`pz6nr3$VttWMK&qIFQmE%H|Eh=*+kO3*Hw%x&QzG literal 0 HcmV?d00001 diff --git a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md index 0f41dc4b..ba1add59 100644 --- a/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md +++ b/docs/zh/tutorial/tutorial_grpo_mcore_qwen3_next.md @@ -98,5 +98,14 @@ cd ${CHATLEARN_ROOT} bash scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh ``` +在解决了一些训练不稳定的问题后，验证集升的评估指标仍然有提升。 +

+ + ChatLearn + +

+ + + ## 使用 Wandb 监控 如需使用 Wandb 记录训练过程，请参考其他最佳实践进行修改。 diff --git a/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh index 95f4ef1f..16bb12a5 100644 --- a/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh +++ b/scripts/mcore_sglang/train_mcore_sglang_qwen3_next_grpo.sh @@ -32,39 +32,42 @@ python chatlearn/entrypoint.py grpo --config-file template/grpo_megatron.yaml \ runtime_args.data_path=${CHATLEARN}/dataset/MATH-lighteval/train.json \ runtime_args.eval_data_path=${CHATLEARN}/dataset/MATH-lighteval/test.json \ runtime_args.output_dir=${CHATLEARN}/output/${exp_name} \ - runtime_args.num_episode=50 \ - runtime_args.sample_per_episode=64 \ - runtime_args.train_global_batch_size=64 \ + runtime_args.num_episode=200 \ + runtime_args.sample_per_episode=512 \ + runtime_args.train_global_batch_size=512 \ runtime_args.train_micro_batch_size=1 \ runtime_args.save_episode_interval=1000000 \ runtime_args.log_args_dict.enable_tensorboard=True \ runtime_args.log_args_dict.tensorboard_dir=${output_dir}/tensorboard \ - runtime_args.eval_episode_interval=1 \ + runtime_args.eval_episode_interval=5 \ runtime_args.enable_eval_before_training=False \ models.policy_trainer.model_provider_module=qwen3_next.pretrain_qwen3_next \ models.policy_trainer.num_gpu=${num_device} \ models.policy_trainer.packing=False \ - models.policy_trainer.max_token_in_packing=4096 \ + models.policy_trainer.trust_remote_code=True \ + models.policy_trainer.max_token_in_packing=3072 \ models.policy_trainer.bf16=True \ models.policy_trainer.sequence_parallel=True \ models.policy_trainer.use_distributed_optimizer=True \ - models.policy_trainer.recompute_granularity=null \ models.policy_trainer.tensor_model_parallel_size=2 \ - models.policy_trainer.pipeline_model_parallel_size=2 \ - models.policy_trainer.expert_tensor_parallel_size=1 \ + models.policy_trainer.pipeline_model_parallel_size=8 \ + models.policy_trainer.expert_tensor_parallel_size=2 \ models.policy_trainer.expert_model_parallel_size=2 \ - models.policy_trainer.generation_batch_size=32 \ + models.policy_trainer.context_parallel_size=1 \ + models.policy_trainer.generation_batch_size=128 \ models.policy_trainer.load=${mcore_ckpt_path} \ models.policy_trainer.optimizer.lr=2e-6 \ models.policy_trainer.optimizer.min_lr=2e-6 \ models.policy_trainer.pos_clip_ratio=0.2 \ models.policy_trainer.neg_clip_ratio=0.2 \ - models.reward.generation_batch_size=8 \ + models.reward.generation_batch_size=64 \ models.policy.load=${hf_ckpt_path} \ - models.policy.generation_batch_size=16 \ - models.policy.tensor_model_parallel_size=2 \ + models.policy.enforce_eager=False \ + models.policy.is_sync_mode=False \ + models.policy.generation_batch_size=64 \ + models.policy.tensor_model_parallel_size=4 \ models.policy.max_prompt_tokens_length=1024 \ - models.policy.max_response_tokens_length=2048 \ + models.policy.max_response_tokens_length=8192 \ models.policy.num_inference_per_prompt=32 \ models.policy.gpu_memory_utilization=0.75 \ models.policy.enable_thinking=False \ From d12316fa475288132d6327a18f07c34593c3a63c Mon Sep 17 00:00:00 2001 From: lostkevin Date: Thu, 23 Oct 2025 14:24:09 +0800 Subject: [PATCH 28/28] fix pylint --- chatlearn/configs/megatron_config.py | 4 +-- chatlearn/models/megatron_module.py | 1 - chatlearn/models/sglang_module.py | 2 +- .../mappers/base_megatron_mapper.py | 30 ++++++++++++------- .../synchronizer/mappers/mapping_helpers.py | 10 +++---- .../mappers/megatron_llm_mapper.py | 21 ++++++------- .../mappers/megatron_vlm_mapper.py | 4 --- .../synchronizer/planners/tensor_planner.py | 14 ++++----- .../utils/mappings/sharded_tensor_info.py | 18 +++++------ chatlearn/utils/megatron_utils.py | 6 ++-- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/chatlearn/configs/megatron_config.py b/chatlearn/configs/megatron_config.py index c2fc27d5..11c57aac 100644 --- a/chatlearn/configs/megatron_config.py +++ b/chatlearn/configs/megatron_config.py @@ -250,7 +250,7 @@ class MegatronModelArchitectureConfig(BaseConfig): hybrid_override_pattern: Optional[str] = None is_hybrid_model: bool = False apply_layernorm_1p: bool = False - + def _post_init_impl(self): if self.moe_aux_loss_coeff == 0: self.moe_router_load_balancing_type = 'none' @@ -340,7 +340,7 @@ class MegatronConfig(BaseConfig): avoid big reseverd memory in ref and policy trainer worker, expandable_segments should be False \ while in parameter sync for efficiency"} ) - + def _validate_impl(self): assert self.num_gpu > 0, "Megatron-Core requires at least one GPU" assert self.num_gpu % self.num_replica == 0, \ diff --git a/chatlearn/models/megatron_module.py b/chatlearn/models/megatron_module.py index 38394889..2854ebdb 100644 --- a/chatlearn/models/megatron_module.py +++ b/chatlearn/models/megatron_module.py @@ -16,7 +16,6 @@ import re from dataclasses import fields -import inspect import torch try: diff --git a/chatlearn/models/sglang_module.py b/chatlearn/models/sglang_module.py index b4093690..37ef4895 100644 --- a/chatlearn/models/sglang_module.py +++ b/chatlearn/models/sglang_module.py @@ -32,7 +32,7 @@ from torch.distributed.device_mesh import init_device_mesh from transformers import AutoTokenizer, AutoModelForImageTextToText, AutoModelForCausalLM, AutoConfig, AutoProcessor -from chatlearn.runtime.decorator import timeit, compute_decorator, monitor_error +from chatlearn.runtime.decorator import timeit, compute_decorator from chatlearn.utils.utils import get_full_proc_memory_info from chatlearn.utils.mappings import ShardedTensorInfo from chatlearn.utils.mappings.huggingface_helpers import build_sharded_info_for_huggingface_model diff --git a/chatlearn/synchronizer/mappers/base_megatron_mapper.py b/chatlearn/synchronizer/mappers/base_megatron_mapper.py index c945ffe9..40d00a45 100644 --- a/chatlearn/synchronizer/mappers/base_megatron_mapper.py +++ b/chatlearn/synchronizer/mappers/base_megatron_mapper.py @@ -167,7 +167,15 @@ def _inner_map_for_gate_up_proj(self, src_key: str, dst_key: str, proj_type: str self._update_mapping(mapping) return mapping - def _inner_map_for_qkv_proj(self, src_key: str, dst_key: str, proj_type: str, num_attention_heads: int, num_query_groups: int, is_gated_attention: bool=False): + def _inner_map_for_qkv_proj( + self, + src_key: str, + dst_key: str, + proj_type: str, + num_attention_heads: int, + num_query_groups: int, + is_gated_attention: bool=False + ): src_info = self._src_name_to_metadata[src_key] dst_info = self._dst_name_to_metadata[dst_key] mapping = defaultdict(list) @@ -196,13 +204,13 @@ def _inner_map_for_mla_down_proj(self, src_key: str, dst_key: str): return results def _inner_map_for_merged_linear( - self, - src_key: str, - dst_key: str, + self, + src_key: str, + dst_key: str, src_layout: List[Tuple[str, int]], required_layout: List[str], - *, - global_expert_id: int=None, + *, + global_expert_id: int=None, num_experts: int=None, axis: int = 0 ): @@ -229,13 +237,13 @@ def _inner_map_for_merged_linear( return mapping def _inner_map_for_linear_attn( - self, - src_key: str, - dst_key: str, + self, + src_key: str, + dst_key: str, src_layout: List[Tuple[str, int]], required_layout: List[str], - *, - global_expert_id: int=None, + *, + global_expert_id: int=None, num_experts: int=None, axis: int = 0, n_groups: int = 1 diff --git a/chatlearn/synchronizer/mappers/mapping_helpers.py b/chatlearn/synchronizer/mappers/mapping_helpers.py index bd664c32..b0ba385a 100644 --- a/chatlearn/synchronizer/mappers/mapping_helpers.py +++ b/chatlearn/synchronizer/mappers/mapping_helpers.py @@ -115,7 +115,7 @@ def process_merged_linear_tensor( raise ValueError(f"Expect all keys of the required layout is the subset of source layout {src_names}, but {required_layout}") mcore_layout = slice_data_list_by_index(_build_merged_linear_layout( - src_layout, + src_layout, n_chunks, src_tp_size ), (src_tp_rank, src_tp_size)) @@ -134,13 +134,13 @@ def process_merged_linear_tensor( ) vllm_layout = _build_merged_linear_layout( - [(name, keyname_to_size[name]) for name in required_layout], + [(name, keyname_to_size[name]) for name in required_layout], n_chunks, dst_tp_size ) results = [] for (name, chunk_id, _), dst_part in zip( - vllm_layout, + vllm_layout, full_dst_info.chunk(sections=[item[2] for item in vllm_layout], axis=axis) ): if (name, chunk_id) not in id_to_frags: @@ -213,7 +213,7 @@ def _build_qkv_layout( if dst_tp_size < num_query_group: vllm_layout = flatten([ flatten([ - (f"q{r_id * vllm_nq + q_id}", f"g{r_id * vllm_nq + q_id}") if is_gated_attention else (f"q{r_id * vllm_nq + q_id}", ) + (f"q{r_id * vllm_nq + q_id}", f"g{r_id * vllm_nq + q_id}") if is_gated_attention else (f"q{r_id * vllm_nq + q_id}", ) for q_id in range(num_heads // dst_tp_size) ]) + [f"k{g_id + r_id * (num_query_group // dst_tp_size)}" for g_id in range(num_query_group // dst_tp_size)] + @@ -223,7 +223,7 @@ def _build_qkv_layout( else: vllm_layout = flatten([ flatten([ - (f"q{r_id * vllm_nq + q_id}", f"g{r_id * vllm_nq + q_id}" if is_gated_attention else (f"q{r_id * vllm_nq + q_id}",)) + (f"q{r_id * vllm_nq + q_id}", f"g{r_id * vllm_nq + q_id}" if is_gated_attention else (f"q{r_id * vllm_nq + q_id}",)) for q_id in range(num_heads // dst_tp_size) ]) + [f"k{r_id * num_query_group // dst_tp_size}", f"v{r_id * num_query_group // dst_tp_size}"] diff --git a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py index 495fc2bb..475dcad3 100644 --- a/chatlearn/synchronizer/mappers/megatron_llm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_llm_mapper.py @@ -16,7 +16,6 @@ """Mapper for Megatron to vLLM""" from typing import TYPE_CHECKING, Union, Dict -import inspect from torch import nn from transformers import AutoConfig @@ -45,8 +44,6 @@ ) from .base_megatron_mapper import BaseMegatronMapper -from chatlearn.utils.logger import logger - if TYPE_CHECKING: from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding from megatron.core.transformer.transformer_layer import TransformerLayer @@ -207,7 +204,7 @@ def _map_transformer_layer(self, module: 'TransformerLayer', cfg: DecoderLayerKe submodule_config = module.submodules_config has_self_attention = submodule_config.self_attention is not IdentityOp has_mlp = submodule_config.mlp is not IdentityOp - assert has_self_attention or has_mlp, f"The TransformerLayer should at least contains one of self_attn or mlp!" + assert has_self_attention or has_mlp, "The TransformerLayer should at least contains one of self_attn or mlp!" if has_self_attention: if module.config.multi_latent_attention: @@ -286,11 +283,11 @@ def _map_mamba_mixer(self, module, src_prefix='', dst_prefix=''): # in_proj src_layout = [ - ('z', Dv * Nv), - ('v', Dv * Nv), - ('q', Dk * Nk), - ('k', Dk * Nk), - ('b', Nv), + ('z', Dv * Nv), + ('v', Dv * Nv), + ('q', Dk * Nk), + ('k', Dk * Nk), + ('b', Nv), ('a', Nv) ] self._inner_map_for_linear_attn( @@ -309,9 +306,9 @@ def _map_mamba_mixer(self, module, src_prefix='', dst_prefix=''): ) # conv1d src_layout = [ - ('conv_v', Dv * Nv), - ('conv_q', Dk * Nk), - ('conv_k', Dk * Nk), + ('conv_v', Dv * Nv), + ('conv_q', Dk * Nk), + ('conv_k', Dk * Nk), ] self._inner_map_for_merged_linear( f"{src_prefix}conv1d.weight", diff --git a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py index 8945119e..44457c96 100644 --- a/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py +++ b/chatlearn/synchronizer/mappers/megatron_vlm_mapper.py @@ -16,12 +16,8 @@ """Mapper for Megatron to vLLM""" from typing import TYPE_CHECKING, Union -import inspect from torch import nn -from megatron.core import mpu -from megatron.core.transformer.transformer_layer import get_transformer_layer_offset - from chatlearn.configs import PolicyConfig from .mapping_helpers import ( diff --git a/chatlearn/synchronizer/planners/tensor_planner.py b/chatlearn/synchronizer/planners/tensor_planner.py index 08fc62fa..8ad12c70 100644 --- a/chatlearn/synchronizer/planners/tensor_planner.py +++ b/chatlearn/synchronizer/planners/tensor_planner.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Sync parameters""" +import random from copy import deepcopy from collections import defaultdict from typing import Dict, List, Tuple, TYPE_CHECKING @@ -83,17 +84,17 @@ def build_iteration( ) -> List[Dict[int, List[SyncIteration]]]: """Build iterations from unbucketized plan according to the given memory constraints. - + Args: - unbucketized_plan (Dict[int, Dict[Ranks, List[ShardedTensorInfo]]]): + unbucketized_plan (Dict[int, Dict[Ranks, List[ShardedTensorInfo]]]): The unbucketized comm plan. - src_rank_to_gpu_id (Dict[int, int]): map ranks of source model to + src_rank_to_gpu_id (Dict[int, int]): map ranks of source model to physical GPU ID. - dst_rank_to_gpu_id (Dict[int, int]): map ranks of destination model + dst_rank_to_gpu_id (Dict[int, int]): map ranks of destination model to physical GPU ID. - mem_infos (Dict[int, Tuple[int, int]]): The used memory and + mem_infos (Dict[int, Tuple[int, int]]): The used memory and total memory for each physical GPU. - max_memory_fraction (float, optional): The maximum ratio of planner + max_memory_fraction (float, optional): The maximum ratio of planner could use. Defaults to 0.8. Returns: @@ -116,7 +117,6 @@ def build_iteration( is_added.add(dst_param.param_id) dst_param_id_to_src_params[dst_param.param_id].append(src_param) t = list(dst_param_id_to_src_params.keys()) - import random random.shuffle(t) dst_param_id_to_src_params = {k: dst_param_id_to_src_params[k] for k in t} diff --git a/chatlearn/utils/mappings/sharded_tensor_info.py b/chatlearn/utils/mappings/sharded_tensor_info.py index 0f9b4173..6d19cc1e 100644 --- a/chatlearn/utils/mappings/sharded_tensor_info.py +++ b/chatlearn/utils/mappings/sharded_tensor_info.py @@ -155,12 +155,12 @@ def unsqueeze(self, offset:int, length: int, axis: int=0) -> 'ShardedTensorInfo' def index(self, tensor: torch.Tensor) -> torch.Tensor: """Indexing tensor with this ShardedTensorInfo. - will check the shape-related information and ignore + will check the shape-related information and ignore inconsistent datatype. - + Args: tensor (torch.Tensor): tensor to be indexed. - + """ tensor_shape = tensor.shape @@ -248,7 +248,7 @@ def concat(shards: List['ShardedTensorInfo'], axis: int=0) -> Optional['ShardedT Returns: - ShardedTensorInfo: The concatenated shard. If the input list is empty, + ShardedTensorInfo: The concatenated shard. If the input list is empty, returns None. """ if len(shards) == 0: @@ -351,13 +351,13 @@ def chunk(self, sections: List[int], axis: int=0) -> List['ShardedTensorInfo']: offset += section chunks.append(result) return chunks - + @property def offset(self): """Return the offset of this shard in the global tensor""" return tuple(l + g * s // a for l, g, s, a in zip( - self.local_offset, - self.global_offset, - self.global_shape, + self.local_offset, + self.global_offset, + self.global_shape, self.axis_fragmentations - )) \ No newline at end of file + )) diff --git a/chatlearn/utils/megatron_utils.py b/chatlearn/utils/megatron_utils.py index f67f338c..6a72b5ad 100644 --- a/chatlearn/utils/megatron_utils.py +++ b/chatlearn/utils/megatron_utils.py @@ -119,7 +119,7 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): hybrid_pattern = ['*-' if (i + 1) % full_attention_interval == 0 else 'M-' for i in range(hf_transformer_config.num_hidden_layers)] cfg.models.policy_trainer.megatron_model_cfg.hybrid_override_pattern = ''.join(hybrid_pattern) - cfg.models.policy_trainer.megatron_model_cfg.is_hybrid_model = True + cfg.models.policy_trainer.megatron_model_cfg.is_hybrid_model = True cfg.models.policy_trainer.megatron_model_cfg.hidden_size = hf_transformer_config.hidden_size cfg.models.policy_trainer.megatron_model_cfg.num_attention_heads = hf_transformer_config.num_attention_heads @@ -140,7 +140,7 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): cfg.models.policy_trainer.megatron_model_cfg.group_query_attention = True cfg.models.policy_trainer.megatron_model_cfg.num_query_groups = hf_transformer_config.num_key_value_heads - + cfg.models.policy_trainer.megatron_model_cfg.moe_grouped_gemm = True cfg.models.policy_trainer.megatron_model_cfg.moe_token_dispatcher_type = "alltoall" cfg.models.policy_trainer.megatron_model_cfg.moe_router_topk = hf_transformer_config.num_experts_per_tok @@ -161,4 +161,4 @@ def update_qwen3_next_cfg(cfg, hf_transformer_config): cfg.models.policy_trainer.distributed_timeout_minutes = 60 cfg.models.ref_policy.megatron_model_cfg = cfg.models.policy_trainer.megatron_model_cfg - return cfg \ No newline at end of file + return cfg