diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 26f9c8c88fdf..41a95e4b6e2f 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -1390,16 +1390,11 @@ def forward( ) aux_hidden_states = None - if self.capture_aux_hidden_states: - hidden_states, hidden_states_before_norm, aux_hidden_states = self.model( - input_ids, - positions, - forward_batch, - input_embeds, - pp_proxy_tensors=pp_proxy_tensors, - ) - elif self._is_multimodal: - hidden_states, hidden_states_before_norm = general_mm_embed_routine( + # Multimodal embedding must run even when aux hidden states are captured + # (DFLASH/MTP). Taking the capture branch first skipped image/audio + # feature placement and left placeholder tokens as plain text embeddings. + if self._is_multimodal: + model_out = general_mm_embed_routine( input_ids=input_ids, forward_batch=forward_batch, language_model=self.model, @@ -1408,13 +1403,17 @@ def forward( pp_proxy_tensors=pp_proxy_tensors, ) else: - hidden_states, hidden_states_before_norm = self.model( + model_out = self.model( input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors=pp_proxy_tensors, ) + if self.capture_aux_hidden_states: + hidden_states, hidden_states_before_norm, aux_hidden_states = model_out + else: + hidden_states, hidden_states_before_norm = model_out if self.pp_group.is_last_rank: return self.logits_processor( diff --git a/python/sglang/srt/models/mimo_vl.py b/python/sglang/srt/models/mimo_vl.py index 3f4c7267bc20..f09269adc33f 100644 --- a/python/sglang/srt/models/mimo_vl.py +++ b/python/sglang/srt/models/mimo_vl.py @@ -317,6 +317,10 @@ def __init__( prefix=add_prefix("merger", prefix), use_data_parallel=self.use_data_parallel, ) + # MiMo-VL merger ln_q is LayerNorm (see modeling_mimo_v2.py + # MiMoVisionPatchMerger), not the Qwen2.5-VL RMSNorm. Checkpoint ships + # visual.merger.ln_q.weight only (bias omitted → zeros). + self.merger.ln_q = nn.LayerNorm(hidden_size, eps=1e-6, bias=False) self._post_init() def apply_index(self, tensor: torch.Tensor, index: torch.Tensor): @@ -327,7 +331,9 @@ def apply_index(self, tensor: torch.Tensor, index: torch.Tensor): def _post_init(self): for name, param in self.named_parameters(): - if "bias" in name: + # Also zero sinks: they are torch.empty at ctor and some checkpoints + # omit individual visual.blocks.*.attn.sinks keys. + if "bias" in name or name.endswith("sinks"): param.data.zero_() def get_window_index_1d(self, grid_thw, col=True): diff --git a/python/sglang/srt/multimodal/processors/mimo_v2.py b/python/sglang/srt/multimodal/processors/mimo_v2.py index e914f00b1a0e..35a43469fdcf 100644 --- a/python/sglang/srt/multimodal/processors/mimo_v2.py +++ b/python/sglang/srt/multimodal/processors/mimo_v2.py @@ -214,8 +214,12 @@ def __post_init__(self): ) -_QWEN2VL_PIXEL_MEAN = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1) -_QWEN2VL_PIXEL_STD = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1) +# OPENAI_CLIP stats, matching Qwen2VLImageProcessor (preprocessor_config.json +# image_mean/image_std with do_rescale 1/255). Kept in 0-255 space because +# standardize_batch sees un-rescaled PIL tensors. ImageNet values were wrong +# for MiMo-VL and scrambled colors. +_QWEN2VL_PIXEL_MEAN = torch.Tensor([0.48145466, 0.4578275, 0.40821073]).view(-1, 1, 1) * 255.0 +_QWEN2VL_PIXEL_STD = torch.Tensor([0.26862954, 0.26130258, 0.27577711]).view(-1, 1, 1) * 255.0 _mean_std_cache = {}