| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990 |
- diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py
- index 26f9c8c88fdf..41a95e4b6e2f 100644
- --- a/python/sglang/srt/models/mimo_v2.py
- +++ b/python/sglang/srt/models/mimo_v2.py
- @@ -1390,16 +1390,11 @@ def forward(
- )
-
- aux_hidden_states = None
- - if self.capture_aux_hidden_states:
- - hidden_states, hidden_states_before_norm, aux_hidden_states = self.model(
- - input_ids,
- - positions,
- - forward_batch,
- - input_embeds,
- - pp_proxy_tensors=pp_proxy_tensors,
- - )
- - elif self._is_multimodal:
- - hidden_states, hidden_states_before_norm = general_mm_embed_routine(
- + # Multimodal embedding must run even when aux hidden states are captured
- + # (DFLASH/MTP). Taking the capture branch first skipped image/audio
- + # feature placement and left placeholder tokens as plain text embeddings.
- + if self._is_multimodal:
- + model_out = general_mm_embed_routine(
- input_ids=input_ids,
- forward_batch=forward_batch,
- language_model=self.model,
- @@ -1408,13 +1403,17 @@ def forward(
- pp_proxy_tensors=pp_proxy_tensors,
- )
- else:
- - hidden_states, hidden_states_before_norm = self.model(
- + model_out = self.model(
- input_ids,
- positions,
- forward_batch,
- input_embeds,
- pp_proxy_tensors=pp_proxy_tensors,
- )
- + if self.capture_aux_hidden_states:
- + hidden_states, hidden_states_before_norm, aux_hidden_states = model_out
- + else:
- + hidden_states, hidden_states_before_norm = model_out
-
- if self.pp_group.is_last_rank:
- return self.logits_processor(
- diff --git a/python/sglang/srt/models/mimo_vl.py b/python/sglang/srt/models/mimo_vl.py
- index 3f4c7267bc20..f09269adc33f 100644
- --- a/python/sglang/srt/models/mimo_vl.py
- +++ b/python/sglang/srt/models/mimo_vl.py
- @@ -317,6 +317,10 @@ def __init__(
- prefix=add_prefix("merger", prefix),
- use_data_parallel=self.use_data_parallel,
- )
- + # MiMo-VL merger ln_q is LayerNorm (see modeling_mimo_v2.py
- + # MiMoVisionPatchMerger), not the Qwen2.5-VL RMSNorm. Checkpoint ships
- + # visual.merger.ln_q.weight only (bias omitted → zeros).
- + self.merger.ln_q = nn.LayerNorm(hidden_size, eps=1e-6, bias=False)
- self._post_init()
-
- def apply_index(self, tensor: torch.Tensor, index: torch.Tensor):
- @@ -327,7 +331,9 @@ def apply_index(self, tensor: torch.Tensor, index: torch.Tensor):
-
- def _post_init(self):
- for name, param in self.named_parameters():
- - if "bias" in name:
- + # Also zero sinks: they are torch.empty at ctor and some checkpoints
- + # omit individual visual.blocks.*.attn.sinks keys.
- + if "bias" in name or name.endswith("sinks"):
- param.data.zero_()
-
- def get_window_index_1d(self, grid_thw, col=True):
- diff --git a/python/sglang/srt/multimodal/processors/mimo_v2.py b/python/sglang/srt/multimodal/processors/mimo_v2.py
- index e914f00b1a0e..35a43469fdcf 100644
- --- a/python/sglang/srt/multimodal/processors/mimo_v2.py
- +++ b/python/sglang/srt/multimodal/processors/mimo_v2.py
- @@ -214,8 +214,12 @@ def __post_init__(self):
- )
-
-
- -_QWEN2VL_PIXEL_MEAN = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
- -_QWEN2VL_PIXEL_STD = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
- +# OPENAI_CLIP stats, matching Qwen2VLImageProcessor (preprocessor_config.json
- +# image_mean/image_std with do_rescale 1/255). Kept in 0-255 space because
- +# standardize_batch sees un-rescaled PIL tensors. ImageNet values were wrong
- +# for MiMo-VL and scrambled colors.
- +_QWEN2VL_PIXEL_MEAN = torch.Tensor([0.48145466, 0.4578275, 0.40821073]).view(-1, 1, 1) * 255.0
- +_QWEN2VL_PIXEL_STD = torch.Tensor([0.26862954, 0.26130258, 0.27577711]).view(-1, 1, 1) * 255.0
- _mean_std_cache = {}
-
-
|
