launcher/docker-images/dsv4-flash-vision/plugin/dsv4_vision/model.py

"""Vision-Exp wrapper around vLLM DeepseekV4ForCausalLM.

Loads official `vision.*` / `aligner.*` / image sentinel tensors and merges
ViT embeddings on prefill. Language decode stays on the B12X DeepSeek V4 path.
"""

from __future__ import annotations

from types import SimpleNamespace
from typing import Any

import torch
import torch.nn as nn

from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.models.deepseek_v4 import DeepseekV4ForCausalLM

from .vision import Aligner, ViT


def _vision_args(hf: Any) -> SimpleNamespace:
    return SimpleNamespace(
        vision_n_layers=int(getattr(hf, "vision_n_layers", 0) or 0),
        vision_dim=int(getattr(hf, "vision_dim", 1024)),
        vision_n_heads=int(getattr(hf, "vision_n_heads", 16)),
        vision_inter_dim=int(getattr(hf, "vision_inter_dim", 2816)),
        vision_patch_size=int(getattr(hf, "vision_patch_size", 14)),
        vision_rope_theta=float(getattr(hf, "vision_rope_theta", 10000.0)),
        vision_downsample_ratio=int(getattr(hf, "vision_downsample_ratio", 3)),
        vision_max_n_token=int(getattr(hf, "vision_max_n_token", 384)),
        vision_min_pixels=int(getattr(hf, "vision_min_pixels", 147456)),
        vision_max_wh_ratio=int(getattr(hf, "vision_max_wh_ratio", 8)),
        dim=int(getattr(hf, "hidden_size", 4096)),
        num_hash_layers=int(getattr(hf, "num_hash_layers", 0) or 0),
    )


class DeepseekV4VisionForCausalLM(DeepseekV4ForCausalLM):
    # ModelConfig.is_multimodal_model keys off this class attribute
    # (interfaces.supports_multimodal) — without it the OpenAI layer
    # answers "not a multimodal model" for image_url requests.
    supports_multimodal = True

    def __init__(self, *, vllm_config, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        hf = vllm_config.model_config.hf_config
        args = _vision_args(hf)
        self._vision_args = args
        if args.vision_n_layers <= 0:
            self.vision = None
            self.aligner = None
            return
        self.vision = ViT(args)
        self.aligner = Aligner(args)
        self.image_start = nn.Parameter(torch.empty(args.dim))
        self.image_end = nn.Parameter(torch.empty(args.dim))
        self.image_newline = nn.Parameter(torch.empty(args.dim))
        self.image_pad = nn.Parameter(torch.empty(args.dim))

    def encode_image(self, patches: torch.Tensor, n_vit_h: int, n_vit_w: int) -> torch.Tensor:
        return self.aligner(self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w)

    def forward(
        self,
        input_ids,
        positions,
        intermediate_tensors=None,
        inputs_embeds=None,
        **kwargs,
    ):
        # Spec-decode extra kwargs must not TypeError. lm_head and
        # get_mtp_target_hidden_states stay on the parent class.
        return super().forward(
            input_ids,
            positions,
            intermediate_tensors,
            inputs_embeds,
        )

    def load_weights(self, weights):
        if self.vision is None:
            return super().load_weights(weights)
        leftover = []
        stacked = {
            "vision.": self.vision,
            "aligner.": self.aligner,
        }
        params = {
            "image_start": self.image_start,
            "image_end": self.image_end,
            "image_newline": self.image_newline,
            "image_pad": self.image_pad,
        }
        for name, tensor in weights:
            if name in params:
                params[name].data.copy_(tensor)
                continue
            routed = False
            for prefix, module in stacked.items():
                if name.startswith(prefix):
                    AutoWeightsLoader(module).load_weights(
                        [(name[len(prefix) :], tensor)]
                    )
                    routed = True
                    break
            if not routed:
                leftover.append((name, tensor))
        leftover = [
            pair
            for pair in leftover
            if not _skip_unmapped_gate(pair[0], self._vision_args)
        ]
        return super().load_weights(leftover)


def _skip_unmapped_gate(name: str, args: SimpleNamespace) -> bool:
    # Hash MoE layers register tid2eid and leave e_score_correction_bias as
    # None. The B12X mapper still rewrites ffn.gate.bias onto that missing
    # name. bias_vl has no mapper at all.
    if name.endswith("ffn.gate.bias_vl"):
        return True
    if name.endswith("ffn.gate.bias"):
        parts = name.split(".")
        if (
            len(parts) >= 5
            and parts[0] == "layers"
            and parts[1].isdigit()
            and int(parts[1]) < args.num_hash_layers
        ):
            return True
    return False