dinov2_siglip

dinov2_siglip

DINOv2+SigLIP RGB encoder producing fused patch-token sequences.

DinoV2SigLIPBackboneConfig dataclass

DinoV2SigLIPBackboneConfig(dino_backbone, siglip_backbone, image_size)

Resolved timm tower configuration for a paired DINOv2+SigLIP backbone.

DinoV2SigLIPRGBEncoder

DinoV2SigLIPRGBEncoder(input_keys, pretrained, frozen, backbone=value, model_dtype=None, lora_config=None)

Bases: RGBEncoderMixin, Encoder

RGB encoder that concatenates DINOv2 and SigLIP patch features.

Initialize paired timm vision towers.

Parameters:

Name	Type	Description	Default
input_keys	str | list[str]	RGB camera observation keys.	required
pretrained	bool	Whether timm should load pretrained tower weights.	required
frozen	bool	Whether to freeze both vision towers.	required
backbone	str	DINOv2+SigLIP paired backbone identifier.	value
model_dtype	str | None	Precision string from experiment config (e.g. "bf16-mixed").	None
lora_config	LoRAAdaptation | None	Optional LoRA adapter configuration for the timm towers.	None

Source code in src/versatil/models/encoding/encoders/rgb/dinov2_siglip.py

def __init__(
    self,
    input_keys: str | list[str],
    pretrained: bool,
    frozen: bool,
    backbone: str = DinoV2SigLIPBackboneType.DINOV2_SIGLIP_VIT_SO_224PX.value,
    model_dtype: str | None = None,
    lora_config: LoRAAdaptation | None = None,
) -> None:
    """Initialize paired timm vision towers.

    Args:
        input_keys: RGB camera observation keys.
        pretrained: Whether timm should load pretrained tower weights.
        frozen: Whether to freeze both vision towers.
        backbone: DINOv2+SigLIP paired backbone identifier.
        model_dtype: Precision string from experiment config (e.g. ``"bf16-mixed"``).
        lora_config: Optional LoRA adapter configuration for the timm towers.
    """
    specification = EncoderInput(
        keys=input_keys,
        required_camera_modalities=[CameraModality.RGB],
    )
    super().__init__(
        input_specification=specification,
        pretrained=pretrained,
        frozen=frozen,
        model_dtype=model_dtype,
    )
    # The mixin resolves self.model_dtype into a torch.dtype; the child
    # tower constructors validate the raw precision string, so keep it.
    self._model_dtype_string = model_dtype
    self._setup_camera_keys(input_keys=self.input_specification.keys)
    backbone_config = self._resolve_backbone_config(backbone=backbone)
    self.backbone_name = backbone
    self.dino_model_name = backbone_config.dino_backbone.value
    self.siglip_model_name = backbone_config.siglip_backbone.value
    self.image_size = backbone_config.image_size
    self.lora_config = lora_config
    self.dino_encoder = self._build_flat_encoder(
        backbone=backbone_config.dino_backbone
    )
    self.siglip_encoder = self._build_flat_encoder(
        backbone=backbone_config.siglip_backbone
    )
    self.register_buffer(
        "dino_standardization_mean",
        torch.tensor(IMAGENET_RGB_MEAN).view(1, 3, 1, 1),
        persistent=False,
    )
    self.register_buffer(
        "dino_standardization_std",
        torch.tensor(IMAGENET_RGB_STD).view(1, 3, 1, 1),
        persistent=False,
    )
    self.register_buffer(
        "siglip_standardization_mean",
        torch.tensor(SIGLIP_RGB_MEAN).view(1, 3, 1, 1),
        persistent=False,
    )
    self.register_buffer(
        "siglip_standardization_std",
        torch.tensor(SIGLIP_RGB_STD).view(1, 3, 1, 1),
        persistent=False,
    )
    self.num_patches = int(self.dino_encoder.backbone.patch_embed.num_patches)
    siglip_num_patches = int(self.siglip_encoder.backbone.patch_embed.num_patches)
    if self.num_patches != siglip_num_patches:
        raise ValueError(
            "DINO and SigLIP patch counts must match, got "
            f"{self.num_patches} and {siglip_num_patches}."
        )
    self.feature_dim = int(
        self.dino_encoder.feature_dim + self.siglip_encoder.feature_dim
    )
    self.embedding_dimension = self.feature_dim
    self.output_dim = (-1, self.feature_dim)
    if frozen:
        super()._freeze_weights()
    self._apply_model_dtype()

encode_image_tokens

encode_image_tokens(images)

Encode images into fused DINOv2+SigLIP patch tokens.

Parameters:

Name	Type	Description	Default
images	Tensor	RGB tensor with shape (B, 3, H, W).	required

Returns:

Type	Description
Tensor	Fused patch tokens with shape (B, P, D_dino + D_siglip).

Source code in src/versatil/models/encoding/encoders/rgb/dinov2_siglip.py

def encode_image_tokens(self, images: torch.Tensor) -> torch.Tensor:
    """Encode images into fused DINOv2+SigLIP patch tokens.

    Args:
        images: RGB tensor with shape ``(B, 3, H, W)``.

    Returns:
        Fused patch tokens with shape ``(B, P, D_dino + D_siglip)``.
    """
    images = resize_to_target_size(
        images=images,
        target_height=self.image_size,
        target_width=self.image_size,
    )
    dino_pixel_values = self._standardize_images(
        pixel_values=images,
        mean=self.dino_standardization_mean,
        standard_deviation=self.dino_standardization_std,
    )
    siglip_pixel_values = self._standardize_images(
        pixel_values=images,
        mean=self.siglip_standardization_mean,
        standard_deviation=self.siglip_standardization_std,
    )
    dino_features = self.dino_encoder._encode_single_image(dino_pixel_values)
    siglip_features = self.siglip_encoder._encode_single_image(siglip_pixel_values)
    return torch.cat([dino_features, siglip_features], dim=2)

encode

encode(inputs)

Encode images into RGB patch-token features.

Source code in src/versatil/models/encoding/encoders/rgb/dinov2_siglip.py

def encode(self, inputs: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
    """Encode images into RGB patch-token features."""
    return self._encode_vision(inputs)

validate_input_metadata

validate_input_metadata(key, metadata)

Validate that input metadata is camera metadata.

Parameters:

Name	Type	Description	Default
key	str	Observation key being validated.	required
metadata	BaseMetadata	Metadata from the observation space.	required

Returns:

Type	Description
str | None	Error message if incompatible, None if valid.

Source code in src/versatil/models/encoding/encoders/rgb/dinov2_siglip.py

def validate_input_metadata(self, key: str, metadata: BaseMetadata) -> str | None:
    """Validate that input metadata is camera metadata.

    Args:
        key: Observation key being validated.
        metadata: Metadata from the observation space.

    Returns:
        Error message if incompatible, None if valid.
    """
    if not isinstance(metadata, CameraMetadata):
        return f"Expected CameraMetadata for '{key}', got {type(metadata).__name__}"
    return None

get_output_specification

get_output_specification()

Get output specification for fused patch-token features.

Source code in src/versatil/models/encoding/encoders/rgb/dinov2_siglip.py

def get_output_specification(self) -> list[FeatureMetadata]:
    """Get output specification for fused patch-token features."""
    feature_names = self._get_vision_feature_names()
    dimension = (-1, self.feature_dim)
    return [
        FeatureMetadata(
            key=name,
            feature_type=infer_feature_type(dimension),
            dimension=dimension,
        )
        for name in feature_names
    ]