conditional_cnn

conditional_cnn

FiLM-conditioned CNN encoder for conditioned vision encoding.

ConditionalCNNEncoder

ConditionalCNNEncoder(input_keys, condition_key, conditioning_dimension, backbone=value, pooling_method=value, batch_norm_handling=value, pretrained=False, frozen=False, model_dtype=None, lora_config=None)

Bases: RGBEncoderMixin, ConditionalEncoder

CNN encoder with FiLM conditioning for conditioned vision, e.g. from language features.

Initialize FiLM-conditioned CNN encoder.

Parameters:

Name	Type	Description	Default
input_keys	str | list[str]	Camera observation keys.	required
condition_key	str	Key for the conditioning feature tensor.	required
conditioning_dimension	int	Dimensionality of the conditioning feature.	required
backbone	str	timm ResNet model name.	value
pooling_method	str	Feature pooling strategy.	value
batch_norm_handling	str	How to handle batch normalization layers.	value
pretrained	bool	Whether to load pretrained weights.	False
frozen	bool	Whether to freeze all parameters.	False
model_dtype	str | None	Precision string from experiment config (e.g. "bf16-mixed").	None
lora_config	LoRAAdaptation | None	Optional PEFT LoRA adapter configuration.	None

Source code in src/versatil/models/encoding/encoders/rgb/conditional_cnn.py

def __init__(
    self,
    input_keys: str | list[str],
    condition_key: str,
    conditioning_dimension: int,
    backbone: str = SpatialBackboneType.RESNET18.value,
    pooling_method: str = PoolingMethod.SPATIAL_SOFTMAX.value,
    batch_norm_handling: str = BatchNormHandling.FROZEN.value,
    pretrained: bool = False,
    frozen: bool = False,
    model_dtype: str | None = None,
    lora_config: LoRAAdaptation | None = None,
) -> None:
    """Initialize FiLM-conditioned CNN encoder.

    Args:
        input_keys: Camera observation keys.
        condition_key: Key for the conditioning feature tensor.
        conditioning_dimension: Dimensionality of the conditioning feature.
        backbone: timm ResNet model name.
        pooling_method: Feature pooling strategy.
        batch_norm_handling: How to handle batch normalization layers.
        pretrained: Whether to load pretrained weights.
        frozen: Whether to freeze all parameters.
        model_dtype: Precision string from experiment config (e.g. ``"bf16-mixed"``).
        lora_config: Optional PEFT LoRA adapter configuration.
    """
    specification = EncoderInput(
        keys=input_keys,
        required_camera_modalities=[CameraModality.RGB],
        conditioning_key=condition_key,
    )
    super().__init__(
        input_specification=specification,
        pretrained=pretrained,
        frozen=frozen,
        model_dtype=model_dtype,
    )
    self._setup_camera_keys(input_keys=self.input_specification.keys)
    self.condition_key = condition_key
    self.conditioning_dimension = conditioning_dimension
    self.batch_norm_handling = batch_norm_handling
    self.backbone_name = backbone
    self.pooling_method = pooling_method
    self.lora_config = lora_config

    if backbone not in self.BACKBONE_CONFIGS:
        raise ValueError(
            f"Backbone {backbone} not supported for FiLM Conditioning. "
            f"Supported: {list(self.BACKBONE_CONFIGS.keys())}"
        )

    self._build_filmed_backbone()
    self.feature_dim = self.BACKBONE_CONFIGS[backbone]["feature_dim"]
    self.pooling_head: PoolingHead | None = None
    self.output_dim: int | tuple[int, ...] = self.feature_dim
    if frozen:
        super()._freeze_weights()
    self._apply_model_dtype()

encode

encode(inputs, conditioning)

Encode images with FiLM conditioning.

Parameters:

Name	Type	Description	Default
inputs	dict[str, Tensor]	Dict with camera images as (B, C, H, W) per camera key.	required
conditioning	Tensor	Conditioning tensor as (B, D).	required

Returns:

Type	Description
dict[str, Tensor]	Dict with RGB features. Single camera: key is rgb.
dict[str, Tensor]	Multiple cameras: keys are rgb:{camera_key} per camera.

Source code in src/versatil/models/encoding/encoders/rgb/conditional_cnn.py

def encode(
    self,
    inputs: dict[str, torch.Tensor],
    conditioning: torch.Tensor,
) -> dict[str, torch.Tensor]:
    """Encode images with FiLM conditioning.

    Args:
        inputs: Dict with camera images as (B, C, H, W) per camera key.
        conditioning: Conditioning tensor as (B, D).

    Returns:
        Dict with RGB features. Single camera: key is ``rgb``.
        Multiple cameras: keys are ``rgb:{camera_key}`` per camera.
    """
    modality = EncoderOutputKeys.RGB.value
    if self.is_multi_camera:
        result = {}
        for camera_key in self.camera_keys:
            features = self._encode_conditioned_image(
                images=inputs[camera_key], conditioning=conditioning
            )
            result[f"{modality}:{camera_key}"] = features
        return result
    features = self._encode_conditioned_image(
        images=inputs[self.camera_keys[0]], conditioning=conditioning
    )
    return {modality: features}

set_image_size

set_image_size(image_height, image_width)

Compute feature map dimensions and create pooling head.

Parameters:

Name	Type	Description	Default
image_height	int	Target image height.	required
image_width	int	Target image width.	required

Source code in src/versatil/models/encoding/encoders/rgb/conditional_cnn.py

def set_image_size(self, image_height: int, image_width: int) -> None:
    """Compute feature map dimensions and create pooling head.

    Args:
        image_height: Target image height.
        image_width: Target image width.
    """
    mock_input_dtype = self._mock_forward_dtype()
    with torch.no_grad(), self._mock_forward_autocast():
        mock_input = torch.zeros(
            1, 3, image_height, image_width, dtype=mock_input_dtype
        )
        mock_condition = torch.zeros(
            1, self.conditioning_dimension, dtype=mock_input_dtype
        )
        features = self.backbone(mock_input, mock_condition)
        _, _, spatial_height, spatial_width = features.shape
    self._setup_pooling(spatial_height=spatial_height, spatial_width=spatial_width)
    if self.frozen:
        self._freeze_weights()
    self._apply_model_dtype()

validate_input_metadata

validate_input_metadata(key, metadata)

Validate that input metadata is camera metadata.

Parameters:

Name	Type	Description	Default
key	str	Observation key being validated.	required
metadata	BaseMetadata	Metadata from the observation space.	required

Returns:

Type	Description
str | None	Error message if incompatible, None if valid.

Source code in src/versatil/models/encoding/encoders/rgb/conditional_cnn.py

def validate_input_metadata(self, key: str, metadata: BaseMetadata) -> str | None:
    """Validate that input metadata is camera metadata.

    Args:
        key: Observation key being validated.
        metadata: Metadata from the observation space.

    Returns:
        Error message if incompatible, None if valid.
    """
    if not isinstance(metadata, CameraMetadata):
        return f"Expected CameraMetadata for '{key}', got {type(metadata).__name__}"
    return None

get_explainability_targets

get_explainability_targets()

Return the last FiLM residual block for spatial attribution maps.

Returns:

Type	Description
list[VisionExplanationTarget]	One NCHW spatial feature-map target pointing at the final block of
list[VisionExplanationTarget]	layer4 in the conditioned ResNet backbone.

Source code in src/versatil/models/encoding/encoders/rgb/conditional_cnn.py

def get_explainability_targets(self) -> list[VisionExplanationTarget]:
    """Return the last FiLM residual block for spatial attribution maps.

    Returns:
        One NCHW spatial feature-map target pointing at the final block of
        ``layer4`` in the conditioned ResNet backbone.
    """
    return [
        VisionExplanationTarget(
            layer=self.backbone.layer4[-1],
            target_kind=ExplanationTargetKind.SPATIAL_FEATURE_MAP.value,
            activation_layout=ActivationLayout.NCHW.value,
        )
    ]

get_output_specification

get_output_specification()

Get structured output specification with feature names and dimensions.

Returns:

Type	Description
list[FeatureMetadata]	List of FeatureMetadata with per-camera feature names and pooled dimensions.

Source code in src/versatil/models/encoding/encoders/rgb/conditional_cnn.py

def get_output_specification(self) -> list[FeatureMetadata]:
    """Get structured output specification with feature names and dimensions.

    Returns:
        List of FeatureMetadata with per-camera feature names and pooled dimensions.
    """
    feature_names = self._get_vision_feature_names()
    dimension = (
        (self.output_dim,) if isinstance(self.output_dim, int) else self.output_dim
    )
    return [
        FeatureMetadata(
            key=name,
            feature_type=infer_feature_type(dimension),
            dimension=dimension,
        )
        for name in feature_names
    ]