huggingface

huggingface

HuggingFace-backed VLM base class.

HuggingFaceGenerativeVLM

HuggingFaceGenerativeVLM(input_keys, pretrained, frozen, model_name, attention_type=value, model_dtype=None, max_text_length=None, lora_config=None)

Bases: GenerativeVLM, ABC

Base class for VLMs loaded through HuggingFace image-text generation APIs.

Load or initialize a HuggingFace VLM component.

Parameters:

Name	Type	Description	Default
input_keys	str | list[str]	RGB camera keys consumed by the VLM.	required
pretrained	bool	Whether to load pretrained HuggingFace weights.	required
frozen	bool	Whether to freeze all model weights.	required
model_name	str	HuggingFace model identifier.	required
attention_type	str	HuggingFace attention implementation.	value
model_dtype	str | None	Optional precision string for model parameter dtype.	None
max_text_length	int | None	Optional text sequence length. Defaults to the text config maximum position count.	None
lora_config	LoRAAdaptation | None	Optional LoRA adapter configuration.	None

Source code in src/versatil/models/decoding/generative_language_models/vision_language/huggingface.py

def __init__(
    self,
    input_keys: str | list[str],
    pretrained: bool,
    frozen: bool,
    model_name: str,
    attention_type: str = AttentionImplementation.SDPA.value,
    model_dtype: str | None = None,
    max_text_length: int | None = None,
    lora_config: LoRAAdaptation | None = None,
):
    """Load or initialize a HuggingFace VLM component.

    Args:
        input_keys: RGB camera keys consumed by the VLM.
        pretrained: Whether to load pretrained HuggingFace weights.
        frozen: Whether to freeze all model weights.
        model_name: HuggingFace model identifier.
        attention_type: HuggingFace attention implementation.
        model_dtype: Optional precision string for model parameter dtype.
        max_text_length: Optional text sequence length. Defaults to the
            text config maximum position count.
        lora_config: Optional LoRA adapter configuration.
    """
    super().__init__(
        input_keys=input_keys,
        pretrained=pretrained,
        frozen=frozen,
        model_dtype=model_dtype,
        max_text_length=max_text_length,
    )
    self.model_name = model_name
    self.lora_config = lora_config
    config = AutoConfig.from_pretrained(model_name)
    if pretrained:
        self.vlm = AutoModelForImageTextToText.from_pretrained(
            model_name,
            attn_implementation=attention_type,
        )
    else:
        self.vlm = AutoModelForImageTextToText.from_config(
            config,
            attn_implementation=attention_type,
        )
    self.vlm = apply_lora_config(
        model=self.vlm,
        lora_config=lora_config,
        frozen=frozen,
    )
    self.image_size: int = config.vision_config.image_size
    self.hidden_dimension: int = config.text_config.hidden_size
    self.num_image_tokens_per_camera: int = self._compute_num_image_tokens(
        config=config
    )
    self.max_text_length = (
        max_text_length
        if max_text_length is not None
        else config.text_config.max_position_embeddings
    )
    if frozen:
        super()._freeze_weights()
    self._apply_model_dtype()

resize_token_embeddings

resize_token_embeddings(vocabulary_size)

Resize the HuggingFace VLM token embeddings.

Source code in src/versatil/models/decoding/generative_language_models/vision_language/huggingface.py

def resize_token_embeddings(self, vocabulary_size: int) -> None:
    """Resize the HuggingFace VLM token embeddings."""
    self.vlm.resize_token_embeddings(vocabulary_size)
    self._get_language_model().config.vocab_size = vocabulary_size

forward_language_model

forward_language_model(input_ids=None, inputs_embeds=None, attention_mask=None, past_key_values=None, use_cache=False, cache_position=None, position_ids=None, output_hidden_states=True)

Run the nested language model over token IDs or embeddings.

Parameters:

Name	Type	Description	Default
input_ids	Tensor | None	Optional token IDs with shape (B, S).	None
inputs_embeds	Tensor | None	Optional token embeddings with shape (B, S, D).	None
attention_mask	Tensor | None	Optional language-model attention mask.	None
past_key_values	Cache | tuple[tuple[Tensor, ...], ...] | None	Optional cached key/value tensors.	None
use_cache	bool	Whether to return/update cached key/value tensors.	False
cache_position	Tensor | None	Optional HuggingFace KV-cache slots for the tokens in this call. During cached decoding, if the prefix has length P, the next token uses cache slot P so its key/value is appended after the prefix.	None
position_ids	Tensor | None	Optional positions for the language model positional encoding, with shape (B, S). These should count real tokens, not padding: [PAD, PAD, t0, t1] should pass [0, 0, 0, 1] so t0 and t1 get positions 0 and 1.	None
output_hidden_states	bool	Whether to return hidden states.	True

Returns:

Type	Description
CausalLanguageModelOutput	Causal language-model output with logits shape (B, S, V).

Source code in src/versatil/models/decoding/generative_language_models/vision_language/huggingface.py

def forward_language_model(
    self,
    input_ids: torch.Tensor | None = None,
    inputs_embeds: torch.Tensor | None = None,
    attention_mask: torch.Tensor | None = None,
    past_key_values: Cache | tuple[tuple[torch.Tensor, ...], ...] | None = None,
    use_cache: bool = False,
    cache_position: torch.Tensor | None = None,
    position_ids: torch.Tensor | None = None,
    output_hidden_states: bool = True,
) -> CausalLanguageModelOutput:
    """Run the nested language model over token IDs or embeddings.

    Args:
        input_ids: Optional token IDs with shape ``(B, S)``.
        inputs_embeds: Optional token embeddings with shape ``(B, S, D)``.
        attention_mask: Optional language-model attention mask.
        past_key_values: Optional cached key/value tensors.
        use_cache: Whether to return/update cached key/value tensors.
        cache_position: Optional HuggingFace KV-cache slots for the tokens
            in this call. During cached decoding, if the prefix has length
            ``P``, the next token uses cache slot ``P`` so its key/value is
            appended after the prefix.
        position_ids: Optional positions for the language model positional
            encoding, with shape ``(B, S)``. These should count real tokens,
            not padding: ``[PAD, PAD, t0, t1]`` should pass
            ``[0, 0, 0, 1]`` so ``t0`` and ``t1`` get positions ``0`` and
            ``1``.
        output_hidden_states: Whether to return hidden states.

    Returns:
        Causal language-model output with logits shape ``(B, S, V)``.
    """
    language_model = self._get_language_model()
    language_model_inputs = {
        "input_ids": input_ids,
        "inputs_embeds": inputs_embeds,
        "attention_mask": attention_mask,
        "past_key_values": past_key_values,
        "use_cache": use_cache,
        "output_hidden_states": output_hidden_states,
        "return_dict": True,
    }
    if cache_position is not None:
        language_model_inputs["cache_position"] = cache_position
    if position_ids is not None:
        language_model_inputs["position_ids"] = position_ids
    language_output = language_model(**language_model_inputs)
    last_hidden_state = language_output.last_hidden_state
    hidden_states = language_output.hidden_states
    if hidden_states is None:
        hidden_states = (last_hidden_state,)
    return CausalLanguageModelOutput(
        hidden_states=hidden_states,
        logits=self.vlm.get_output_embeddings()(last_hidden_state),
        past_key_values=language_output.past_key_values,
    )