learned

learned

LearnedPositionalEncoding1D

LearnedPositionalEncoding1D(embedding_dimension, position_source=TENSOR_INDICES.value, maximum_sequence_length=5000, mlp_hidden_dimensions=None, mlp_activation=SiLU)

Bases: PositionalEncoding1D

Learned positional encoding for 1D.

Source code in src/versatil/models/layers/positional_encoding/learned.py

def __init__(
    self,
    embedding_dimension: int,
    position_source: str = PositionSource.TENSOR_INDICES.value,
    maximum_sequence_length: int = 5000,
    mlp_hidden_dimensions: list[int] | None = None,
    mlp_activation: Callable | None = nn.SiLU,
):
    if maximum_sequence_length is None:
        raise ValueError(
            "maximum_sequence_length must be provided for 1D learned encoding"
        )
    super().__init__(
        embedding_dimension=embedding_dimension,
        position_source=position_source,
        precompute_encodings=False,
        maximum_sequence_length=maximum_sequence_length,
        mlp_hidden_dimensions=mlp_hidden_dimensions,
        mlp_activation=mlp_activation,
    )
    self.learned_encoding = nn.Embedding(
        maximum_sequence_length, embedding_dimension
    )

LearnedPositionalEncoding2D

LearnedPositionalEncoding2D(embedding_dimension, max_height=50, max_width=50, mlp_hidden_dimensions=None, mlp_activation=SiLU)

Bases: PositionalEncoding2D

Learned positional encoding for 2D.

Source code in src/versatil/models/layers/positional_encoding/learned.py

def __init__(
    self,
    embedding_dimension: int,
    max_height: int = 50,
    max_width: int = 50,
    mlp_hidden_dimensions: list[int] | None = None,
    mlp_activation: Callable | None = nn.SiLU,
):
    if embedding_dimension % 2 != 0:
        raise ValueError("embedding_dimension must be even for 2D learned encoding")
    if max_height is None or max_width is None:
        raise ValueError(
            "max_height and max_width must be provided for 2D learned encoding"
        )
    half_dim = embedding_dimension // 2

    super().__init__(
        embedding_dimension=embedding_dimension,
        mlp_hidden_dimensions=mlp_hidden_dimensions,
        mlp_activation=mlp_activation,
    )
    self.row_encoding = nn.Embedding(max_height, half_dim)
    self.col_encoding = nn.Embedding(max_width, half_dim)