Skip to content

Math And Masking

Operations live under legato.math; tensor selection is legato.tensor.where. A language operation being accepted by the frontend does not imply that its standalone form can lower to an EVT0 binary.

This runnable example supplies already-masked scores and their row maxima from the host, then performs legato.math.softmax(scores, -1, maximum) on the device. It keeps mask and maximum preparation on the host and avoids a standalone full_like fill. Standalone last-axis reduce_max is also supported under the EVT0 constraints below. For a mask generated on device as part of a supported attention pattern, see Causal Attention.

maximum must be the maximum of each post-mask row. The caller must keep at least one finite score in every row. The example uses a finite -30 mask for BF16 scores; it checks the exact same finite-mask formulation on CPU.

Download math_and_masking.py, or run the checked-out files. See Running the examples for environment setup.

1
HA_BACKEND=sim HA_TOOLCHAIN=1 python "$DOCS_ROOT/docs/legato-language/tutorials/code/math_and_masking.py"
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
"""Masked softmax using a host-computed maximum of the already-masked scores."""

import os
from pathlib import Path

import torch

import legato
import legato.model.bertha as bertha

# Store compiler artifacts and optional simulator logs in a configurable directory.
OUTPUT = Path(os.environ.get("LEGATO_EXAMPLE_OUTPUT", "legato_example_output"))


@legato.compile(backend=lambda ctx: bertha.Bertha(ctx, "bertha"))
def masked_softmax(
    scores: legato.get_model().tensor_type(legato.types.float("bfloat16"), [4, 256], "shared_dram"),
    maximum: legato.get_model().tensor_type(legato.types.float("bfloat16"), (4, (1, 256)), "shared_dram"),
    out: legato.get_model().tensor_type(legato.types.float("bfloat16"), [4, 256], "shared_dram"),
):
    """Consume already-masked scores and the maximum of each masked row."""
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(scores, device.get_core(0), "sram")
        legato.tensor.request_load(maximum, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(scores, 0, "sram")
        m = legato.tensor.receive(maximum, 0, "sram")
        legato.tensor.memcpy(out, legato.math.softmax(x, -1, m))


def main():
    """Build the mask and maximum on the host, then verify the device normalization."""
    # Register the LPU PyTorch backend before allocating device tensors.
    import hart
    import torch_ha

    torch_ha._autoload()
    # HA_BACKEND=sim selects EVT0 Torch FSim; otherwise use the hardware runtime.
    if os.environ.get("HA_BACKEND") == "sim":
        OUTPUT.mkdir(parents=True, exist_ok=True)
        hart.set_fsim_env(
            hart.HaDeviceVersion.BERTHA_CLOUD_EVT0,
            hart.HaFSimComputeMode.TORCH,
            str(OUTPUT / "fsim.log"),
            "error",
        )
    hart.set_device(0)
    # Keep CPU inputs reproducible without initializing a device RNG.
    torch.random.default_generator.manual_seed(0)
    scores_cpu = torch.randn(4, 256, dtype=torch.bfloat16)
    positions = torch.arange(252, 256).unsqueeze(1)
    scores_cpu = scores_cpu.masked_fill(torch.arange(256).unsqueeze(0) > positions, -30.0)
    max_host = torch.zeros(4, 256, dtype=torch.bfloat16)
    max_host[:, :1] = scores_cpu.max(-1, keepdim=True).values
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    maximum_storage = torch.ops.lpu.empty_with_kind(
        list(max_host.shape), "dram_word_interleaved", dtype=max_host.dtype, device="lpu:0"
    )
    maximum_storage.copy_(max_host.contiguous())
    maximum = maximum_storage[:, :1]
    out_cpu = torch.zeros_like(scores_cpu)
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    out = torch.ops.lpu.empty_with_kind(
        list(out_cpu.shape), "dram_word_interleaved", dtype=out_cpu.dtype, device="lpu:0"
    )
    out.copy_(out_cpu.contiguous())
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    scores = torch.ops.lpu.empty_with_kind(
        list(scores_cpu.shape), "dram_word_interleaved", dtype=scores_cpu.dtype, device="lpu:0"
    )
    scores.copy_(scores_cpu.contiguous())
    # Compile a binary and launch it on the selected device.
    with legato.session(output_type=legato.OutputType.BINARY, output_path=str(OUTPUT), launch=True):
        masked_softmax(scores, maximum, out)
    # Surface asynchronous execution errors before reading the result.
    hart.stream_synchronize()
    # Read back the synchronized result and compare every element with the CPU reference.
    actual = out.cpu().float()
    assert torch.isfinite(actual).all(), "Non-finite device result"
    torch.testing.assert_close(actual, (torch.softmax(scores_cpu.float(), -1)).float(), atol=0.001, rtol=0.05)


if __name__ == "__main__":
    main()
  • legato.math.exp(tensor)
  • legato.math.sqrt(tensor)
  • legato.math.reduce_max(tensor, axis)
  • legato.math.reduce_min(tensor, axis)
  • legato.math.reduce_sum(tensor, axis)
  • legato.tensor.tril(tensor, diagonal)
  • legato.tensor.where(condition, true_value, false_value)

Joined attention scores on Bertha EVT0

For attention split across head bands, add the partial score tensors before subtracting legato.math.reduce_max(scores, -1, True). The sum of the separate band maxima can exceed every joined score and underflow all exponentials.

The standalone VPU maximum lowering supports FP16, BF16, and FP32 last-axis reductions with keepdim=True, and row-major static logical rows of 1–16384 elements. Physical capacity must hold the row rounded up to a 64-element group. Lowering fills an unaligned consumed tail with -inf, while aligned rows need no fill. Capacity beyond the reduction extent is untouched. Each row address retains its physical pitch; dynamic prefix dimensions are allowed, but dynamic logical reduction widths remain unsupported. Attention-specific MPU maximum fusion remains unchanged. A known unsupported axis or dtype is recorded by the component check and normally reported at executor assignment. See the compile diagnostics example for headline and candidate reason codes. For sliding-window attention, reduce the maximum after applying the window mask so an excluded attention sink cannot determine the shift.