Skip to content

Tensor Memory Patterns

Runtime allocation and partial copies

The script compiles allocation from a runtime extent and executes a partial copy between SRAM views. The allocation/copy pattern uses launch=False: it can block during stream synchronization in the current EVT0 Torch FSim build, so no numerical runtime claim is made for that part. The destination SRAM buffer is explicitly stored back to DRAM; otherwise changing its view would not update the host's output tensor. The copy uses full FP32 rows with a legal 32-element physical width (one 128-byte SRAM word) and verifies both copied and preserved output rows.

Download tensor_memory.py, or run the checked-out files. See Running the examples for environment setup.

1
HA_BACKEND=sim HA_TOOLCHAIN=1 python "$DOCS_ROOT/docs/legato-language/tutorials/code/tensor_memory.py"
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
"""Dynamic allocations and partial SRAM copies with externally visible outputs."""

import os
from pathlib import Path

import torch

import legato
import legato.model.bertha as bertha
from legato.language import arith

# Store compiler artifacts and optional simulator logs in a configurable directory.
OUTPUT = Path(os.environ.get("LEGATO_EXAMPLE_OUTPUT", "legato_example_output"))


@legato.compile(backend=lambda ctx: bertha.Bertha(ctx, "bertha"), param_kinds={"input": "universal", "out": "core"})
def dynamic_to_sram(
    input: legato.get_model().tensor_type(legato.types.float("bfloat16"), [16, 256], "shared_dram"),
    out: legato.get_model().tensor_type(legato.types.float("bfloat16"), [16, 256], "shared_dram"),
    count: legato.types.integer("int64"),
):
    """Use a runtime dimension for allocation and copy the result back to DRAM."""
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(input, device.get_core(0), "sram")
    with device.get_core(0):
        loaded = legato.tensor.receive(input, 0, "sram")
        rows = legato.range(0, arith.cast(count, legato.types.index()))
        loaded_view = legato.tensor.view(loaded, (rows, legato.range(0, 256)))
        dim0 = legato.tensor.get_dimension(loaded_view, 0)
        tmp = legato.tensor.allocate(legato.types.tensor(legato.types.float("bfloat16"), ((-1, 16), 256)), (dim0,))
        legato.tensor.memcpy(tmp, loaded_view)
        out_view = legato.tensor.view(out, (rows, legato.range(0, 256)))
        legato.tensor.memcpy(out_view, tmp)


@legato.compile(backend=lambda ctx: bertha.Bertha(ctx, "bertha"))
def copy_view(
    src: legato.get_model().tensor_type(legato.types.float("float32"), [16, 32], "shared_dram"),
    dst: legato.get_model().tensor_type(legato.types.float("float32"), [16, 32], "shared_dram"),
):
    """Copy eight full rows within SRAM, then store the modified destination."""
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(src, device.get_core(0), "sram")
        legato.tensor.request_load(dst, device.get_core(0), "sram")
    with device.get_core(0):
        loaded_src = legato.tensor.receive(src, 0, "sram")
        loaded_dst = legato.tensor.receive(dst, 0, "sram")
        src_view = legato.tensor.view(loaded_src, (legato.range(4, 12), legato.range(0, 32)))
        dst_view = legato.tensor.view(loaded_dst, (legato.range(0, 8), legato.range(0, 32)))
        legato.tensor.memcpy(dst_view, src_view)
        legato.tensor.memcpy(dst, loaded_dst)


def verify_dynamic_to_sram():
    """Compile the runtime-sized allocation without launching it."""
    cpu = torch.empty(16, 256, dtype=torch.bfloat16)
    # Compile-only: runtime-sized allocate + memcpy can block in current EVT0 Torch FSim.
    # Compile a binary without initializing a device or launching.
    with legato.session(output_type=legato.OutputType.BINARY, output_path=str(OUTPUT), launch=False):
        dynamic_to_sram(cpu, legato.Arg(cpu, bertha.core(0)), 10)


def verify_copy_view():
    """Check copied rows and every untouched destination element."""
    # Register the LPU PyTorch backend before allocating device tensors.
    import hart
    import torch_ha

    torch_ha._autoload()
    # HA_BACKEND=sim selects EVT0 Torch FSim; otherwise use the hardware runtime.
    if os.environ.get("HA_BACKEND") == "sim":
        OUTPUT.mkdir(parents=True, exist_ok=True)
        hart.set_fsim_env(
            hart.HaDeviceVersion.BERTHA_CLOUD_EVT0,
            hart.HaFSimComputeMode.TORCH,
            str(OUTPUT / "fsim.log"),
            "error",
        )
    hart.set_device(0)
    # Keep CPU inputs reproducible without initializing a device RNG.
    torch.random.default_generator.manual_seed(0)
    src_cpu = torch.randn(16, 32)
    dst_cpu = torch.randn(16, 32)
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    src = torch.ops.lpu.empty_with_kind(
        list(src_cpu.shape), "dram_word_interleaved", dtype=src_cpu.dtype, device="lpu:0"
    )
    src.copy_(src_cpu.contiguous())
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    dst = torch.ops.lpu.empty_with_kind(
        list(dst_cpu.shape), "dram_word_interleaved", dtype=dst_cpu.dtype, device="lpu:0"
    )
    dst.copy_(dst_cpu.contiguous())
    # Compile a binary and launch it on the selected device.
    with legato.session(output_type=legato.OutputType.BINARY, output_path=str(OUTPUT), launch=True):
        copy_view(src, dst)
    # Surface asynchronous execution errors before reading the result.
    hart.stream_synchronize()
    expected = dst_cpu.clone()
    expected[:8] = src_cpu[4:12]
    # Read back the synchronized result and compare every element with the CPU reference.
    actual = dst.cpu().float()
    assert torch.isfinite(actual).all(), "Non-finite device result"
    torch.testing.assert_close(actual, expected.float(), atol=0, rtol=0)


def main():
    """Compile the dynamic pattern; execute and verify the partial-copy destination."""
    verify_dynamic_to_sram()
    verify_copy_view()


if __name__ == "__main__":
    main()

Fill a tensor: frontend only

full and standalone full_like currently produce frontend MLIR but lack standalone EVT0 backend lowering. The following script deliberately uses OutputType.MLIR, launch=False; it is not an executable device fill. Use a host-initialized tensor copied to DRAM when a runnable fill is needed. If a standalone fill survives into assignment, the refined diagnostic reports LE0121 with the LE2104 fill restriction and host-initialization advice. The compile diagnostics example demonstrates how to inspect recorded reasons and distinguish frontend success from backend support.

A fixed-shape fill takes dynamic_shape=None. Only genuinely dynamic dimensions accept runtime index operands. Passing constant extents to a nominally dynamic fill may infer a fixed result while retaining shape operands, which fails later verification. See Dynamic shapes.

Download tensor_fill_frontend.py, then run:

1
python "$DOCS_ROOT/docs/legato-language/tutorials/code/tensor_fill_frontend.py"
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""Frontend fill patterns; standalone full/full_like lack EVT0 backend lowering."""

import os
from pathlib import Path

import torch

import legato
import legato.model.bertha as bertha
from legato.language import arith

# Store compiler artifacts and optional simulator logs in a configurable directory.
OUTPUT = Path(os.environ.get("LEGATO_EXAMPLE_OUTPUT", "legato_example_output"))


@legato.compile(backend=lambda ctx: bertha.Bertha(ctx, "bertha"))
def fill_tensor(
    out: legato.get_model().tensor_type(legato.types.float("bfloat16"), [16, 64], "shared_dram"),
):
    """A fixed-shape fill takes no dynamic-shape operands."""
    with legato.get_context().get_device().get_core(0):
        fill = arith.cast(-1.0, legato.types.float("bfloat16"))
        result = legato.tensor.full(
            legato.get_model().tensor_type(legato.types.float("bfloat16"), [16, 64], "row_major"), fill, None
        )
        legato.tensor.memcpy(out, result)


@legato.compile(backend=lambda ctx: bertha.Bertha(ctx, "bertha"))
def fill_like_loaded(
    x: legato.get_model().tensor_type(legato.types.float("bfloat16"), [16, 64], "shared_dram"),
    out: legato.get_model().tensor_type(legato.types.float("bfloat16"), [16, 64], "shared_dram"),
):
    """Describe a filled result with the loaded tensor's type and shape."""
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(x, device.get_core(0), "sram")
    with device.get_core(0):
        loaded = legato.tensor.receive(x, 0, "sram")
        fill = arith.cast(-1.0, legato.types.float("bfloat16"))
        legato.tensor.memcpy(out, legato.tensor.full_like(loaded, fill))


def verify_fill_tensor():
    """Emit the fixed-shape fill operation and its minus-one value."""
    cpu = torch.empty(16, 64, dtype=torch.bfloat16)
    # Emit frontend MLIR without initializing a device or launching.
    with legato.session(output_type=legato.OutputType.MLIR, output_path=str(OUTPUT), launch=False):
        fill_tensor(cpu)


def verify_fill_like_loaded():
    """Emit the fill-like operation and its minus-one value."""
    cpu = torch.empty(16, 64, dtype=torch.bfloat16)
    # Emit frontend MLIR without initializing a device or launching.
    with legato.session(output_type=legato.OutputType.MLIR, output_path=str(OUTPUT), launch=False):
        fill_like_loaded(cpu, cpu)


def main():
    """Emit frontend MLIR only, without advertising unsupported binary execution."""
    verify_fill_tensor()
    verify_fill_like_loaded()


if __name__ == "__main__":
    main()