"""Single-core BF16 matmul with a bounded dynamic activation row count."""
import os
from pathlib import Path
import torch
import legato
import legato.model.bertha as bertha
# Store compiler artifacts and optional simulator logs in a configurable directory.
OUTPUT = Path(os.environ.get("LEGATO_EXAMPLE_OUTPUT", "legato_example_output"))
@legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"a": "universal", "b": "top", "out": "core"},
)
def simple_matmul(
a: legato.get_model().tensor_type(legato.types.float("bfloat16"), ((-1, 10), 256), "shared_dram"),
b: legato.get_model().tensor_type(legato.types.float("bfloat16"), [256, 128], "mpu_dram"),
out: legato.get_model().tensor_type(legato.types.float("bfloat16"), [10, 128], "shared_dram"),
):
"""Load BF16 operands, multiply on core zero, and store the output."""
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(a, device.get_core(0), "sram")
legato.tensor.request_load(b, device.get_core(0), "mpu_buffer")
with device.get_core(0):
weight_type = legato.get_model().tensor_type(legato.types.float("bfloat16"), [256, 128], "mpu_weight")
loaded_a = legato.tensor.receive(a, 0, "sram")
loaded_b = legato.tensor.receive_type(weight_type, 0, None, "mpu_buffer")
result = loaded_a @ loaded_b
legato.tensor.memcpy(out, result)
def main():
"""Compare the complete output with a CPU matrix product."""
# Register the LPU PyTorch backend before allocating device tensors.
import hart
import torch_ha
torch_ha._autoload()
# HA_BACKEND=sim selects EVT0 Torch FSim; otherwise use the hardware runtime.
if os.environ.get("HA_BACKEND") == "sim":
OUTPUT.mkdir(parents=True, exist_ok=True)
hart.set_fsim_env(
hart.HaDeviceVersion.BERTHA_CLOUD_EVT0,
hart.HaFSimComputeMode.TORCH,
str(OUTPUT / "fsim.log"),
"error",
)
hart.set_device(0)
# Keep CPU inputs reproducible without initializing a device RNG.
torch.random.default_generator.manual_seed(0)
a_cpu = (torch.randn(10, 256) * 0.25).to(torch.bfloat16)
b_cpu = (torch.randn(256, 128) * 0.25).to(torch.bfloat16)
# Allocate the physical layout expected by the kernel and copy contiguous host data.
a = torch.ops.lpu.empty_with_kind(list(a_cpu.shape), "dram_word_interleaved", dtype=a_cpu.dtype, device="lpu:0")
a.copy_(a_cpu.contiguous())
# Allocate the physical layout expected by the kernel and copy contiguous host data.
b = torch.ops.lpu.empty_with_kind(list(b_cpu.shape), "mpu_tensor", dtype=b_cpu.dtype, device="lpu:0")
b.copy_(b_cpu.contiguous())
out_cpu = torch.zeros(10, 128, dtype=torch.bfloat16)
# Allocate the physical layout expected by the kernel and copy contiguous host data.
out = torch.ops.lpu.empty_with_kind(
list(out_cpu.shape), "dram_word_interleaved", dtype=out_cpu.dtype, device="lpu:0"
)
out.copy_(out_cpu.contiguous())
# Compile a binary and launch it on the selected device.
with legato.session(output_type=legato.OutputType.BINARY, output_path=str(OUTPUT), launch=True):
simple_matmul(a, legato.Arg(b, bertha.top), legato.Arg(out, bertha.core(0)))
# Surface asynchronous execution errors before reading the result.
hart.stream_synchronize()
# Read back the synchronized result and compare every element with the CPU reference.
actual = out.cpu().float()
assert torch.isfinite(actual).all(), "Non-finite device result"
torch.testing.assert_close(actual, (a_cpu.float() @ b_cpu.float()).float(), atol=0.05, rtol=0.05)
if __name__ == "__main__":
main()