1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119 | """Causal attention with a runtime K/V length shorter than its physical capacity."""
import os
from pathlib import Path
import torch
import legato
import legato.model.bertha as bertha
from legato.language import arith
# Store compiler artifacts and optional simulator logs in a configurable directory.
OUTPUT = Path(os.environ.get("LEGATO_EXAMPLE_OUTPUT", "legato_example_output"))
@legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"query": "top", "key": "top", "value": "top", "length": "universal", "out": "core"},
)
def kernel(
query: legato.get_model().tensor_type(legato.types.float("bfloat16"), (32, 128), "shared_dram"),
key: legato.get_model().tensor_type(legato.types.float("bfloat16"), (2048, 128), "transposed_mpu_dram"),
value: legato.get_model().tensor_type(legato.types.float("bfloat16"), (2048, 128), "mpu_dram"),
length: legato.types.integer("int64"),
out: legato.get_model().tensor_type(legato.types.float("bfloat16"), (32, 128), "shared_dram"),
):
"""Compute causal attention for a runtime prefix with a fixed physical capacity."""
device = legato.get_context().get_device()
dtype = legato.types.float("bfloat16")
core = device.get_core(0)
with device.get_top():
rows = legato.range(0, arith.cast(length, legato.types.index()))
legato.tensor.request_load(query, core, "sram")
legato.tensor.request_load(legato.tensor.view(key, (rows, legato.range(0, 128))), core, "mpu_buffer")
legato.tensor.request_load(legato.tensor.view(value, (rows, legato.range(0, 128))), core, "mpu_buffer")
with core:
cache_count = arith.cast(length, legato.types.index())
key_type = legato.get_model().tensor_type(dtype, (128, (-1, 2048)), "mpu_weight")
value_type = legato.get_model().tensor_type(dtype, ((-1, 2048), 128), "mpu_weight")
q = legato.tensor.receive(query, 0, "sram")
k = legato.tensor.receive_type(key_type, 0, (cache_count,), "mpu_buffer")
v = legato.tensor.receive_type(value_type, 1, (cache_count,), "mpu_buffer")
# Same score/softmax/P@V sequence as the generated fused GQA attention kernels.
sequence_start = cache_count - arith.cast(32, legato.types.index())
scores = legato.math.attention_scores(q, k, 0.08838834764831845, True, 1, sequence_start)
row_max = legato.math.reduce_max(scores, -1, True)
exp_scores = legato.math.exp(scores - row_max)
probabilities = exp_scores / legato.math.reduce_sum(exp_scores, -1, True)
legato.tensor.memcpy(out, probabilities @ v)
def main():
"""Read only the initialized K/V prefix and compare all query rows."""
# Register the LPU PyTorch backend before allocating device tensors.
import hart
import torch_ha
torch_ha._autoload()
# HA_BACKEND=sim selects EVT0 Torch FSim; otherwise use the hardware runtime.
if os.environ.get("HA_BACKEND") == "sim":
OUTPUT.mkdir(parents=True, exist_ok=True)
hart.set_fsim_env(
hart.HaDeviceVersion.BERTHA_CLOUD_EVT0,
hart.HaFSimComputeMode.TORCH,
str(OUTPUT / "fsim.log"),
"error",
)
hart.set_device(0)
# Keep CPU inputs reproducible without initializing a device RNG.
torch.random.default_generator.manual_seed(0)
length = 256
query_cpu = (torch.rand(32, 128) * 2 - 1).to(torch.bfloat16)
key_cpu = (torch.rand(2048, 128) * 2 - 1).to(torch.bfloat16)
value_cpu = (torch.rand(2048, 128) * 2 - 1).to(torch.bfloat16)
# Allocate the physical layout expected by the kernel and copy contiguous host data.
query = torch.ops.lpu.empty_with_kind(
list(query_cpu.shape), "dram_word_interleaved", dtype=query_cpu.dtype, device="lpu:0"
)
query.copy_(query_cpu.contiguous())
# Allocate the physical layout expected by the kernel and copy contiguous host data.
key = torch.ops.lpu.empty_with_kind(
list(key_cpu.shape), "mpu_tensor_transposed", dtype=key_cpu.dtype, device="lpu:0"
)
key.copy_(key_cpu.contiguous())
# Allocate the physical layout expected by the kernel and copy contiguous host data.
value = torch.ops.lpu.empty_with_kind(list(value_cpu.shape), "mpu_tensor", dtype=value_cpu.dtype, device="lpu:0")
value.copy_(value_cpu.contiguous())
out_cpu = torch.zeros_like(query_cpu)
# Allocate the physical layout expected by the kernel and copy contiguous host data.
out = torch.ops.lpu.empty_with_kind(
list(out_cpu.shape), "dram_word_interleaved", dtype=out_cpu.dtype, device="lpu:0"
)
out.copy_(out_cpu.contiguous())
# Compile a binary and launch it on the selected device.
with legato.session(output_type=legato.OutputType.BINARY, output_path=str(OUTPUT), launch=True):
kernel(
legato.Arg(query, bertha.top),
legato.Arg(key, bertha.top),
legato.Arg(value, bertha.top),
length,
legato.Arg(out, bertha.core(0)),
)
# Surface asynchronous execution errors before reading the result.
hart.stream_synchronize()
scores = query_cpu.float() @ key_cpu[:length].float().T / 128**0.5
positions = torch.arange(length - 32, length).unsqueeze(1)
mask = torch.arange(length).unsqueeze(0) > positions
probabilities = torch.softmax(scores.masked_fill(mask, float("-inf")), dim=-1)
# Read back the synchronized result and compare every element with the CPU reference.
actual = out.cpu().float()
assert torch.isfinite(actual).all(), "Non-finite device result"
torch.testing.assert_close(actual, (probabilities @ value_cpu[:length].float()).float(), atol=0.005, rtol=0.05)
if __name__ == "__main__":
main()
|