Broadcasting
Kernel examples for shape broadcast and rank decomposition. Theory is in
Broadcasting. Bertha lowering (VPU / MPU,
SRAM vs mpu_buffer) is in
Broadcasting on Bertha.
Each decorator creates the Bertha backend directly with
lambda ctx: bertha.Bertha(ctx, "bertha"), using the current EVT0 defaults.
See Typical Backend Setup
for the backend factory interface.
Add — leading 1
[1, 256] + [256, 256] → [256, 256]. The first row of a stretches down
every row of b. See
Leading-1 add
and VPU leaves.
For the meaning of a leaf, see Broadcasting.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 | @legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"a": "universal", "b": "universal", "out": "universal"},
)
def add_leading_one(
a: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [1, 256], "shared_memory"
),
b: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [256, 256], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [256, 256], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(a, device.get_core(0), "sram")
legato.tensor.request_load(b, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(a, 0, "sram")
y = legato.tensor.receive(b, 0, "sram")
legato.tensor.memcpy(out, x + y)
|
Conv — batched N*
[2, 8, 64] is two unbatched [8, 64] MPU convs. The weight has no batch
prefix. See
Spatial N*
and Batched conv_1d.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 | @legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"inp": "universal", "weight": "universal", "out": "universal"},
)
def conv1d_nstar(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [3, 64, 64], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.conv_1d(x, w, None, (1,), (1,), (1, 1), "zeros")
legato.tensor.memcpy(out, y)
|
Dwconv — batched N*
[2, 19, 256] / weight [4, 256] → two [19, 256] depthwise convs, same
w. Keep the weight in SRAM when the batch repeats. See
Batched dwconv_1d.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 | @legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"inp": "universal", "weight": "universal", "out": "universal"},
)
def dwconv1d_nstar(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 19, 256], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 256], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 16, 256], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.dwconv_1d(x, w)
legato.tensor.memcpy(out, y)
|
Pool — unbatched leaf
[8, 64] average pool is already a leaf (k=2, s=2 → [4, 64]).
More information about "leaf", See Broadcasting.
See avg_pool_1d.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 | @legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"inp": "universal", "out": "universal"},
)
def avg_pool1d_leaf(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [8, 64], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 64], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
y = legato.math.avg_pool_1d(x, (2,), (2,))
legato.tensor.memcpy(out, y)
|
LayerNorm — last-axis rows
[4, 128] becomes four rank-1 [128] leaves, the smallest units of decomposition.
Affine rows (multiplying by a weight and adding a bias after normalization) are
(1, (128, 256)) for the current loading requirements; the math uses length 128.
For the meaning of a leaf, see Broadcasting.
See Last-axis rows and layer_norm.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29 | @legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"inp": "universal", "weight": "universal", "bias": "universal", "out": "universal"},
)
def layernorm_rows(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 128], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
),
bias: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 128], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
legato.tensor.request_load(bias, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
b = legato.tensor.receive(bias, 0, "sram")
y = legato.math.layer_norm(x, w, b)
legato.tensor.memcpy(out, y)
|
RMSNorm — padded last dim
(4, (70, 256)): four rows, logical D=70 in a 256-wide row.
The remaining elements are padding. RMSNorm divides by the actual number of
elements, 70, rather than the physical width, 256.
See rms_norm padded.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 | @legato.compile(
backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
param_kinds={"inp": "universal", "weight": "universal", "out": "universal"},
)
def rmsnorm_padded(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (1, (70, 256)), "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.rms_norm(x, w)
legato.tensor.memcpy(out, y)
|