Broadcasting on Bertha
This page shows supported broadcast patterns for the selected Bertha backend.
Read Broadcasting for shape rules, and
Bertha API for operation-specific input constraints.
Use the backend factory from Typical Backend Setup.
Applying broadcast rules
Leading batch dimensions describe independent applications of the same operation.
Physical padding is storage capacity rather than additional input values.
Keep the declared storage kinds, logical shapes, and physical capacities compatible.
Core PerformBroadcast produces either one leaf operation or a prefix loop
over independent batch items. Bertha converts those loops from hscf.for to
scf.for and assigns each supported leaf to an execution unit:
| Leaf operation |
Execution unit |
| Rank-1 elementwise arithmetic |
VPU (vector unit) |
| Last-axis softmax, RMSNorm, or LayerNorm |
VPU (vector unit) |
| Unbatched convolution, pooling, or depthwise convolution |
MPU (matrix unit) |
| Rank-2 matmul |
MPU (matrix unit) |
VPU leaves
The examples below cover a leading size-1 input and matching inputs with a
padded last dimension. A packed SRAM singleton last dimension can also
broadcast to a row; a padded singleton column remains unsupported.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 | @legato.compile(backend=get_bertha, param_kinds={"a": "universal", "b": "universal", "out": "universal"})
def add_leading_one(
a: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [1, 256], "shared_memory"
),
b: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [256, 256], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [256, 256], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(a, device.get_core(0), "sram")
legato.tensor.request_load(b, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(a, 0, "sram")
y = legato.tensor.receive(b, 0, "sram")
legato.tensor.memcpy(out, x + y)
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29 | @legato.compile(backend=get_bertha, param_kinds={"a": "universal", "b": "universal", "out": "universal"})
def add_padded_last_dim(
a: legato.get_model().tensor_type(
legato.types.float("bfloat16"), ((-1, 4), 2, (128, 256)), "shared_dram"
),
b: legato.get_model().tensor_type(
legato.types.float("bfloat16"), ((-1, 4), 2, (128, 256)), "shared_dram"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), ((-1, 4), 2, (128, 256)), "shared_dram"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(a, device.get_core(0), "sram")
legato.tensor.request_load(b, device.get_core(0), "sram")
with device.get_core(0):
seq_len = legato.tensor.get_dimension(a, 0)
loaded_a = legato.tensor.receive(a, 0, "sram")
loaded_b = legato.tensor.receive(b, 1, "sram")
out_view = legato.tensor.view(
out,
(
legato.range(0, seq_len),
legato.range(0, 2),
legato.range(0, 128),
),
)
legato.tensor.memcpy(out_view, loaded_a + loaded_b)
|
MPU leaves
These operations apply the same input rules to each batch item. Keep spatial
weights independent of leading batch dimensions. Reusable SRAM weights remain
available across iterations; one-use streamed weights must not be duplicated
or replayed. See Weight storage requirements.
Unit leading dimensions of a matmul RHS are squeezed; varying batch dimensions
select a weight slice for each iteration. If that pattern needs reusable
weights, use compatible SRAM storage. Reuse is constrained by the configured
max_reuse_num limit (32 by default on EVT0). Unsupported streamed replay
reports LE0184.
conv_1d [8, 64] — already a leaf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 | @legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def conv1d_leaf(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [8, 64], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [3, 64, 64], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [8, 64], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.conv_1d(x, w, None, (1,), (1,), (1, 1), "zeros")
legato.tensor.memcpy(out, y)
|
Batched conv_1d [2, 8, 64]
Two [8, 64] MPU convolutions. The [3, 64, 64] SRAM weight is forwarded
unchanged and may be replayed.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 | @legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def conv1d_nstar(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [3, 64, 64], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.conv_1d(x, w, None, (1,), (1,), (1, 1), "zeros")
legato.tensor.memcpy(out, y)
|
Batched dwconv_1d [2, 19, 256]
Same prefix-loop idea. Weight [4, 64] is not sliced. Keep it in SRAM
if the batch repeats.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 | @legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def dwconv1d_nstar(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 19, 256], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 256], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [2, 16, 256], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.dwconv_1d(x, w)
legato.tensor.memcpy(out, y)
|
avg_pool_1d [8, 64]
Already a leaf. EVT0 divides by the full kernel area, including pads
(Operations).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 | @legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "out": "universal"})
def avg_pool1d_leaf(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [8, 64], "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 64], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
y = legato.math.avg_pool_1d(x, (2,), (2,))
legato.tensor.memcpy(out, y)
|
Normalization leaves
LayerNorm and RMSNorm use the VPU to normalize the last logical axis of each row.
Padding capacity does not affect the divisor or add input elements.
See RMSNorm and LayerNorm.
layer_norm [4, 128]
Four [128] leaves. Affine rows use (1, (128, 256)) to meet the current loading requirements; the math still uses length 128.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29 | @legato.compile(
backend=get_bertha,
param_kinds={"inp": "universal", "weight": "universal", "bias": "universal", "out": "universal"},
)
def layernorm_rows(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 128], "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
),
bias: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), [4, 128], "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
legato.tensor.request_load(bias, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
b = legato.tensor.receive(bias, 0, "sram")
y = legato.math.layer_norm(x, w, b)
legato.tensor.memcpy(out, y)
|
rms_norm padded D=70 on physical 256
Still four rank-1 leaves. The divisor is 70, not 256.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 | @legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def rmsnorm_padded(
inp: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
),
weight: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (1, (70, 256)), "shared_memory"
),
out: legato.get_model().tensor_type(
legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
),
):
device = legato.get_context().get_device()
with device.get_top():
legato.tensor.request_load(inp, device.get_core(0), "sram")
legato.tensor.request_load(weight, device.get_core(0), "sram")
with device.get_core(0):
x = legato.tensor.receive(inp, 0, "sram")
w = legato.tensor.receive(weight, 0, "sram")
y = legato.math.rms_norm(x, w)
legato.tensor.memcpy(out, y)
|