Skip to content

Broadcasting on Bertha

This page shows supported broadcast patterns for the selected Bertha backend. Read Broadcasting for shape rules, and Bertha API for operation-specific input constraints. Use the backend factory from Typical Backend Setup.

Applying broadcast rules

Leading batch dimensions describe independent applications of the same operation. Physical padding is storage capacity rather than additional input values. Keep the declared storage kinds, logical shapes, and physical capacities compatible.

Core PerformBroadcast produces either one leaf operation or a prefix loop over independent batch items. Bertha converts those loops from hscf.for to scf.for and assigns each supported leaf to an execution unit:

Leaf operation Execution unit
Rank-1 elementwise arithmetic VPU (vector unit)
Last-axis softmax, RMSNorm, or LayerNorm VPU (vector unit)
Unbatched convolution, pooling, or depthwise convolution MPU (matrix unit)
Rank-2 matmul MPU (matrix unit)

VPU leaves

The examples below cover a leading size-1 input and matching inputs with a padded last dimension. A packed SRAM singleton last dimension can also broadcast to a row; a padded singleton column remains unsupported.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
@legato.compile(backend=get_bertha, param_kinds={"a": "universal", "b": "universal", "out": "universal"})
def add_leading_one(
    a: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [1, 256], "shared_memory"
    ),
    b: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [256, 256], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [256, 256], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(a, device.get_core(0), "sram")
        legato.tensor.request_load(b, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(a, 0, "sram")
        y = legato.tensor.receive(b, 0, "sram")
        legato.tensor.memcpy(out, x + y)
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
@legato.compile(backend=get_bertha, param_kinds={"a": "universal", "b": "universal", "out": "universal"})
def add_padded_last_dim(
    a: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), ((-1, 4), 2, (128, 256)), "shared_dram"
    ),
    b: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), ((-1, 4), 2, (128, 256)), "shared_dram"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), ((-1, 4), 2, (128, 256)), "shared_dram"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(a, device.get_core(0), "sram")
        legato.tensor.request_load(b, device.get_core(0), "sram")
    with device.get_core(0):
        seq_len = legato.tensor.get_dimension(a, 0)
        loaded_a = legato.tensor.receive(a, 0, "sram")
        loaded_b = legato.tensor.receive(b, 1, "sram")
        out_view = legato.tensor.view(
            out,
            (
                legato.range(0, seq_len),
                legato.range(0, 2),
                legato.range(0, 128),
            ),
        )
        legato.tensor.memcpy(out_view, loaded_a + loaded_b)

MPU leaves

These operations apply the same input rules to each batch item. Keep spatial weights independent of leading batch dimensions. Reusable SRAM weights remain available across iterations; one-use streamed weights must not be duplicated or replayed. See Weight storage requirements.

Unit leading dimensions of a matmul RHS are squeezed; varying batch dimensions select a weight slice for each iteration. If that pattern needs reusable weights, use compatible SRAM storage. Reuse is constrained by the configured max_reuse_num limit (32 by default on EVT0). Unsupported streamed replay reports LE0184.

conv_1d [8, 64] — already a leaf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
@legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def conv1d_leaf(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [8, 64], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [3, 64, 64], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [8, 64], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.conv_1d(x, w, None, (1,), (1,), (1, 1), "zeros")
        legato.tensor.memcpy(out, y)

Batched conv_1d [2, 8, 64]

Two [8, 64] MPU convolutions. The [3, 64, 64] SRAM weight is forwarded unchanged and may be replayed.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
@legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def conv1d_nstar(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [3, 64, 64], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.conv_1d(x, w, None, (1,), (1,), (1, 1), "zeros")
        legato.tensor.memcpy(out, y)

Batched dwconv_1d [2, 19, 256]

Same prefix-loop idea. Weight [4, 64] is not sliced. Keep it in SRAM if the batch repeats.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
@legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def dwconv1d_nstar(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 19, 256], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 256], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 16, 256], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.dwconv_1d(x, w)
        legato.tensor.memcpy(out, y)

avg_pool_1d [8, 64]

Already a leaf. EVT0 divides by the full kernel area, including pads (Operations).

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
@legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "out": "universal"})
def avg_pool1d_leaf(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [8, 64], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 64], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        y = legato.math.avg_pool_1d(x, (2,), (2,))
        legato.tensor.memcpy(out, y)

Normalization leaves

LayerNorm and RMSNorm use the VPU to normalize the last logical axis of each row. Padding capacity does not affect the divisor or add input elements. See RMSNorm and LayerNorm.

layer_norm [4, 128]

Four [128] leaves. Affine rows use (1, (128, 256)) to meet the current loading requirements; the math still uses length 128.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
@legato.compile(
    backend=get_bertha,
    param_kinds={"inp": "universal", "weight": "universal", "bias": "universal", "out": "universal"},
)
def layernorm_rows(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 128], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
    ),
    bias: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 128], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
        legato.tensor.request_load(bias, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        b = legato.tensor.receive(bias, 0, "sram")
        y = legato.math.layer_norm(x, w, b)
        legato.tensor.memcpy(out, y)

rms_norm padded D=70 on physical 256

Still four rank-1 leaves. The divisor is 70, not 256.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
@legato.compile(backend=get_bertha, param_kinds={"inp": "universal", "weight": "universal", "out": "universal"})
def rmsnorm_padded(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (1, (70, 256)), "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.rms_norm(x, w)
        legato.tensor.memcpy(out, y)