Skip to content

Broadcasting

Kernel examples for shape broadcast and rank decomposition. Theory is in Broadcasting. Bertha lowering (VPU / MPU, SRAM vs mpu_buffer) is in Broadcasting on Bertha.

Each decorator creates the Bertha backend directly with lambda ctx: bertha.Bertha(ctx, "bertha"), using the current EVT0 defaults. See Typical Backend Setup for the backend factory interface.

Add — leading 1

[1, 256] + [256, 256] → [256, 256]. The first row of a stretches down every row of b. See Leading-1 add and VPU leaves. For the meaning of a leaf, see Broadcasting.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"a": "universal", "b": "universal", "out": "universal"},
)
def add_leading_one(
    a: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [1, 256], "shared_memory"
    ),
    b: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [256, 256], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [256, 256], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(a, device.get_core(0), "sram")
        legato.tensor.request_load(b, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(a, 0, "sram")
        y = legato.tensor.receive(b, 0, "sram")
        legato.tensor.memcpy(out, x + y)

Conv — batched N*

[2, 8, 64] is two unbatched [8, 64] MPU convs. The weight has no batch prefix. See Spatial N* and Batched conv_1d.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"inp": "universal", "weight": "universal", "out": "universal"},
)
def conv1d_nstar(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [3, 64, 64], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 8, 64], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.conv_1d(x, w, None, (1,), (1,), (1, 1), "zeros")
        legato.tensor.memcpy(out, y)

Dwconv — batched N*

[2, 19, 256] / weight [4, 256] → two [19, 256] depthwise convs, same w. Keep the weight in SRAM when the batch repeats. See Batched dwconv_1d.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"inp": "universal", "weight": "universal", "out": "universal"},
)
def dwconv1d_nstar(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 19, 256], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 256], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [2, 16, 256], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.dwconv_1d(x, w)
        legato.tensor.memcpy(out, y)

Pool — unbatched leaf

[8, 64] average pool is already a leaf (k=2, s=2 → [4, 64]).
More information about "leaf", See Broadcasting. See avg_pool_1d.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"inp": "universal", "out": "universal"},
)
def avg_pool1d_leaf(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [8, 64], "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 64], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        y = legato.math.avg_pool_1d(x, (2,), (2,))
        legato.tensor.memcpy(out, y)

LayerNorm — last-axis rows

[4, 128] becomes four rank-1 [128] leaves, the smallest units of decomposition. Affine rows (multiplying by a weight and adding a bias after normalization) are (1, (128, 256)) for the current loading requirements; the math uses length 128. For the meaning of a leaf, see Broadcasting. See Last-axis rows and layer_norm.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"inp": "universal", "weight": "universal", "bias": "universal", "out": "universal"},
)
def layernorm_rows(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 128], "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
    ),
    bias: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (1, (128, 256)), "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), [4, 128], "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
        legato.tensor.request_load(bias, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        b = legato.tensor.receive(bias, 0, "sram")
        y = legato.math.layer_norm(x, w, b)
        legato.tensor.memcpy(out, y)

RMSNorm — padded last dim

(4, (70, 256)): four rows, logical D=70 in a 256-wide row. The remaining elements are padding. RMSNorm divides by the actual number of elements, 70, rather than the physical width, 256. See rms_norm padded.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"inp": "universal", "weight": "universal", "out": "universal"},
)
def rmsnorm_padded(
    inp: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
    ),
    weight: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (1, (70, 256)), "shared_memory"
    ),
    out: legato.get_model().tensor_type(
        legato.types.float("bfloat16"), (4, (70, 256)), "shared_memory"
    ),
):
    device = legato.get_context().get_device()
    with device.get_top():
        legato.tensor.request_load(inp, device.get_core(0), "sram")
        legato.tensor.request_load(weight, device.get_core(0), "sram")
    with device.get_core(0):
        x = legato.tensor.receive(inp, 0, "sram")
        w = legato.tensor.receive(weight, 0, "sram")
        y = legato.math.rms_norm(x, w)
        legato.tensor.memcpy(out, y)