콘텐츠로 이동

단순 Matmul

이 예제는 전체 코드를 담은 BF16 예제로, 현재 백엔드 기본값, 상한이 있는 동적 activation 차원, 명시적 param_kinds, MPU weight receive를 사용합니다. activation과 출력은 word-interleaved shared DRAM을 쓰고, weight는 mpu_tensor 저장 공간을 씁니다. 실행을 동기화한 뒤 모든 출력을 CPU matmul 결과와 비교합니다.

simple_matmul.py를 내려받거나, 체크아웃한 파일을 실행하십시오. 환경 설정은 예제 실행하기를 참고하십시오.

1
HA_BACKEND=sim HA_TOOLCHAIN=1 python "$DOCS_ROOT/docs/legato-language/tutorials/code/simple_matmul.py"
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
"""Single-core BF16 matmul with a bounded dynamic activation row count."""

import os
from pathlib import Path

import torch

import legato
import legato.model.bertha as bertha

# Store compiler artifacts and optional simulator logs in a configurable directory.
OUTPUT = Path(os.environ.get("LEGATO_EXAMPLE_OUTPUT", "legato_example_output"))


@legato.compile(
    backend=lambda ctx: bertha.Bertha(ctx, "bertha"),
    param_kinds={"a": "universal", "b": "top", "out": "core"},
)
def simple_matmul(
    a: legato.get_model().tensor_type(legato.types.float("bfloat16"), ((-1, 10), 256), "shared_dram"),
    b: legato.get_model().tensor_type(legato.types.float("bfloat16"), [256, 128], "mpu_dram"),
    out: legato.get_model().tensor_type(legato.types.float("bfloat16"), [10, 128], "shared_dram"),
):
    """Load BF16 operands, multiply on core zero, and store the output."""
    device = legato.get_context().get_device()

    with device.get_top():
        legato.tensor.request_load(a, device.get_core(0), "sram")
        legato.tensor.request_load(b, device.get_core(0), "mpu_buffer")

    with device.get_core(0):
        weight_type = legato.get_model().tensor_type(legato.types.float("bfloat16"), [256, 128], "mpu_weight")
        loaded_a = legato.tensor.receive(a, 0, "sram")
        loaded_b = legato.tensor.receive_type(weight_type, 0, None, "mpu_buffer")
        result = loaded_a @ loaded_b
        legato.tensor.memcpy(out, result)


def main():
    """Compare the complete output with a CPU matrix product."""
    # Register the LPU PyTorch backend before allocating device tensors.
    import hart
    import torch_ha

    torch_ha._autoload()
    # HA_BACKEND=sim selects EVT0 Torch FSim; otherwise use the hardware runtime.
    if os.environ.get("HA_BACKEND") == "sim":
        OUTPUT.mkdir(parents=True, exist_ok=True)
        hart.set_fsim_env(
            hart.HaDeviceVersion.BERTHA_CLOUD_EVT0,
            hart.HaFSimComputeMode.TORCH,
            str(OUTPUT / "fsim.log"),
            "error",
        )
    hart.set_device(0)
    # Keep CPU inputs reproducible without initializing a device RNG.
    torch.random.default_generator.manual_seed(0)
    a_cpu = (torch.randn(10, 256) * 0.25).to(torch.bfloat16)
    b_cpu = (torch.randn(256, 128) * 0.25).to(torch.bfloat16)
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    a = torch.ops.lpu.empty_with_kind(list(a_cpu.shape), "dram_word_interleaved", dtype=a_cpu.dtype, device="lpu:0")
    a.copy_(a_cpu.contiguous())
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    b = torch.ops.lpu.empty_with_kind(list(b_cpu.shape), "mpu_tensor", dtype=b_cpu.dtype, device="lpu:0")
    b.copy_(b_cpu.contiguous())
    out_cpu = torch.zeros(10, 128, dtype=torch.bfloat16)
    # Allocate the physical layout expected by the kernel and copy contiguous host data.
    out = torch.ops.lpu.empty_with_kind(
        list(out_cpu.shape), "dram_word_interleaved", dtype=out_cpu.dtype, device="lpu:0"
    )
    out.copy_(out_cpu.contiguous())
    # Compile a binary and launch it on the selected device.
    with legato.session(output_type=legato.OutputType.BINARY, output_path=str(OUTPUT), launch=True):
        simple_matmul(a, legato.Arg(b, bertha.top), legato.Arg(out, bertha.core(0)))
    # Surface asynchronous execution errors before reading the result.
    hart.stream_synchronize()
    # Read back the synchronized result and compare every element with the CPU reference.
    actual = out.cpu().float()
    assert torch.isfinite(actual).all(), "Non-finite device result"
    torch.testing.assert_close(actual, (a_cpu.float() @ b_cpu.float()).float(), atol=0.05, rtol=0.05)


if __name__ == "__main__":
    main()

receive(a, 0, "sram")은 a와 연결된 첫 번째 receive tile을 선택합니다. receive_type(..., 0, ..., "mpu_buffer")는 MPU 버퍼 slot 0을 선택합니다. 출력은 SRAM에서 디바이스에 상주하는 DRAM 텐서로 다시 복사됩니다.