Init

README.md

@@ -0,0 +1,57 @@
+# Tenstorrent Software (unofficial, WIP)
+
+## Installing kmod
+
+Add this flake to your configuration flake's inputs and pin its `nixpkgs` to your version.
+
+```
+tt-flake = {
+  url = "git+https://git.ziguana.dev/ziguana/tt-flake";
+  inputs.nixpkgs.follows = "nixpkgs";
+};
+```
+
+
+Enable hugepages and IOMMU passthrough. One 1G page is needed per Grayskull, and two for each Wormhole. Add the kernel module to your configuration.
+
+`flake.nix`:
+
+```
+hostname = nixpkgs.lib.nixosSystem {
+  inherit system;
+  modules = [
+    ...
+    (import ./machines/hostname/configuration.nix)
+    # perhaps this module could be shipped by the flake in the future
+    ({ pkgs, ... }: {
+      boot.extraModulePackages = [ tt-flake.packages.x86_64-linux.kmd ];
+      boot.kernelParams = [ "hugepagesz=1G" "hugepages=2" "iommu=pt" ];
+      boot.kernelModules = [ "tenstorrent" ];
+      services.udev.packages = [ tt-flake.packages.x86_64-linux.udev-rules ];
+    })
+    ...
+```
+
+Reboot and run the tests. Some of the tests may require root.
+
+```
+nix run git+https://git.ziguana.dev/ziguana/tt-flake#kmd-test 
+```
+
+You should see testing output on stdout, and some errors (with a possible stack trace) in dmesg.
+
+`stdout`:
+
+```
+Testing /dev/tenstorrent/0 @ 0000:a8:00.0
+Testing /dev/tenstorrent/1 @ 0000:76:00.0
+```
+
+`dmesg`:
+
+```
+[  173.045092] tenstorrent: pin_user_pages_longterm failed: -14
+[  173.046086] tenstorrent: could only pin 1 of 2 pages
+```
+
+As far as I can tell, these failures are exercised by the tests, and a clean `stdout` means there is no issue.

flake.nix

@@ -0,0 +1,36 @@
+{
+  description = "Tenstorrent software stack.";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs }: let
+    system = "x86_64-linux";
+    pkgs = nixpkgs.legacyPackages.${system};
+    kernel = pkgs.linux_latest;
+    kmd = import ./pkgs/kmd { inherit pkgs kernel; };
+    sfpi = import ./pkgs/sfpi { inherit pkgs; };
+    luwen = import ./pkgs/luwen { inherit pkgs; };
+    common = import ./pkgs/common { inherit pkgs; };
+    flash = import ./pkgs/flash { inherit pkgs; pyluwen = luwen.pyluwen_0_1; };
+    smi = import ./pkgs/smi { inherit pkgs; pyluwen = luwen.pyluwen; tools-common = common; };
+    umd = import ./pkgs/umd { inherit pkgs; };
+  in {
+    packages.${system} = {
+      kmd = kmd.kmd;
+      udev-rules = kmd.udev-rules;
+      kmd-test = kmd.test;
+      sfpi = sfpi.sfpi;
+      tt-gcc = sfpi.tt-gcc;
+      smi = smi;
+      luwen = luwen.luwen;
+      pyluwen = luwen.pyluwen;
+      tools-common = common;
+      flash = flash;
+      umd = umd;
+    };
+
+    defaultPackage.${system} = self.packages.${system}.smi;
+  };
+}

pkgs/common/default.nix

@@ -0,0 +1,25 @@
+{ pkgs }:
+
+with pkgs.python3Packages;
+
+buildPythonPackage rec {
+  pname = "tools-common";
+  version = "main-2024-01-31";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "tenstorrent";
+    repo = "tt-tools-common";
+    rev = "b23ce52352fdf19bf8cd3e3fcea181aa9d2e7dc9";
+    sha256 = "sha256-+BMYCI0+G4zYTI7uyPp+RLyUkKt1fS1WNltnD3xMk2g=";
+  };
+
+  patches = [ ./pyproject.patch ];
+  
+  format = "pyproject";
+
+  propagatedBuildInputs = [ setuptools distro elasticsearch psutil pyyaml rich textual requests ];
+
+  pythonImportsCheck = [
+    "tt_tools_common"
+  ];
+}

pkgs/common/pyproject.patch

@@ -0,0 +1,13 @@
+diff --git a/pyproject.toml b/pyproject.toml
+index 8be9d63..0dd0736 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -29,7 +29,7 @@ dependencies = [
+   'psutil==5.9.6',
+   'pyyaml==6.0.1',
+   'rich==13.7.0',
+-  'textual==0.42.0',
++  'textual>=0.42.0',
+   'requests==2.31.0',
+ ]
+ 

pkgs/flash/default.nix

@@ -0,0 +1,21 @@
+{ pkgs, pyluwen }:
+
+with pkgs.python3Packages;
+
+buildPythonPackage rec {
+  pname = "tt-flash";
+  version = "main-01-31-24";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "tenstorrent";
+    repo = "tt-flash";
+    rev = "09db4103efe0c63adc3ea6e61f19eac7eb06d46f";
+    hash = "sha256-fNAP/XuPdn51TtBEelSjh93NgMiyP1j6RqjnrzX9dc4=";
+  };
+
+  format = "pyproject";
+
+  # patches = [ ./pyproject.patch ./log.patch ];
+
+  propagatedBuildInputs = [ setuptools pyyaml pyluwen tabulate ]; #requests textual black distro elasticsearch jsons pydantic psutil pyyaml pyluwen importlib-resources pkgs.pre-commit tools-common ];
+}

pkgs/kmd/default.nix

@@ -0,0 +1,59 @@
+{ pkgs, kernel }:
+
+let
+  src = pkgs.fetchFromGitHub {
+    owner = "tenstorrent";
+    repo = "tt-kmd";
+    rev = "455b948";
+    hash = "sha256-oJzAPd85Cyd1hcuFPJNyHA/YXX89fTUK5o4a+M4Nu00=";
+  };
+  version = "main-01-31-24";
+in
+{
+  kmd = pkgs.stdenv.mkDerivation {
+    pname = "tt-kmd";
+
+    inherit src version;
+
+    nativeBuildInputs = kernel.moduleBuildDependencies;
+    hardeningDisable = [ "all" ];
+    buildPhase = ''
+      make modules -C ${kernel.dev}/lib/modules/${kernel.modDirVersion}/build M=$(pwd -P)
+    '';
+    installPhase = ''
+      mkdir -p $out/lib/modules/${kernel.modDirVersion}/extra
+      cp tenstorrent.ko $out/lib/modules/${kernel.modDirVersion}/extra/
+    '';
+  };
+
+  udev-rules = pkgs.stdenv.mkDerivation rec {
+    pname = "tenstorrent-udev-rules";
+
+    inherit src version;
+
+    dontUnpack = true;
+
+    installPhase = ''
+      install -Dpm644 $src/udev-50-tenstorrent.rules $out/lib/udev/rules.d/50-tenstorrent.rules
+    '';
+  };
+
+  test = pkgs.stdenv.mkDerivation {
+    pname = "tt-kmd-test";
+
+    inherit src version;
+
+    patches = [ ./missing_header.patch ];
+
+    nativeBuildInputs = [ pkgs.gnumake ];
+
+    buildPhase = ''
+      make -C test
+    '';
+
+    installPhase = ''
+      mkdir -p $out/bin
+      cp test/ttkmd_test $out/bin/tt-kmd-test
+    '';
+  };
+}

pkgs/kmd/missing_header.patch

@@ -0,0 +1,24 @@
+diff --git a/test/dma_buf.cpp b/test/dma_buf.cpp
+index a34bc44..95718a1 100644
+--- a/test/dma_buf.cpp
++++ b/test/dma_buf.cpp
+@@ -4,6 +4,7 @@
+ #include <limits>
+ #include <variant>
+ #include <cerrno>
++#include <cstdint>
+ 
+ #include <sys/ioctl.h>
+ #include <sys/mman.h>
+diff --git a/test/lock.cpp b/test/lock.cpp
+index 941006e..0272126 100644
+--- a/test/lock.cpp
++++ b/test/lock.cpp
+@@ -3,6 +3,7 @@
+ 
+ #include <optional>
+ #include <string>
++#include <cstdint>
+ 
+ #include <errno.h>
+ #include <sys/ioctl.h>

pkgs/luwen/default.nix

@@ -0,0 +1,92 @@
+{ pkgs }:
+
+{
+  luwen = pkgs.rustPlatform.buildRustPackage rec {
+    pname = "luwen";
+    version = "main-2024-01-31";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "tenstorrent";
+      repo = "luwen";
+      rev = "4753a930adb217b296e32f8c682344d929b561bd";
+      sha256 = "sha256-UiTVZZt0ZFwZ6wCTpk+8ZLYjtdSiMFklXoh6bDFZXKQ=";
+    };
+
+    postPatch = ''
+      ln -s ${./Cargo.lock} Cargo.lock
+    '';
+
+    cargoLock.lockFile = ./Cargo.lock;
+    cargoHash = "";
+  };
+
+  pyluwen = pkgs.python3.pkgs.buildPythonPackage rec {
+    pname = "pyluwen";
+    version = "main-2024-01-31";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "tenstorrent";
+      repo = "luwen";
+      rev = "4753a930adb217b296e32f8c682344d929b561bd";
+      sha256 = "sha256-UiTVZZt0ZFwZ6wCTpk+8ZLYjtdSiMFklXoh6bDFZXKQ=";
+    };
+
+    postPatch = ''
+      ln -s ${./Cargo_0_2.lock} Cargo.lock
+    '';
+
+    buildAndTestSubdir = "crates/pyluwen";
+
+    format = "pyproject";
+
+    cargoDeps = pkgs.rustPlatform.fetchCargoTarball {
+      inherit src;
+      name = "${pname}-${version}";
+      hash = "sha256-7FiLEdgZZgsNXHt81tdP+L6rOA1MqlzGz0SkFWvg10I=";
+    };
+
+    nativeBuildInputs = [
+      pkgs.rustPlatform.cargoSetupHook
+      pkgs.rustPlatform.maturinBuildHook
+    ];
+
+    pythonImportsCheck = [
+      "pyluwen"
+    ];
+  };
+
+  pyluwen_0_1 = pkgs.python3.pkgs.buildPythonPackage rec {
+    pname = "pyluwen";
+    version = "v0.1.0";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "tenstorrent";
+      repo = "luwen";
+      rev = "${version}";
+      sha256 = "sha256-MyOzm3dfEkL7MsVzV51DaO+Op3+QhUzsYCTDsvYsvpk=";
+    };
+
+    postPatch = ''
+      ln -s ${./Cargo_0_1.lock} Cargo.lock
+    '';
+
+    buildAndTestSubdir = "crates/pyluwen";
+
+    format = "pyproject";
+
+    cargoDeps = pkgs.rustPlatform.fetchCargoTarball {
+      inherit src postPatch;
+      name = "${pname}-${version}";
+      hash = "sha256-ZXcj/pzQ/tAROdmi2w+AWYBvLSEZFayizxw+BmNDj70=";
+    };
+
+    nativeBuildInputs = [
+      pkgs.rustPlatform.cargoSetupHook
+      pkgs.rustPlatform.maturinBuildHook
+    ];
+
+    pythonImportsCheck = [
+      "pyluwen"
+    ];
+  };
+}

pkgs/luwen/version_check.patch

@@ -0,0 +1,18 @@
+diff --git a/crates/luwen-if/src/chip/grayskull.rs b/crates/luwen-if/src/chip/grayskull.rs
+index 83f9f7a..2f4dc4a 100644
+--- a/crates/luwen-if/src/chip/grayskull.rs
++++ b/crates/luwen-if/src/chip/grayskull.rs
+@@ -462,13 +462,6 @@ impl ChipImpl for Grayskull {
+             }
+         };
+ 
+-        if version <= 0x01030000 {
+-            return Err(crate::error::PlatformError::UnsupportedFwVersion {
+-                version,
+-                required: 0x01040000,
+-            });
+-        }
+-
+         let result = self.arc_msg(ArcMsgOptions {
+             msg: ArcMsg::GetSmbusTelemetryAddr,
+             ..Default::default()

pkgs/sfpi/default.nix

@@ -0,0 +1,30 @@
+{ pkgs }:
+
+rec {
+  sfpi = pkgs.stdenv.mkDerivation {
+    pname = "sfpi";
+    version = "master-01-30-24";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "tenstorrent-metal";
+      repo = "sfpi";
+      rev = "aa4e71d";
+      hash = "sha256-JWSEDx7CCAfuhEhrmcrZunEwWdrsXl71pLJA4Fqme0s=";
+    };
+
+    postPatch = ''
+      ln -s ${tt-gcc} compiler
+    '';
+
+    buildPhase = ''
+      make -C tests all
+    '';
+
+    installPhase = ''
+      mkdir -p $out/compiler/libexec
+      bin/release.sh $out
+    '';
+  };
+
+  tt-gcc = import ./tt-gcc.nix { inherit pkgs; };
+}

pkgs/sfpi/tt-gcc.nix

@@ -0,0 +1,57 @@
+{ pkgs }:
+
+pkgs.stdenv.mkDerivation rec {
+  pname = "tt-gcc";
+  version = "master-01-30-24";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "tenstorrent-metal";
+    repo = "sfpi-tt-gcc";
+    rev = "94a51a7";
+    # this takes a while and we don't need all of them
+    fetchSubmodules = true;
+    leaveDotGit = true;
+    hash = "sha256-VliX4Npw8FqTq3vmdsDFRThXFfDgaTomJ+egCEyhOyU=";
+  };
+
+  nativeBuildInputs = with pkgs; [
+    python3
+    util-linux
+    git
+    cacert
+    autoconf
+    automake
+    curl
+    gawk
+    bison
+    flex
+    texinfo
+    gperf
+    bc
+  ];
+
+  buildInputs = with pkgs; [
+    libmpc
+    mpfr
+    gmp
+    zlib
+    expat
+  ];
+
+  configureFlags = [
+    "-disable-multilib"
+    "-with-abi=ilp32"
+    "-with-arch=rv32i"
+    "--prefix=${placeholder "out"}"
+  ];
+
+  hardeningDisable = [
+    "format"
+  ];
+
+  enableParallelBuilding = true;
+
+  # this is an absolute travesty, but i'm not about to
+  # properly repackage all of riscv-gnu-toolchain
+  __noChroot = true;
+}

pkgs/smi/default.nix

@@ -0,0 +1,21 @@
+{ pkgs, pyluwen, tools-common }:
+
+with pkgs.python3Packages;
+
+buildPythonPackage rec {
+  pname = "tt-smi";
+  version = "main-01-31-24";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "tenstorrent";
+    repo = "tt-smi";
+    rev = "2071978";
+    hash = "sha256-sqwGWeeMBxOyHiVI2GcQ5CyZ8Zaty7FjhkS0C7H7QkM=";
+  };
+
+  format = "pyproject";
+
+  patches = [ ./pyproject.patch ./log.patch ];
+
+  propagatedBuildInputs = [ setuptools requests textual black distro elasticsearch jsons pydantic psutil pyyaml pyluwen importlib-resources pkgs.pre-commit tools-common ];
+}

pkgs/smi/log.patch

@@ -0,0 +1,52 @@
+diff --git a/tt_smi/log.py b/tt_smi/log.py
+index e7ae8fd..083bdb7 100644
+--- a/tt_smi/log.py
++++ b/tt_smi/log.py
+@@ -10,8 +10,8 @@ import base64
+ import inspect
+ import datetime
+ from pathlib import Path
+-from typing import Any, Union, List, TypeVar, Generic
+-from pydantic import BaseModel
++from typing import Any, Union, List, TypeVar, Generic, Optional
++from pydantic import BaseModel, create_model
+ from pydantic.fields import Field
+ 
+ 
+@@ -38,26 +38,24 @@ class Date(datetime.datetime):
+ 
+ 
+ def optional(*fields):
+-    """Decorator function used to modify a pydantic model's fields to all be optional.
+-    Alternatively, you can  also pass the field names that should be made optional as arguments
+-    to the decorator.
+-    Taken from https://github.com/samuelcolvin/pydantic/issues/1223#issuecomment-775363074
+-    """
+-
+-    def dec(_cls):
++    def dec(cls):
++        fields_dict = {}
+         for field in fields:
+-            _cls.__fields__[field].required = False
+-            _cls.__fields__[field].default = None
+-        return _cls
++            field_info = cls.__annotations__.get(field)
++            if field_info is not None:
++                fields_dict[field] = (Optional[field_info], None)
++        OptionalModel = create_model(cls.__name__, **fields_dict)
++        OptionalModel.__module__ = cls.__module__
++
++        return OptionalModel
+ 
+     if fields and inspect.isclass(fields[0]) and issubclass(fields[0], BaseModel):
+         cls = fields[0]
+-        fields = cls.__fields__
++        fields = cls.__annotations__
+         return dec(cls)
+ 
+     return dec
+ 
+-
+ def type_to_mapping(type: Any):
+     if issubclass(type, float):
+         return {"type": "float"}

pkgs/smi/pyproject.patch

@@ -0,0 +1,22 @@
+diff --git a/pyproject.toml b/pyproject.toml
+index a6e73a4..5085274 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -29,14 +29,14 @@ dependencies = [
+   'elasticsearch==8.11.0',
+   'jsons==1.6.3',
+   'linkify-it-py==2.0.2',
+-  'pydantic==1.*',
++  'pydantic>=1.0.0',
+   'psutil==5.9.6',
+   'pyyaml==6.0.1',
+   'tt_tools_common @ git+https://github.com/tenstorrent/tt-tools-common.git',
+   'pyluwen @ git+https://github.com/tenstorrent/luwen.git@v0.1.0#subdirectory=crates/pyluwen',
+   'rich==13.7.0',
+-  'textual==0.42.0',
+-  'pre-commit==3.5.0',
++  'textual>=0.42.0',
++  'pre-commit>=3.5.0',
+   'importlib_resources==6.1.1',
+   'requests==2.31.0',
+ ]

pkgs/umd/default.nix

@@ -0,0 +1,27 @@
+{ pkgs }:
+
+pkgs.stdenv.mkDerivation {
+  pname = "tt-umd";
+  version = "main-2024-02-01";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "tenstorrent-metal";
+    repo = "umd";
+    rev = "341f5b7b299f128faaf2ca446a03298cb781a645";
+    hash = "sha256-jMxhhFWnCjNZZvFiTCeuEHvxvE0+IoaP4NJkr/CDLy8=";
+  };
+
+  patches = [ ./fmt_mystery.patch ./missing_headers.patch ];
+
+  makeFlags = [
+    "DEVICE_CXX=${pkgs.stdenv.cc.targetPrefix}c++"
+    "ARCH_NAME=grayskull"
+  ];
+
+  buildInputs = with pkgs; [ libyamlcpp boost fmt hwloc ];
+
+  installPhase = ''
+    mkdir $out
+    mv build/lib $out
+  '';
+}

pkgs/umd/fmt_mystery.patch

@@ -0,0 +1,25 @@
+diff --git a/device/cpuset_lib.cpp b/device/cpuset_lib.cpp
+index f8d5e03..93e89b7 100644
+--- a/device/cpuset_lib.cpp
++++ b/device/cpuset_lib.cpp
+@@ -9,6 +9,7 @@
+ #include <thread>
+ #include "device/device_api.h"
+ #include <filesystem>
++#include <fmt/std.h>
+ namespace tt {
+ 
+ namespace fs = std::filesystem;
+diff --git a/device/tt_silicon_driver.cpp b/device/tt_silicon_driver.cpp
+index fb243ed..c8950ae 100644
+--- a/device/tt_silicon_driver.cpp
++++ b/device/tt_silicon_driver.cpp
+@@ -4342,7 +4342,7 @@ void tt_SiliconDevice::set_power_state(tt_DevicePowerState device_state) {
+             set_pcie_power_state(device_state);
+         } else {
+             int exit_code = set_remote_power_state(chip, device_state);
+-            log_assert(exit_code == 0, "Failed to set power state to {} with exit code: {}", device_state, exit_code);
++            log_assert(exit_code == 0, "Failed to set power state with exit code: {}", exit_code);
+         }
+     }
+ }

pkgs/umd/missing_headers.patch

@@ -0,0 +1,24 @@
+diff --git a/device/tt_cluster_descriptor.h b/device/tt_cluster_descriptor.h
+index c5c066e..84f17ce 100644
+--- a/device/tt_cluster_descriptor.h
++++ b/device/tt_cluster_descriptor.h
+@@ -11,6 +11,7 @@
+ 
+ #include <unordered_map>
+ #include <unordered_set>
++#include <cstdint>
+ #include <set>
+ #include <map>
+ #include <tuple>
+diff --git a/device/tt_soc_descriptor.cpp b/device/tt_soc_descriptor.cpp
+index adf77c2..3813eab 100644
+--- a/device/tt_soc_descriptor.cpp
++++ b/device/tt_soc_descriptor.cpp
+@@ -5,6 +5,7 @@
+ #include "tt_soc_descriptor.h"
+ 
+ #include <assert.h>
++#include <cstdint>
+ #include <fstream>
+ #include <iostream>
+ #include <regex>