Support setting version in build.toml (#342)

This change adds the `version` option to the `general` section in
`build.toml`. The version will be written to the build metadata and will
be used by `kernels upload` to upload builds to a versioned branch.

Author: danieldk

build2cmake/src/config/mod.rs

@@ -35,6 +35,7 @@ impl Build {
 
 pub struct General {
     pub name: String,
+    pub version: Option<usize>,
 
     /// Hugging Face Hub license identifier.
     pub license: Option<String>,

build2cmake/src/config/v1.rs

@@ -98,6 +98,7 @@ impl TryFrom<Build> for super::Build {
         Ok(Self {
             general: super::General {
                 name: build.general.name,
+                version: None,
                 license: None,
                 backends,
                 hub: None,

build2cmake/src/config/v2.rs

@@ -163,6 +163,7 @@ impl General {
 
         super::General {
             name: general.name,
+            version: None,
             license: None,
             backends,
             cuda,

build2cmake/src/config/v3.rs

@@ -21,6 +21,8 @@ pub struct Build {
 pub struct General {
     pub name: String,
 
+    pub version: Option<usize>,
+
     pub license: Option<String>,
 
     pub backends: Vec<Backend>,
@@ -143,6 +145,7 @@ impl From<General> for super::General {
     fn from(general: General) -> Self {
         Self {
             name: general.name,
+            version: general.version,
             license: general.license,
             backends: general.backends.into_iter().map(Into::into).collect(),
             cuda: general.cuda.map(Into::into),
@@ -296,6 +299,7 @@ impl From<super::General> for General {
     fn from(general: super::General) -> Self {
         Self {
             name: general.name,
+            version: general.version,
             license: general.license,
             backends: general.backends.into_iter().map(Into::into).collect(),
             cuda: general.cuda.map(Into::into),

build2cmake/src/metadata.rs

@@ -3,6 +3,8 @@ use serde::{Deserialize, Serialize};
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(deny_unknown_fields, rename_all = "kebab-case")]
 pub struct Metadata {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub version: Option<usize>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub license: Option<String>,
     pub python_depends: Vec<String>,

build2cmake/src/torch/common.rs

@@ -43,11 +43,12 @@ pub fn write_metadata(backend: Backend, general: &General, file_set: &mut FileSe
         .collect::<Result<Vec<_>>>()?;
 
     let metadata = Metadata {
+        version: general.version,
         license: general.license.clone(),
         python_depends,
     };
 
-    serde_json::to_writer(writer, &metadata)?;
+    serde_json::to_writer_pretty(writer, &metadata)?;
 
     Ok(())
 }

docs/writing-kernels.md

@@ -93,6 +93,10 @@ depends = [ "torch" ]
 
 - `name` (required): the name of the kernel. The Python code for a Torch
   extension must be stored in `torch-ext/<name>`.
+- `version` (int, **experimental**): the major version of the kernel.
+  The version is written to the kernel's `metadata.json` and is used
+  by the `kernels upload` command to upload the kernel to a version
+  branch named `v<version>`.
 - `backends` (required): a list of supported backends. Must be one or
   more of `cpu`, `cuda`, `metal`, `rocm`, or `xpu`.
 - `python-depends` (**experimental**): a list of additional Python dependencies
@@ -124,9 +128,9 @@ options:
 - `include` (optional): include directories relative to the project root.
   Default: `[]`.
 - `maxver` (optional): only build for this Torch version and earlier. Use cautiously, since this option produces
-   non-compliant kernels if the version range does not correspond to the [required variants](build-variants.md).
+  non-compliant kernels if the version range does not correspond to the [required variants](build-variants.md).
 - `minver` (optional): only build for this Torch version and later. Use cautiously, since this option produces
-   non-compliant kernels if the version range does not correspond to the [required variants](build-variants.md).
+  non-compliant kernels if the version range does not correspond to the [required variants](build-variants.md).
 
 ### `kernel.<name>`
 

lib/gen-flake-outputs.nix

@@ -197,12 +197,13 @@ in
             "hub"
             "repo-id"
           ] "kernels-community/${buildToml.general.name}" buildToml;
-          branch = lib.attrByPath [ "general" "hub" "branch" ] "main" buildToml;
+          branch = lib.attrByPath [ "general" "hub" "branch" ] null buildToml;
+          branchOpt = lib.optionalString (branch != null) "--branch ${branch}";
           # `kernels upload` fails when there are no build variants to upload.
           # However, we do not want this command to error out in that case, so
           # only insert the upload command when there is something to upload.
           uploadStr = lib.optionalString (applicableBuildSets != [ ]) ''
-            ${pkgs.python3.pkgs.kernels}/bin/kernels upload --repo-id ${repo_id} --branch ${branch} ${bundle}
+            ${pkgs.python3.pkgs.kernels}/bin/kernels upload --repo-id ${repo_id} ${branchOpt} ${bundle}
           '';
         in
         writeScriptBin "build-and-upload" ''

overlay.nix

@@ -95,8 +95,8 @@ in
           src = final.fetchFromGitHub {
             owner = "huggingface";
             repo = "kernels";
-            tag = "v${version}";
-            hash = "sha256-kTBGje4oMiRqN/m98rvg3r3gqoV1Tg5APleRZbPlziY=";
+            rev = "ff4de4eafdfe5c2346b252cf09c6c412e13f3135";
+            hash = "sha256-hn9PZsGnd2BsBejjDQXVKye8abnS5K9jWga7YVkvXOg=";
           };
         });