From 5a859224c1b3e3f9d8b34790039d9f00b7c60850 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Wed, 21 Jan 2026 11:24:12 -0600 Subject: [PATCH 01/32] Update the version upgrade tool --- tools/upgrade-version/src/main.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 212f82e9fc4..66b1c40a207 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -184,7 +184,18 @@ fn main() -> anyhow::Result<()> { process_license_file("LICENSE.txt", &full_version); process_license_file("licenses/BSL.txt", &full_version); + println!("$> cargo check"); cmd!("cargo", "check").run().expect("Cargo check failed!"); + + println!("$> pnpm --dir templates/quickstart-chat-typescript generate"); + cmd!("pnpm", "--dir", "templates/quickstart-chat-typescript", "generate") + .run() + .expect("pnpm generate failed!"); + + println!("$> cargo test -p spacetimedb-codegen"); + cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; + println!("$> cargo insta review"); + cmd!("cargo", "insta", "review").run()?; } if matches.get_flag("typescript") || matches.get_flag("all") { From beca5be7484efdbdc2381de50bdfb2a306876478 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Wed, 21 Jan 2026 11:31:12 -0600 Subject: [PATCH 02/32] Allow the user to just accept the insta snapshot change --- tools/upgrade-version/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 66b1c40a207..e163e8ba2ee 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -192,8 +192,11 @@ fn main() -> anyhow::Result<()> { .run() .expect("pnpm generate failed!"); + // This gives an error because the snapshot has been updated println!("$> cargo test -p spacetimedb-codegen"); - cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; + let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); + + // Let the user approve the snapshot change println!("$> cargo insta review"); cmd!("cargo", "insta", "review").run()?; } From 9d040e7c2d35dac9bf1b925df2195693e30b461b Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 05:13:10 -0600 Subject: [PATCH 03/32] Fix CI check --- .github/workflows/upgrade-version-check.yml | 2 +- tools/upgrade-version/src/main.rs | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/upgrade-version-check.yml b/.github/workflows/upgrade-version-check.yml index 41f8f178573..3fbbb64cf93 100644 --- a/.github/workflows/upgrade-version-check.yml +++ b/.github/workflows/upgrade-version-check.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Verify that upgrade-version still works - run: cargo bump-versions 123.456.789 --rust-and-cli --csharp --typescript + run: cargo bump-versions 123.456.789 --rust-and-cli --csharp --typescript --accept-snapshots - name: Show diff run: git diff HEAD diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index e163e8ba2ee..dc9f2269bb2 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -127,6 +127,12 @@ fn main() -> anyhow::Result<()> { .help("Update all targets (equivalent to --typescript --rust-and-cli --csharp)") .conflicts_with_all(["typescript", "rust-and-cli", "csharp"]), ) + .arg( + Arg::new("accept-snapshots") + .long("accept-snapshots") + .action(clap::ArgAction::SetTrue) + .help("If there are snapshots to review automatically accept them all."), + ) .group( ArgGroup::new("update-targets") .args(["all", "typescript", "rust-and-cli", "csharp"]) @@ -196,9 +202,15 @@ fn main() -> anyhow::Result<()> { println!("$> cargo test -p spacetimedb-codegen"); let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); - // Let the user approve the snapshot change - println!("$> cargo insta review"); - cmd!("cargo", "insta", "review").run()?; + if matches.get_flag("accept-snapshots") { + // automatically accept the snapshot + println!("$> cargo insta accept"); + cmd!("cargo", "insta", "accept").run()?; + } else { + // Let the user manually approve the snapshot change + println!("$> cargo insta review"); + cmd!("cargo", "insta", "review").run()?; + } } if matches.get_flag("typescript") || matches.get_flag("all") { From 7d6ea9aff226ab4784bd3d9f8759aca6fa382582 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 05:33:37 -0600 Subject: [PATCH 04/32] Run pnpm build as well --- tools/upgrade-version/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index dc9f2269bb2..5bd2a70ee22 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -193,6 +193,11 @@ fn main() -> anyhow::Result<()> { println!("$> cargo check"); cmd!("cargo", "check").run().expect("Cargo check failed!"); + println!("$> pnpm run build"); + cmd!("pnpm", "run", "build") + .run() + .expect("pnpm run build failed!"); + println!("$> pnpm --dir templates/quickstart-chat-typescript generate"); cmd!("pnpm", "--dir", "templates/quickstart-chat-typescript", "generate") .run() From 179ef5aaf4b0fb7878c370bc0c7c851fe37274b5 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 14:23:21 -0600 Subject: [PATCH 05/32] Also pnpm install --- tools/upgrade-version/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 79377105bca..3d2f05e481b 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -193,6 +193,11 @@ fn main() -> anyhow::Result<()> { println!("$> cargo check"); cmd!("cargo", "check").run().expect("Cargo check failed!"); + println!("$> pnpm install"); + cmd!("pnpm", "install") + .run() + .expect("pnpm run build failed!"); + println!("$> pnpm run build"); cmd!("pnpm", "run", "build") .run() From 47ce799e98a4b44394db2a3d77862eada3415b0d Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 14:35:16 -0600 Subject: [PATCH 06/32] Small fix from template rename --- tools/upgrade-version/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 3d2f05e481b..6cafaeba9be 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -203,8 +203,8 @@ fn main() -> anyhow::Result<()> { .run() .expect("pnpm run build failed!"); - println!("$> pnpm --dir templates/quickstart-chat-typescript generate"); - cmd!("pnpm", "--dir", "templates/quickstart-chat-typescript", "generate") + println!("$> pnpm --dir templates/chat-react-ts generate"); + cmd!("pnpm", "--dir", "templates/chat-react-ts", "generate") .run() .expect("pnpm generate failed!"); From 89ce89caf090449db2ed83ed1ac2a6235e450a7b Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 14:37:53 -0600 Subject: [PATCH 07/32] Install npm/pnpm in workflow --- .github/workflows/upgrade-version-check.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/upgrade-version-check.yml b/.github/workflows/upgrade-version-check.yml index 3fbbb64cf93..849657047d4 100644 --- a/.github/workflows/upgrade-version-check.yml +++ b/.github/workflows/upgrade-version-check.yml @@ -15,6 +15,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + # pnpm is required for regenerating the typescript bindings + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: pnpm/action-setup@v4 + with: + run_install: true - name: Verify that upgrade-version still works run: cargo bump-versions 123.456.789 --rust-and-cli --csharp --typescript --accept-snapshots - name: Show diff From be961a0307af5c823e03e0565673f6c1cf897aa4 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 14:42:26 -0600 Subject: [PATCH 08/32] Upgrade npm version --- .github/workflows/upgrade-version-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-version-check.yml b/.github/workflows/upgrade-version-check.yml index 849657047d4..f6abff297e5 100644 --- a/.github/workflows/upgrade-version-check.yml +++ b/.github/workflows/upgrade-version-check.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 20 - uses: pnpm/action-setup@v4 with: run_install: true From 6a8b3001c5a09bc03388a17057082059057574af Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 14:47:14 -0600 Subject: [PATCH 09/32] Test after accepting new snapshot --- tools/upgrade-version/src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 6cafaeba9be..f969d6d0b87 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -208,10 +208,6 @@ fn main() -> anyhow::Result<()> { .run() .expect("pnpm generate failed!"); - // This gives an error because the snapshot has been updated - println!("$> cargo test -p spacetimedb-codegen"); - let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); - if matches.get_flag("accept-snapshots") { // automatically accept the snapshot println!("$> cargo insta accept"); @@ -221,6 +217,10 @@ fn main() -> anyhow::Result<()> { println!("$> cargo insta review"); cmd!("cargo", "insta", "review").run()?; } + + // This gives an error because the snapshot has been updated + println!("$> cargo test -p spacetimedb-codegen"); + let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); } if matches.get_flag("typescript") || matches.get_flag("all") { From 3ade9ed84796c41a1ef3a3053b6c3ab44c4c5fe7 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 14:51:31 -0600 Subject: [PATCH 10/32] cargo fmt --- tools/upgrade-version/src/main.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index f969d6d0b87..065f1c74032 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -194,33 +194,40 @@ fn main() -> anyhow::Result<()> { cmd!("cargo", "check").run().expect("Cargo check failed!"); println!("$> pnpm install"); - cmd!("pnpm", "install") - .run() - .expect("pnpm run build failed!"); + cmd!("pnpm", "install").run().expect("pnpm run build failed!"); println!("$> pnpm run build"); - cmd!("pnpm", "run", "build") - .run() - .expect("pnpm run build failed!"); - + cmd!("pnpm", "run", "build").run().expect("pnpm run build failed!"); + println!("$> pnpm --dir templates/chat-react-ts generate"); cmd!("pnpm", "--dir", "templates/chat-react-ts", "generate") .run() .expect("pnpm generate failed!"); + println!("$> cargo install cargo-insta"); + cmd!("cargo", "install", "cargo-insta") + .run() + .expect("cargo install cargo-insta failed!"); + if matches.get_flag("accept-snapshots") { // automatically accept the snapshot println!("$> cargo insta accept"); - cmd!("cargo", "insta", "accept").run()?; + cmd!("cargo", "insta", "accept") + .run() + .expect("cargo insta accept failed!"); } else { // Let the user manually approve the snapshot change println!("$> cargo insta review"); - cmd!("cargo", "insta", "review").run()?; + cmd!("cargo", "insta", "review") + .run() + .expect("cargo insta review failed!"); } // This gives an error because the snapshot has been updated println!("$> cargo test -p spacetimedb-codegen"); - let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); + cmd!("cargo", "test", "-p", "spacetimedb-codegen") + .run() + .expect("cargo test -p spacetimedb-codegen failed!"); } if matches.get_flag("typescript") || matches.get_flag("all") { From 43504e41d1af232c91c97eda3779df62726558f7 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 15:06:22 -0600 Subject: [PATCH 11/32] One more fix --- tools/upgrade-version/src/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 065f1c74032..bdc82ef8478 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -209,6 +209,10 @@ fn main() -> anyhow::Result<()> { .run() .expect("cargo install cargo-insta failed!"); + // This gives an error because the snapshot will be updated during the test + println!("$> cargo test -p spacetimedb-codegen (expected to fail)"); + let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); + if matches.get_flag("accept-snapshots") { // automatically accept the snapshot println!("$> cargo insta accept"); @@ -223,7 +227,7 @@ fn main() -> anyhow::Result<()> { .expect("cargo insta review failed!"); } - // This gives an error because the snapshot has been updated + // This should now pass println!("$> cargo test -p spacetimedb-codegen"); cmd!("cargo", "test", "-p", "spacetimedb-codegen") .run() From 2804fdc8fd2add4ee70558ead99fc18321f3c292 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 15:10:22 -0600 Subject: [PATCH 12/32] cargo fmt --- tools/upgrade-version/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index bdc82ef8478..876ad503e8f 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -212,7 +212,7 @@ fn main() -> anyhow::Result<()> { // This gives an error because the snapshot will be updated during the test println!("$> cargo test -p spacetimedb-codegen (expected to fail)"); let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); - + if matches.get_flag("accept-snapshots") { // automatically accept the snapshot println!("$> cargo insta accept"); From e44fa54236cb52dae8173dce82a3ec5109e512b8 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 19:28:30 -0600 Subject: [PATCH 13/32] Small fix for CI --- tools/upgrade-version/src/main.rs | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 876ad503e8f..b1b7247631e 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -209,23 +209,22 @@ fn main() -> anyhow::Result<()> { .run() .expect("cargo install cargo-insta failed!"); - // This gives an error because the snapshot will be updated during the test - println!("$> cargo test -p spacetimedb-codegen (expected to fail)"); - let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen").run(); - - if matches.get_flag("accept-snapshots") { - // automatically accept the snapshot - println!("$> cargo insta accept"); - cmd!("cargo", "insta", "accept") - .run() - .expect("cargo insta accept failed!"); + println!("$> INSTA_UPDATE=always cargo test -p spacetimedb-codegen --test codegen"); + cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") + .env("INSTA_UPDATE", "always") + .run() + .expect("cargo test -p spacetimedb-codegen --test codegen (INSTA_UPDATE=always) failed!"); + + + let insta_command = if matches.get_flag("accept-snapshots") { + "accept" } else { - // Let the user manually approve the snapshot change - println!("$> cargo insta review"); - cmd!("cargo", "insta", "review") - .run() - .expect("cargo insta review failed!"); - } + "review" + }; + println!("$> cargo insta {}", insta_command); + cmd!("cargo", "insta", insta_command) + .run() + .expect(format!("cargo insta {} failed!", insta_command).as_str()); // This should now pass println!("$> cargo test -p spacetimedb-codegen"); From ac9b4946a05bf04d6be07165c49fd7ee95f92d22 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 19:42:02 -0600 Subject: [PATCH 14/32] Cleanup --- tools/upgrade-version/src/main.rs | 51 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index b1b7247631e..3165894daed 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -204,33 +204,32 @@ fn main() -> anyhow::Result<()> { .run() .expect("pnpm generate failed!"); - println!("$> cargo install cargo-insta"); - cmd!("cargo", "install", "cargo-insta") - .run() - .expect("cargo install cargo-insta failed!"); - - println!("$> INSTA_UPDATE=always cargo test -p spacetimedb-codegen --test codegen"); - cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") - .env("INSTA_UPDATE", "always") - .run() - .expect("cargo test -p spacetimedb-codegen --test codegen (INSTA_UPDATE=always) failed!"); - - - let insta_command = if matches.get_flag("accept-snapshots") { - "accept" + if matches.get_flag("accept_snapshots") { + // Generate and auto-accept snapshots + println!("$> INSTA_UPDATE=always cargo test -p spacetimedb-codegen --test codegen"); + cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") + .env("INSTA_UPDATE", "always") + .run() + .expect("cargo test -p spacetimedb-codegen --test codegen (INSTA_UPDATE=always) failed!"); } else { - "review" - }; - println!("$> cargo insta {}", insta_command); - cmd!("cargo", "insta", insta_command) - .run() - .expect(format!("cargo insta {} failed!", insta_command).as_str()); - - // This should now pass - println!("$> cargo test -p spacetimedb-codegen"); - cmd!("cargo", "test", "-p", "spacetimedb-codegen") - .run() - .expect("cargo test -p spacetimedb-codegen failed!"); + // Initial test - this will generate snapshots + println!("$> cargo test -p spacetimedb-codegen --test codegen"); + cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") + .run() + .expect("cargo test -p spacetimedb-codegen --test codegen failed!"); + + // Review the new snapshots + println!("$> cargo insta review"); + cmd!("cargo", "insta", "review") + .run() + .expect("cargo insta review failed!"); + + // Test again now that the user has had a chance to accept the snapshots + println!("$> cargo test -p spacetimedb-codegen --test codegen"); + cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") + .run() + .expect("cargo test -p spacetimedb-codegen --test codegen failed!"); + } } if matches.get_flag("typescript") || matches.get_flag("all") { From ea0b07c52ff4fbd7287ba00ecfe81a7e804143ba Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 19:53:59 -0600 Subject: [PATCH 15/32] Small bug fix --- tools/upgrade-version/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 3165894daed..11ddeda256d 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -204,7 +204,7 @@ fn main() -> anyhow::Result<()> { .run() .expect("pnpm generate failed!"); - if matches.get_flag("accept_snapshots") { + if matches.get_flag("accept-snapshots") { // Generate and auto-accept snapshots println!("$> INSTA_UPDATE=always cargo test -p spacetimedb-codegen --test codegen"); cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") From 8009ae912e1b0496e7ace3896fef9fd8ac535657 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 20:04:13 -0600 Subject: [PATCH 16/32] Install cargo insta when a review is required --- tools/upgrade-version/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index 11ddeda256d..c9018f117f0 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -212,6 +212,11 @@ fn main() -> anyhow::Result<()> { .run() .expect("cargo test -p spacetimedb-codegen --test codegen (INSTA_UPDATE=always) failed!"); } else { + println!("$> cargo install cargo-insta"); + cmd!("cargo", "install", "cargo-insta") + .run() + .expect("cargo install cargo-insta failed!"); + // Initial test - this will generate snapshots println!("$> cargo test -p spacetimedb-codegen --test codegen"); cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") From e09974f4bcf525ab825e63db058a04c0799f520c Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 20:11:12 -0600 Subject: [PATCH 17/32] When review is required, the first pass of tests is expected to fail --- tools/upgrade-version/src/main.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index c9018f117f0..c5a28785ecf 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -217,11 +217,10 @@ fn main() -> anyhow::Result<()> { .run() .expect("cargo install cargo-insta failed!"); - // Initial test - this will generate snapshots + // Initial test - this will generate snapshots. This is expected to fail. println!("$> cargo test -p spacetimedb-codegen --test codegen"); - cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") - .run() - .expect("cargo test -p spacetimedb-codegen --test codegen failed!"); + let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") + .run(); // Review the new snapshots println!("$> cargo insta review"); From 4a6228fec73c9fefa0ee32ceebf06c91933453ab Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 20:15:11 -0600 Subject: [PATCH 18/32] Fix lints --- tools/upgrade-version/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/upgrade-version/src/main.rs b/tools/upgrade-version/src/main.rs index c5a28785ecf..c9a02732ce8 100644 --- a/tools/upgrade-version/src/main.rs +++ b/tools/upgrade-version/src/main.rs @@ -219,8 +219,7 @@ fn main() -> anyhow::Result<()> { // Initial test - this will generate snapshots. This is expected to fail. println!("$> cargo test -p spacetimedb-codegen --test codegen"); - let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen") - .run(); + let _ = cmd!("cargo", "test", "-p", "spacetimedb-codegen", "--test", "codegen").run(); // Review the new snapshots println!("$> cargo insta review"); From 030d5fd6c9fbc0087c1cada7e2dbabe37e2b3f0c Mon Sep 17 00:00:00 2001 From: rekhoff Date: Sun, 25 Jan 2026 13:49:29 -0800 Subject: [PATCH 19/32] Broken out explicit `dotnet restore` for dotnet tests --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f70b677ac0c..3daa2eddb0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -571,6 +571,10 @@ jobs: cd sdks/csharp ./tools~/write-nuget-config.sh ../.. + - name: Restore .NET solution + working-directory: sdks/csharp + run: dotnet restore --configfile nuget.config SpacetimeDB.ClientSDK.sln + # Now, setup the Unity tests. - name: Patch spacetimedb dependency in Cargo.toml working-directory: demo/Blackholio/server-rust @@ -693,9 +697,13 @@ jobs: cd sdks/csharp ./tools~/write-nuget-config.sh ../.. + - name: Restore .NET solution + working-directory: sdks/csharp + run: dotnet restore --configfile nuget.config SpacetimeDB.ClientSDK.sln + - name: Run .NET tests working-directory: sdks/csharp - run: dotnet test -warnaserror + run: dotnet test -warnaserror --no-restore - name: Verify C# formatting working-directory: sdks/csharp From 3db9710acf03693aa9be815e55df9a44c658d603 Mon Sep 17 00:00:00 2001 From: rekhoff Date: Sun, 25 Jan 2026 14:06:38 -0800 Subject: [PATCH 20/32] Update `write-nuget-config` with explicit clear and a key for nuget.org --- sdks/csharp/tools~/write-nuget-config.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/csharp/tools~/write-nuget-config.sh b/sdks/csharp/tools~/write-nuget-config.sh index 7849ed08a96..5fddb7c5efe 100755 --- a/sdks/csharp/tools~/write-nuget-config.sh +++ b/sdks/csharp/tools~/write-nuget-config.sh @@ -14,10 +14,12 @@ cat >nuget.config < + + From e4ab0022d66c69d36e3303a1185101712765cba0 Mon Sep 17 00:00:00 2001 From: rekhoff Date: Sun, 25 Jan 2026 14:29:53 -0800 Subject: [PATCH 21/32] Patched CI to copy `sdks/csharp/nuget.config` to repo root --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3daa2eddb0e..acb990d326c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -570,6 +570,7 @@ jobs: # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. cd sdks/csharp ./tools~/write-nuget-config.sh ../.. + cp nuget.config ../.. - name: Restore .NET solution working-directory: sdks/csharp From 8ea2e1481e38587040a20c785e223f574ede39a7 Mon Sep 17 00:00:00 2001 From: rekhoff Date: Sun, 25 Jan 2026 14:50:05 -0800 Subject: [PATCH 22/32] Corrected `NuGet.Config` capitalization and add another NuGet Override --- .github/workflows/ci.yml | 12 ++++++++++-- sdks/csharp/tools~/write-nuget-config.sh | 8 +++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acb990d326c..34d392e46a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,14 @@ jobs: with: global-json-file: global.json + - name: Override NuGet packages + shell: bash + run: | + dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime + dotnet pack -c Release crates/bindings-csharp/Runtime + bash sdks/csharp/tools~/write-nuget-config.sh . + cp sdks/csharp/NuGet.Config NuGet.Config + # nodejs and pnpm are required for the typescript quickstart smoketest - name: Set up Node.js uses: actions/setup-node@v4 @@ -570,11 +578,11 @@ jobs: # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - cp nuget.config ../.. + cp NuGet.Config ../.. - name: Restore .NET solution working-directory: sdks/csharp - run: dotnet restore --configfile nuget.config SpacetimeDB.ClientSDK.sln + run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln # Now, setup the Unity tests. - name: Patch spacetimedb dependency in Cargo.toml diff --git a/sdks/csharp/tools~/write-nuget-config.sh b/sdks/csharp/tools~/write-nuget-config.sh index 5fddb7c5efe..cdea515344f 100755 --- a/sdks/csharp/tools~/write-nuget-config.sh +++ b/sdks/csharp/tools~/write-nuget-config.sh @@ -10,7 +10,7 @@ cd .. # available. # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file, # and https://tldp.org/LDP/abs/html/here-docs.html for more info on this bash feature. -cat >nuget.config <NuGet.Config < @@ -38,5 +38,7 @@ cat >nuget.config < EOF -echo "Wrote nuget.config contents:" -cat nuget.config +cp NuGet.Config nuget.config + +echo "Wrote NuGet.Config contents:" +cat NuGet.Config From 773c450002df0b3f8e3ed6f5542435719a6f16ba Mon Sep 17 00:00:00 2001 From: rekhoff Date: Sun, 25 Jan 2026 16:19:12 -0800 Subject: [PATCH 23/32] Corrected another NuGet..Config casing, added discrete context for NuGet.Configs --- .github/workflows/ci.yml | 7 +++--- sdks/csharp/tools~/write-nuget-config.sh | 30 ++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34d392e46a1..46627105cc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,8 @@ jobs: run: | dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime dotnet pack -c Release crates/bindings-csharp/Runtime - bash sdks/csharp/tools~/write-nuget-config.sh . - cp sdks/csharp/NuGet.Config NuGet.Config + cd sdks/csharp + ./tools~/write-nuget-config.sh ../.. # nodejs and pnpm are required for the typescript quickstart smoketest - name: Set up Node.js @@ -578,7 +578,6 @@ jobs: # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - cp NuGet.Config ../.. - name: Restore .NET solution working-directory: sdks/csharp @@ -708,7 +707,7 @@ jobs: - name: Restore .NET solution working-directory: sdks/csharp - run: dotnet restore --configfile nuget.config SpacetimeDB.ClientSDK.sln + run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln - name: Run .NET tests working-directory: sdks/csharp diff --git a/sdks/csharp/tools~/write-nuget-config.sh b/sdks/csharp/tools~/write-nuget-config.sh index cdea515344f..aabae5a19e3 100755 --- a/sdks/csharp/tools~/write-nuget-config.sh +++ b/sdks/csharp/tools~/write-nuget-config.sh @@ -38,7 +38,33 @@ cat >NuGet.Config < EOF -cp NuGet.Config nuget.config +cat >"${SPACETIMEDB_REPO_PATH}/NuGet.Config" < + + + + + + + + + + + + + + + + + + + + + + + + +EOF -echo "Wrote NuGet.Config contents:" +echo "Wrote sdks/csharp/NuGet.Config contents:" cat NuGet.Config From f0fd738b6dad4f802330f8c5cdbb1226a7681015 Mon Sep 17 00:00:00 2001 From: rekhoff Date: Sun, 25 Jan 2026 16:57:33 -0800 Subject: [PATCH 24/32] Incorporating similar fixes to the quickstart test --- smoketests/tests/quickstart.py | 38 ++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index 254db8bed04..f9abdc4ab64 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -57,6 +57,18 @@ def load_nuget_config(p: Path): return xmltodict.parse(f.read(), force_list=["add", "packageSource", "package"]) return {} + +def _nuget_config_path(project_dir: Path) -> Path: + p_upper = project_dir / "NuGet.Config" + if p_upper.exists(): + return p_upper + + p_lower = project_dir / "nuget.config" + if p_lower.exists(): + return p_lower + + return p_upper + def save_nuget_config(p: Path, doc: dict): # Write back (pretty, UTF-8, no BOM) xml = xmltodict.unparse(doc, pretty=True) @@ -66,8 +78,13 @@ def add_source(doc: dict, *, key: str, path: str) -> None: cfg = doc.setdefault("configuration", {}) sources = cfg.setdefault("packageSources", {}) source_entries = sources.setdefault("add", []) - source = {"@key": key, "@value": path} - source_entries.append(source) + + for source in source_entries: + if source.get("@key") == key: + source["@value"] = str(path) + return + + source_entries.append({"@key": key, "@value": str(path)}) def add_mapping(doc: dict, *, key: str, pattern: str) -> None: cfg = doc.setdefault("configuration", {}) @@ -90,12 +107,25 @@ def add_mapping(doc: dict, *, key: str, pattern: str) -> None: def override_nuget_package(*, project_dir: Path, package: str, source_dir: Path, build_subdir: str): """Override nuget config to use a local NuGet package on a .NET project""" # Make sure the local package is built - run_cmd("dotnet", "pack", cwd=source_dir) + repo_nuget_config = STDB_DIR / "NuGet.Config" + if repo_nuget_config.exists(): + run_cmd( + "dotnet", + "restore", + "--configfile", + str(repo_nuget_config), + cwd=source_dir, + capture_stderr=True, + ) + run_cmd("dotnet", "pack", "-c", "Release", "--no-restore", cwd=source_dir) + else: + run_cmd("dotnet", "pack", "-c", "Release", cwd=source_dir) - p = Path(project_dir) / "nuget.config" + p = _nuget_config_path(Path(project_dir)) doc = load_nuget_config(p) add_source(doc, key=package, path=source_dir/build_subdir) add_mapping(doc, key=package, pattern=package) + add_source(doc, key="nuget.org", path="https://api.nuget.org/v3/index.json") # Fallback for other packages add_mapping(doc, key="nuget.org", pattern="*") save_nuget_config(p, doc) From 0de5e4df698ea0b23b7fce3a78f18a8d24c7107e Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Sat, 24 Jan 2026 20:22:31 -0600 Subject: [PATCH 25/32] Version upgrade v1.12.0 --- Cargo.lock | 248 +++++++++--------- Cargo.toml | 70 ++--- LICENSE.txt | 4 +- .../BSATN.Codegen/BSATN.Codegen.csproj | 2 +- .../BSATN.Runtime/BSATN.Runtime.csproj | 2 +- crates/bindings-csharp/Codegen/Codegen.csproj | 2 +- crates/bindings-csharp/Runtime/Runtime.csproj | 2 +- crates/bindings-typescript/package.json | 2 +- .../project/typescript/package._json | 2 +- .../codegen__codegen_typescript.snap | 2 +- .../server-csharp/StdbModule.csproj | 2 +- licenses/BSL.txt | 4 +- sdks/csharp/SpacetimeDB.ClientSDK.csproj | 6 +- .../regression-tests/server/StdbModule.csproj | 2 +- sdks/csharp/package.json | 2 +- .../basic-cs/spacetimedb/StdbModule.csproj | 2 +- templates/basic-rs/Cargo.toml | 2 +- templates/basic-rs/spacetimedb/Cargo.toml | 2 +- .../spacetimedb/StdbModule.csproj | 2 +- .../src/module_bindings/index.ts | 4 +- 20 files changed, 182 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79823fbcf32..2a78a7f8670 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,7 +449,7 @@ name = "benchmarks-module" version = "0.1.0" dependencies = [ "anyhow", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -1103,7 +1103,7 @@ dependencies = [ [[package]] name = "connect_disconnect_client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "spacetimedb-sdk", @@ -3545,7 +3545,7 @@ name = "keynote-benchmarks" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -3931,7 +3931,7 @@ version = "0.0.0" dependencies = [ "anyhow", "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -5099,7 +5099,7 @@ name = "perf-test-module" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -5507,12 +5507,12 @@ dependencies = [ [[package]] name = "procedure-client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "env_logger 0.10.2", "serde_json", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-sdk", "test-counter", ] @@ -5716,7 +5716,7 @@ name = "quickstart-chat-module" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -6813,7 +6813,7 @@ dependencies = [ "anyhow", "log", "paste", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -6823,7 +6823,7 @@ dependencies = [ "anyhow", "log", "paste", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -6833,12 +6833,12 @@ dependencies = [ "anyhow", "log", "paste", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] name = "sdk-unreal-test-harness" -version = "1.11.3" +version = "1.12.0" dependencies = [ "serial_test", "spacetimedb-testing", @@ -7284,7 +7284,7 @@ name = "spacetime-module" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -7307,7 +7307,7 @@ dependencies = [ [[package]] name = "spacetimedb" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytemuck", @@ -7320,29 +7320,29 @@ dependencies = [ "rand 0.8.5", "scoped-tls", "serde_json", - "spacetimedb-bindings-macro 1.11.3", - "spacetimedb-bindings-sys 1.11.3", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", + "spacetimedb-bindings-macro 1.12.0", + "spacetimedb-bindings-sys 1.12.0", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", "spacetimedb-query-builder", "trybuild", ] [[package]] name = "spacetimedb-auth" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "serde", "serde_json", "serde_with", "spacetimedb-jsonwebtoken", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", ] [[package]] name = "spacetimedb-bench" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "anyhow", @@ -7372,11 +7372,11 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-datastore", "spacetimedb-execution", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-query", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-standalone", "spacetimedb-table", @@ -7405,13 +7405,13 @@ dependencies = [ [[package]] name = "spacetimedb-bindings-macro" -version = "1.11.3" +version = "1.12.0" dependencies = [ "heck 0.4.1", "humantime", "proc-macro2", "quote", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "syn 2.0.107", ] @@ -7426,14 +7426,14 @@ dependencies = [ [[package]] name = "spacetimedb-bindings-sys" -version = "1.11.3" +version = "1.12.0" dependencies = [ - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", ] [[package]] name = "spacetimedb-cli" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "assert_cmd", @@ -7481,9 +7481,9 @@ dependencies = [ "spacetimedb-fs-utils", "spacetimedb-guard", "spacetimedb-jsonwebtoken", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-schema", "syntect", "tabled", @@ -7508,7 +7508,7 @@ dependencies = [ [[package]] name = "spacetimedb-client-api" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-stream", @@ -7547,7 +7547,7 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-datastore", "spacetimedb-jsonwebtoken", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", "spacetimedb-schema", "tempfile", @@ -7565,7 +7565,7 @@ dependencies = [ [[package]] name = "spacetimedb-client-api-messages" -version = "1.11.3" +version = "1.12.0" dependencies = [ "bytes", "bytestring", @@ -7579,16 +7579,16 @@ dependencies = [ "serde_json", "serde_with", "smallvec", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "strum", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-codegen" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -7597,15 +7597,15 @@ dependencies = [ "itertools 0.12.1", "regex", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", "spacetimedb-schema", "spacetimedb-testing", ] [[package]] name = "spacetimedb-commitlog" -version = "1.11.3" +version = "1.12.0" dependencies = [ "async-stream", "bitflags 2.10.0", @@ -7626,8 +7626,8 @@ dependencies = [ "spacetimedb-commitlog", "spacetimedb-fs-utils", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "tempfile", "thiserror 1.0.69", "tokio", @@ -7638,7 +7638,7 @@ dependencies = [ [[package]] name = "spacetimedb-core" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "arrayvec", @@ -7721,14 +7721,14 @@ dependencies = [ "spacetimedb-fs-utils", "spacetimedb-jsonwebtoken", "spacetimedb-jwks", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-memory-usage", "spacetimedb-metrics", "spacetimedb-paths", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-query", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-snapshot", "spacetimedb-subscription", @@ -7764,7 +7764,7 @@ dependencies = [ [[package]] name = "spacetimedb-data-structures" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "crossbeam-queue", @@ -7778,7 +7778,7 @@ dependencies = [ [[package]] name = "spacetimedb-datastore" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytes", @@ -7798,11 +7798,11 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-durability", "spacetimedb-execution", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-metrics", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-snapshot", "spacetimedb-table", @@ -7813,7 +7813,7 @@ dependencies = [ [[package]] name = "spacetimedb-durability" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "futures", @@ -7823,7 +7823,7 @@ dependencies = [ "spacetimedb-commitlog", "spacetimedb-fs-utils", "spacetimedb-paths", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "tempfile", "thiserror 1.0.69", "tokio", @@ -7832,22 +7832,22 @@ dependencies = [ [[package]] name = "spacetimedb-execution" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "itertools 0.12.1", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-sql-parser", "spacetimedb-table", ] [[package]] name = "spacetimedb-expr" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bigdecimal", @@ -7855,10 +7855,10 @@ dependencies = [ "derive_more 0.99.20", "ethnum", "pretty_assertions", - "spacetimedb 1.11.3", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb 1.12.0", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-sql-parser", "thiserror 1.0.69", @@ -7866,7 +7866,7 @@ dependencies = [ [[package]] name = "spacetimedb-fs-utils" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "fs2", @@ -7880,7 +7880,7 @@ dependencies = [ [[package]] name = "spacetimedb-guard" -version = "1.11.3" +version = "1.12.0" dependencies = [ "portpicker", "reqwest 0.12.24", @@ -7938,7 +7938,7 @@ dependencies = [ [[package]] name = "spacetimedb-lib" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bitflags 2.10.0", @@ -7957,17 +7957,17 @@ dependencies = [ "ron", "serde", "serde_json", - "spacetimedb-bindings-macro 1.11.3", + "spacetimedb-bindings-macro 1.12.0", "spacetimedb-memory-usage", "spacetimedb-metrics", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-memory-usage" -version = "1.11.3" +version = "1.12.0" dependencies = [ "decorum", "ethnum", @@ -7977,7 +7977,7 @@ dependencies = [ [[package]] name = "spacetimedb-metrics" -version = "1.11.3" +version = "1.12.0" dependencies = [ "arrayvec", "itertools 0.12.1", @@ -7987,7 +7987,7 @@ dependencies = [ [[package]] name = "spacetimedb-paths" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "chrono", @@ -8003,7 +8003,7 @@ dependencies = [ [[package]] name = "spacetimedb-pg" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-trait", @@ -8014,22 +8014,22 @@ dependencies = [ "pgwire", "spacetimedb-client-api", "spacetimedb-client-api-messages", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "thiserror 1.0.69", "tokio", ] [[package]] name = "spacetimedb-physical-plan" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "derive_more 0.99.20", "either", "pretty_assertions", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", "spacetimedb-schema", "spacetimedb-sql-parser", "spacetimedb-table", @@ -8049,7 +8049,7 @@ dependencies = [ [[package]] name = "spacetimedb-primitives" -version = "1.11.3" +version = "1.12.0" dependencies = [ "bitflags 2.10.0", "either", @@ -8062,7 +8062,7 @@ dependencies = [ [[package]] name = "spacetimedb-query" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "itertools 0.12.1", @@ -8070,18 +8070,18 @@ dependencies = [ "spacetimedb-client-api-messages", "spacetimedb-execution", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-sql-parser", "spacetimedb-table", ] [[package]] name = "spacetimedb-query-builder" -version = "1.11.3" +version = "1.12.0" dependencies = [ - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", ] [[package]] @@ -8112,7 +8112,7 @@ dependencies = [ [[package]] name = "spacetimedb-sats" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "anyhow", @@ -8137,17 +8137,17 @@ dependencies = [ "serde_json", "sha3", "smallvec", - "spacetimedb-bindings-macro 1.11.3", + "spacetimedb-bindings-macro 1.12.0", "spacetimedb-memory-usage", "spacetimedb-metrics", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "thiserror 1.0.69", "uuid", ] [[package]] name = "spacetimedb-schema" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "derive_more 0.99.20", @@ -8163,10 +8163,10 @@ dependencies = [ "serial_test", "smallvec", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-memory-usage", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-sql-parser", "spacetimedb-testing", "termcolor", @@ -8177,7 +8177,7 @@ dependencies = [ [[package]] name = "spacetimedb-sdk" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anymap", "base64 0.21.7", @@ -8196,10 +8196,10 @@ dependencies = [ "rand 0.9.2", "spacetimedb-client-api-messages", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-metrics", "spacetimedb-query-builder", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "spacetimedb-testing", "thiserror 1.0.69", "tokio", @@ -8208,7 +8208,7 @@ dependencies = [ [[package]] name = "spacetimedb-snapshot" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "blake3", @@ -8225,10 +8225,10 @@ dependencies = [ "spacetimedb-datastore", "spacetimedb-durability", "spacetimedb-fs-utils", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-table", "tempfile", @@ -8241,17 +8241,17 @@ dependencies = [ [[package]] name = "spacetimedb-sql-parser" -version = "1.11.3" +version = "1.12.0" dependencies = [ "derive_more 0.99.20", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "sqlparser", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-standalone" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-trait", @@ -8276,7 +8276,7 @@ dependencies = [ "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-datastore", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", "spacetimedb-pg", "spacetimedb-schema", @@ -8293,20 +8293,20 @@ dependencies = [ [[package]] name = "spacetimedb-subscription" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "spacetimedb-execution", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-query", ] [[package]] name = "spacetimedb-table" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "blake3", @@ -8324,17 +8324,17 @@ dependencies = [ "rand 0.9.2", "smallvec", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-memory-usage", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-testing" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytes", @@ -8352,7 +8352,7 @@ dependencies = [ "spacetimedb-client-api", "spacetimedb-core", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", "spacetimedb-schema", "spacetimedb-standalone", @@ -8363,7 +8363,7 @@ dependencies = [ [[package]] name = "spacetimedb-update" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytes", @@ -8388,7 +8388,7 @@ dependencies = [ [[package]] name = "spacetimedb-vm" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "arrayvec", @@ -8398,9 +8398,9 @@ dependencies = [ "smallvec", "spacetimedb-data-structures", "spacetimedb-execution", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-table", "tempfile", @@ -8476,7 +8476,7 @@ dependencies = [ [[package]] name = "sqltest" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-trait", @@ -8495,8 +8495,8 @@ dependencies = [ "rust_decimal", "spacetimedb-core", "spacetimedb-datastore", - "spacetimedb-lib 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-vm", "sqllogictest", "sqllogictest-engines", @@ -8891,7 +8891,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "test-client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "env_logger 0.10.2", @@ -8903,7 +8903,7 @@ dependencies = [ [[package]] name = "test-counter" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "spacetimedb-data-structures", @@ -9633,7 +9633,7 @@ version = "0.1.0" dependencies = [ "anyhow", "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -9835,11 +9835,11 @@ dependencies = [ [[package]] name = "view-client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "env_logger 0.10.2", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-sdk", "test-counter", ] @@ -11013,7 +11013,7 @@ dependencies = [ "reqwest 0.12.24", "serde", "serde_json", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", "spacetimedb-guard", "tempfile", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index 3fd9392aca3..9b2a4b29070 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,46 +100,46 @@ inherits = "release" debug = true [workspace.package] -version = "1.11.3" +version = "1.12.0" edition = "2021" # update rust-toolchain.toml too! rust-version = "1.90.0" [workspace.dependencies] -spacetimedb = { path = "crates/bindings", version = "=1.11.3" } -spacetimedb-auth = { path = "crates/auth", version = "=1.11.3" } -spacetimedb-bindings-macro = { path = "crates/bindings-macro", version = "=1.11.3" } -spacetimedb-bindings-sys = { path = "crates/bindings-sys", version = "=1.11.3" } -spacetimedb-cli = { path = "crates/cli", version = "=1.11.3" } -spacetimedb-client-api = { path = "crates/client-api", version = "=1.11.3" } -spacetimedb-client-api-messages = { path = "crates/client-api-messages", version = "=1.11.3" } -spacetimedb-codegen = { path = "crates/codegen", version = "=1.11.3" } -spacetimedb-commitlog = { path = "crates/commitlog", version = "=1.11.3" } -spacetimedb-core = { path = "crates/core", version = "=1.11.3" } -spacetimedb-data-structures = { path = "crates/data-structures", version = "=1.11.3" } -spacetimedb-datastore = { path = "crates/datastore", version = "=1.11.3" } -spacetimedb-durability = { path = "crates/durability", version = "=1.11.3" } -spacetimedb-execution = { path = "crates/execution", version = "=1.11.3" } -spacetimedb-expr = { path = "crates/expr", version = "=1.11.3" } -spacetimedb-guard = { path = "crates/guard", version = "=1.11.3" } -spacetimedb-lib = { path = "crates/lib", default-features = false, version = "=1.11.3" } -spacetimedb-memory-usage = { path = "crates/memory-usage", version = "=1.11.3", default-features = false } -spacetimedb-metrics = { path = "crates/metrics", version = "=1.11.3" } -spacetimedb-paths = { path = "crates/paths", version = "=1.11.3" } -spacetimedb-pg = { path = "crates/pg", version = "=1.11.3" } -spacetimedb-physical-plan = { path = "crates/physical-plan", version = "=1.11.3" } -spacetimedb-primitives = { path = "crates/primitives", version = "=1.11.3" } -spacetimedb-query = { path = "crates/query", version = "=1.11.3" } -spacetimedb-sats = { path = "crates/sats", version = "=1.11.3" } -spacetimedb-schema = { path = "crates/schema", version = "=1.11.3" } -spacetimedb-standalone = { path = "crates/standalone", version = "=1.11.3" } -spacetimedb-sql-parser = { path = "crates/sql-parser", version = "=1.11.3" } -spacetimedb-table = { path = "crates/table", version = "=1.11.3" } -spacetimedb-vm = { path = "crates/vm", version = "=1.11.3" } -spacetimedb-fs-utils = { path = "crates/fs-utils", version = "=1.11.3" } -spacetimedb-snapshot = { path = "crates/snapshot", version = "=1.11.3" } -spacetimedb-subscription = { path = "crates/subscription", version = "=1.11.3" } -spacetimedb-query-builder = { path = "crates/query-builder", version = "=1.11.3" } +spacetimedb = { path = "crates/bindings", version = "=1.12.0" } +spacetimedb-auth = { path = "crates/auth", version = "=1.12.0" } +spacetimedb-bindings-macro = { path = "crates/bindings-macro", version = "=1.12.0" } +spacetimedb-bindings-sys = { path = "crates/bindings-sys", version = "=1.12.0" } +spacetimedb-cli = { path = "crates/cli", version = "=1.12.0" } +spacetimedb-client-api = { path = "crates/client-api", version = "=1.12.0" } +spacetimedb-client-api-messages = { path = "crates/client-api-messages", version = "=1.12.0" } +spacetimedb-codegen = { path = "crates/codegen", version = "=1.12.0" } +spacetimedb-commitlog = { path = "crates/commitlog", version = "=1.12.0" } +spacetimedb-core = { path = "crates/core", version = "=1.12.0" } +spacetimedb-data-structures = { path = "crates/data-structures", version = "=1.12.0" } +spacetimedb-datastore = { path = "crates/datastore", version = "=1.12.0" } +spacetimedb-durability = { path = "crates/durability", version = "=1.12.0" } +spacetimedb-execution = { path = "crates/execution", version = "=1.12.0" } +spacetimedb-expr = { path = "crates/expr", version = "=1.12.0" } +spacetimedb-guard = { path = "crates/guard", version = "=1.12.0" } +spacetimedb-lib = { path = "crates/lib", default-features = false, version = "=1.12.0" } +spacetimedb-memory-usage = { path = "crates/memory-usage", version = "=1.12.0", default-features = false } +spacetimedb-metrics = { path = "crates/metrics", version = "=1.12.0" } +spacetimedb-paths = { path = "crates/paths", version = "=1.12.0" } +spacetimedb-pg = { path = "crates/pg", version = "=1.12.0" } +spacetimedb-physical-plan = { path = "crates/physical-plan", version = "=1.12.0" } +spacetimedb-primitives = { path = "crates/primitives", version = "=1.12.0" } +spacetimedb-query = { path = "crates/query", version = "=1.12.0" } +spacetimedb-sats = { path = "crates/sats", version = "=1.12.0" } +spacetimedb-schema = { path = "crates/schema", version = "=1.12.0" } +spacetimedb-standalone = { path = "crates/standalone", version = "=1.12.0" } +spacetimedb-sql-parser = { path = "crates/sql-parser", version = "=1.12.0" } +spacetimedb-table = { path = "crates/table", version = "=1.12.0" } +spacetimedb-vm = { path = "crates/vm", version = "=1.12.0" } +spacetimedb-fs-utils = { path = "crates/fs-utils", version = "=1.12.0" } +spacetimedb-snapshot = { path = "crates/snapshot", version = "=1.12.0" } +spacetimedb-subscription = { path = "crates/subscription", version = "=1.12.0" } +spacetimedb-query-builder = { path = "crates/query-builder", version = "=1.12.0" } # Prevent `ahash` from pulling in `getrandom` by disabling default features. # Modules use `getrandom02` and we need to prevent an incompatible version diff --git a/LICENSE.txt b/LICENSE.txt index 3e4177f2240..105d78746b8 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -5,7 +5,7 @@ Business Source License 1.1 Parameters Licensor: Clockwork Laboratories, Inc. -Licensed Work: SpacetimeDB 1.11.3 +Licensed Work: SpacetimeDB 1.12.0 The Licensed Work is (c) 2023 Clockwork Laboratories, Inc. @@ -21,7 +21,7 @@ Additional Use Grant: You may make use of the Licensed Work provided your Licensed Work by creating tables whose schemas are controlled by such third parties. -Change Date: 2031-01-14 +Change Date: 2031-01-24 Change License: GNU Affero General Public License v3.0 with a linking exception diff --git a/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj b/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj index 01036e7a28f..48df32eb1dd 100644 --- a/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj +++ b/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj @@ -2,7 +2,7 @@ SpacetimeDB.BSATN.Codegen - 1.11.2 + 1.12.0 SpacetimeDB BSATN Codegen The SpacetimeDB BSATN Codegen implements the Roslyn incremental generators for BSATN serialization/deserialization in C#. diff --git a/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj b/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj index 1bd693c0d9d..38c8c9b1ea3 100644 --- a/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj +++ b/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj @@ -1,7 +1,7 @@ SpacetimeDB.BSATN.Runtime - 1.11.2 + 1.12.0 SpacetimeDB BSATN Runtime The SpacetimeDB BSATN Runtime implements APIs for BSATN serialization/deserialization in C#. true diff --git a/crates/bindings-csharp/Codegen/Codegen.csproj b/crates/bindings-csharp/Codegen/Codegen.csproj index 3618428a00f..a38a9be1b32 100644 --- a/crates/bindings-csharp/Codegen/Codegen.csproj +++ b/crates/bindings-csharp/Codegen/Codegen.csproj @@ -1,7 +1,7 @@ SpacetimeDB.Codegen - 1.11.2 + 1.12.0 SpacetimeDB Module Codegen The SpacetimeDB Codegen implements the Roslyn incremental generators for writing SpacetimeDB modules in C#. diff --git a/crates/bindings-csharp/Runtime/Runtime.csproj b/crates/bindings-csharp/Runtime/Runtime.csproj index 2d4329c965e..268cffcf592 100644 --- a/crates/bindings-csharp/Runtime/Runtime.csproj +++ b/crates/bindings-csharp/Runtime/Runtime.csproj @@ -2,7 +2,7 @@ SpacetimeDB.Runtime - 1.11.2 + 1.12.0 SpacetimeDB Module Runtime The SpacetimeDB Runtime implements the database runtime bindings for writing SpacetimeDB modules in C#. diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index 4b30509038e..29a1540f1e0 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -1,6 +1,6 @@ { "name": "spacetimedb", - "version": "1.11.2", + "version": "1.12.0", "description": "API and ABI bindings for the SpacetimeDB TypeScript module library", "homepage": "https://github.com/clockworklabs/SpacetimeDB#readme", "bugs": { diff --git a/crates/cli/src/subcommands/project/typescript/package._json b/crates/cli/src/subcommands/project/typescript/package._json index 969a85f8dce..40c51e78b41 100644 --- a/crates/cli/src/subcommands/project/typescript/package._json +++ b/crates/cli/src/subcommands/project/typescript/package._json @@ -10,6 +10,6 @@ "author": "", "license": "ISC", "dependencies": { - "spacetimedb": "1.11.*" + "spacetimedb": "1.12.*" } } \ No newline at end of file diff --git a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap index c4fbcde807f..c2dee2a19b0 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap @@ -547,7 +547,7 @@ const proceduresSchema = __procedures( /** The remote SpacetimeDB module schema, both runtime and type information. */ const REMOTE_MODULE = { versionInfo: { - cliVersion: "1.11.3" as const, + cliVersion: "1.12.0" as const, }, tables: tablesSchema.schemaType.tables, reducers: reducersSchema.reducersType.reducers, diff --git a/demo/Blackholio/server-csharp/StdbModule.csproj b/demo/Blackholio/server-csharp/StdbModule.csproj index 9610e3caa40..45e503f262a 100644 --- a/demo/Blackholio/server-csharp/StdbModule.csproj +++ b/demo/Blackholio/server-csharp/StdbModule.csproj @@ -13,7 +13,7 @@ - + diff --git a/licenses/BSL.txt b/licenses/BSL.txt index c2dcfcf84cb..74636fe1f48 100644 --- a/licenses/BSL.txt +++ b/licenses/BSL.txt @@ -5,7 +5,7 @@ Business Source License 1.1 Parameters Licensor: Clockwork Laboratories, Inc. -Licensed Work: SpacetimeDB 1.11.3 +Licensed Work: SpacetimeDB 1.12.0 The Licensed Work is (c) 2023 Clockwork Laboratories, Inc. @@ -21,7 +21,7 @@ Additional Use Grant: You may make use of the Licensed Work provided your Licensed Work by creating tables whose schemas are controlled by such third parties. -Change Date: 2031-01-14 +Change Date: 2031-01-24 Change License: GNU Affero General Public License v3.0 with a linking exception diff --git a/sdks/csharp/SpacetimeDB.ClientSDK.csproj b/sdks/csharp/SpacetimeDB.ClientSDK.csproj index 540213c769d..f2416ae875f 100644 --- a/sdks/csharp/SpacetimeDB.ClientSDK.csproj +++ b/sdks/csharp/SpacetimeDB.ClientSDK.csproj @@ -16,8 +16,8 @@ logo.png README.md https://github.com/clockworklabs/com.clockworklabs.spacetimedbsdk - 1.11.2 - 1.11.2 + 1.12.0 + 1.12.0 $(DefaultItemExcludes);*~/** packages @@ -25,7 +25,7 @@ - + diff --git a/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj b/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj index 6259cd0f3b9..d0056e5c9f4 100644 --- a/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj +++ b/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj @@ -8,7 +8,7 @@ - + diff --git a/sdks/csharp/package.json b/sdks/csharp/package.json index 22d2fcd7b82..7493b06b217 100644 --- a/sdks/csharp/package.json +++ b/sdks/csharp/package.json @@ -1,7 +1,7 @@ { "name": "com.clockworklabs.spacetimedbsdk", "displayName": "SpacetimeDB SDK", - "version": "1.11.2", + "version": "1.12.0", "description": "The SpacetimeDB Client SDK is a software development kit (SDK) designed to interact with and manipulate SpacetimeDB modules..", "keywords": [], "author": { diff --git a/templates/basic-cs/spacetimedb/StdbModule.csproj b/templates/basic-cs/spacetimedb/StdbModule.csproj index 6259cd0f3b9..d0056e5c9f4 100644 --- a/templates/basic-cs/spacetimedb/StdbModule.csproj +++ b/templates/basic-cs/spacetimedb/StdbModule.csproj @@ -8,7 +8,7 @@ - + diff --git a/templates/basic-rs/Cargo.toml b/templates/basic-rs/Cargo.toml index 5d1fabdc702..fe1aca10515 100644 --- a/templates/basic-rs/Cargo.toml +++ b/templates/basic-rs/Cargo.toml @@ -4,4 +4,4 @@ version = "0.1.0" edition = "2021" [dependencies] -spacetimedb-sdk = "1.11.*" +spacetimedb-sdk = "1.12.*" diff --git a/templates/basic-rs/spacetimedb/Cargo.toml b/templates/basic-rs/spacetimedb/Cargo.toml index 271b883365e..67ba62bad82 100644 --- a/templates/basic-rs/spacetimedb/Cargo.toml +++ b/templates/basic-rs/spacetimedb/Cargo.toml @@ -9,5 +9,5 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -spacetimedb = "1.11.*" +spacetimedb = "1.12.*" log = "0.4" diff --git a/templates/chat-console-cs/spacetimedb/StdbModule.csproj b/templates/chat-console-cs/spacetimedb/StdbModule.csproj index fa9208f2da5..0e47bfed155 100644 --- a/templates/chat-console-cs/spacetimedb/StdbModule.csproj +++ b/templates/chat-console-cs/spacetimedb/StdbModule.csproj @@ -14,7 +14,7 @@ - + diff --git a/templates/chat-react-ts/src/module_bindings/index.ts b/templates/chat-react-ts/src/module_bindings/index.ts index 5dc82db57f6..8d00c314360 100644 --- a/templates/chat-react-ts/src/module_bindings/index.ts +++ b/templates/chat-react-ts/src/module_bindings/index.ts @@ -1,7 +1,7 @@ // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. -// This was generated using spacetimedb cli version 1.11.3 (commit f9bca6a8df856d950360b40cbce744fcbffc9a63). +// This was generated using spacetimedb cli version 1.12.0 (commit 4a6228fec73c9fefa0ee32ceebf06c91933453ab). /* eslint-disable */ /* tslint:disable */ @@ -107,7 +107,7 @@ const proceduresSchema = __procedures(); /** The remote SpacetimeDB module schema, both runtime and type information. */ const REMOTE_MODULE = { versionInfo: { - cliVersion: '1.11.3' as const, + cliVersion: '1.12.0' as const, }, tables: tablesSchema.schemaType.tables, reducers: reducersSchema.reducersType.reducers, From d02b889e940683098c09dd8d085b33c73b34fe53 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 25 Jan 2026 02:39:52 +0000 Subject: [PATCH 26/32] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 382 +++--- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1559 +++++++++++++------------ docs/llms/docs-benchmark-summary.json | 60 +- 4 files changed, 1028 insertions(+), 991 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 17b5c26d78c..7ed34de5610 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,251 +4,235 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 36 +- **Total failures analyzed**: 35 --- -# SpacetimeDB Benchmark Failures Analysis +# SpacetimeDB Benchmark Test Failure Analysis -This document analyzes test failures in the SpacetimeDB benchmark organized by language and mode. For each failure, we provide the generated code, the expected code, the error message, and a detailed explanation along with actionable recommendations. +## Summary of Analysis +This document organizes the test failures by language (Rust, C#, etc.) and provides detailed analysis of common issues. Each failure will include comparisons between the LLM-generated and expected code, identify the root causes, and recommend actionable changes to the documentation. -## Rust / rustdoc_json Failures (8 total) +--- -### Compile/Publish Errors (2 failures) +## Rust / rustdoc_json Failures (8 total) -#### t_002_scheduled_table & t_017_scheduled_columns +### Common Issues +1. **Faulty Function Signatures and Types**: Many failures stem from incorrect function signatures and types in declared reducers and tables. + +2. **Missing Expected Traits/Attributes**: LLM-generated code often lacks the required public visibility (e.g., `pub`), causing structure and attribute mismatches. -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; +3. **Incorrect Use of Scheduling**: The scheduling function setup was incorrect in multiple instances, failing to align with the expected API-style. - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } +--- - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)), - }); - } - } +### 1. Compile/Publish Errors - #[reducer] - pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {} - ``` - -2. **The expected code**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } +#### a. t_002_scheduled_table +**LLM Output**: +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} +#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: every_50ms, + scheduled_at: ScheduleAt::RepeatMicros(50_000), }); } - ``` - -3. **The error**: `publish_error: spacetime publish failed (exit=1)` - -4. **Explain the difference**: - - Incorrect `#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]` should use `#[table(name = tick_timer, scheduled(tick))]` - - Use of `ScheduleAt::repeat` is incorrect; it should use a proper time duration constructor. - -5. **Root cause**: The documentation lacks clarity on scheduling syntax and constructors for time intervals in scheduled tasks. - -6. **Recommendation**: Update documentation to emphasize using `scheduled(tick)` and correct constructors for `ScheduleAt` using `Duration::from_millis`. +} +``` + +**Expected (golden)**: +```rust +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) -> Result<(), String> { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()), + }); + Ok(()) +} +``` +**Error**: `publish_error: spacetime publish failed (exit=1)` + +**Difference**: +- Missing `pub` visibility on struct fields. +- Usage of `ScheduleAt::RepeatMicros` instead of `ScheduleAt::Interval(Duration::from_millis(...).into())`. + +**Root Cause**: +The documentation lacks clear examples of field visibility and correct usage of scheduling parameters. + +**Recommendation**: +Update the documentation to include visibility modifiers and examples utilizing `ScheduleAt::Interval`. --- -### Other Failures (6 failures) - -#### t_003_struct_in_table, t_004_insert, t_007_crud, t_011_helper_function, t_016_sum_type_columns - -1. **The generated code** (e.g., for `t_003`): - ```rust - use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType}; - - #[derive(SpacetimeType, Clone)] - pub struct Position { - pub x: i32, - pub y: i32, - } - - #[spacetimedb::table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, - } - ``` +#### b. t_017_scheduled_columns +**Similar to t_002_scheduled_table**. -2. **The expected code**: - ```rust - use spacetimedb::{table, SpacetimeType}; +- **Action Required**: Same recommendations as above. - #[derive(SpacetimeType, Clone, Debug)] - pub struct Position { - pub x: i32, - pub y: i32, - } - - #[table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, - } - ``` - -3. **The error**: `schema_parity: reducers differ - expected [], got [...]` - -4. **Explain the difference**: Missing `pub` for fields in structs which are not public, causing access issues. - -5. **Root cause**: Insufficient detail in documentation about struct visibility and reducing/scheduling attributes. +--- -6. **Recommendation**: Clarify that public fields are required for structs defining database tables. +### 2. Other Failures + +#### a. t_012_spacetime_product_type +**LLM Output**: +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; + +#[derive(SpacetimeType)] +struct Score { + left: i32, + right: i32, +} + +#[table(name = result)] +struct ResultRow { + #[primary_key] + id: i32, + value: Score, +} + +#[reducer] +fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { + ctx.db.result().insert(ResultRow { + id, + value: Score { left, right }, + }); +} +``` + +**Expected (golden)**: +```rust +use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + +#[derive(SpacetimeType, Clone, Debug)] +pub struct Score { + pub left: i32, + pub right: i32, +} + +#[table(name = result)] +pub struct ResultRow { + #[primary_key] + pub id: i32, + pub value: Score, +} + +#[reducer] +pub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { + ctx.db.result().insert(ResultRow { id, value: Score { left, right } }); +} +``` + +**Error**: `no such table: "result"` + +**Difference**: +- Missing `pub` declaration on the struct and fields. + +**Root Cause**: +The need for public access levels on types and fields is not sufficiently covered in the documentation. + +**Recommendation**: +Add clear examples demonstrating the use of `pub` in type definitions. --- -#### Additional Observations: - -- The focus must be on both visibility modifiers and correct API signatures for reducers and tables. -- The need for `Result<(), String>` in reducer functions is missing in many generated snippets. +### Additional Failures +- The patterns and problems arising in these similar failures (t_013, t_015, t_016, t_018) align closely with public visibility issues or missing API attributes. +- **Action Required**: Follow similar recommendations emphasizing public access in documented type structures. --- ## Rust / docs Failures (22 total) -### Other Failures (22 failures) +The issues observed in the Rust documentation failures mirror the previous section, particularly around building, struct declarations, and public visibility. -#### t_000_empty_reducers, t_001_basic_tables, t_002_scheduled_table, t_004_insert - -1. **The generated code** (for `t_000_empty_reducers`): - ```rust - use spacetimedb::ReducerContext; - - #[spacetimedb::reducer] - pub fn empty_reducer_no_args(_ctx: &ReducerContext) {} - ``` - -2. **The expected code**: - ```rust - use spacetimedb::{reducer, ReducerContext}; - - #[reducer] - pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { - Ok(()) - } - ``` +### Example: t_000_empty_reducers +The LLM-generated code lacks return types on various reducers, while the golden example properly uses `Result<(), String>`. -3. **The error**: `schema_parity: describe failed: WARNING: This command is UNSTABLE` - -4. **Explain the difference**: Missing return type `Result<(), String>` for all reducer functions causes the failure. - -5. **Root cause**: Documentation does not clearly specify that all reducer functions must return a `Result`. - -6. **Recommendation**: Update the documentation to explicitly require a `Result` return type for all reducer functions to avoid compilation errors. +**Actionable Insight**: Clearly document the importance of declaring function return types on reducers. --- -## C# / docs Failures (6 total) +## C# / docs Failures (5 total) -### Other Failures (6 failures) +### Common Issues: +1. **Visibility Modifiers**: Many C# failures stem from missing or wrong visibility modifiers. +2. **Attributes and Enums**: Incorrect use or omission of expected attributes (like `[Type]`) was frequent. -#### t_008_index_lookup, t_013_spacetime_sum_type - -1. **The generated code** (for `t_008_index_lookup`): - ```csharp - using SpacetimeDB; +--- - public static partial class Module +### Example: t_013_spacetime_sum_type +**LLM Output**: +```csharp +using SpacetimeDB; +using SpacetimeDB.Types; + +public static partial class Module +{ + [SpacetimeDB.Type] + public partial struct Circle { public int Radius; } + + [SpacetimeDB.Table(Name = "Result", Public = true)] + public partial struct Result { - [SpacetimeDB.Table(Name = "User")] - public partial struct User - { - [SpacetimeDB.PrimaryKey] - public int Id; - public string Name; - public int Age; - public bool Active; - } - - [SpacetimeDB.Reducer] - public static void LookupUserName(ReducerContext ctx, int id) - { - var user = ctx.Db.User.Id.Find(id); - if (user != null) - { - ctx.Db.Result.Insert(new Result - { - Id = user.Id, - Name = user.Name - }); - } - } + [SpacetimeDB.PrimaryKey] public int Id; + public Shape Value; } - ``` - -2. **The expected code**: - ```csharp - using SpacetimeDB; - - public static partial class Module +} +``` + +**Expected (golden)**: +```csharp +using SpacetimeDB; + +public static partial class Module +{ + [Type] public partial struct Circle { public int Radius; } + + [Table(Name = "Result")] + public partial struct Result { - [Table(Name = "User")] - public partial struct User - { - [PrimaryKey] public int Id; - public string Name; - public int Age; - public bool Active; - } - - [Reducer] - public static void LookupUserName(ReducerContext ctx, int id) - { - var u = ctx.Db.User.Id.Find(id); - if (u.HasValue) - { - var row = u.Value; - ctx.Db.Result.Insert(new Result { Id = row.Id, Name = row.Name }); - } - } + [PrimaryKey] public int Id; + public Shape Value; } - ``` +} +``` +**Error**: `publish_error: spacetime build (csharp) failed` -3. **The error**: `publish_error: spacetime build (csharp) failed (exit=1)` +**Difference**: Incorrectly specified `Public = true` attribute in the table declaration; missing the `[Type]` attribute. -4. **Explain the difference**: Use of `user != null` instead of checking `u.HasValue`, which is necessary for nullable types. +**Root Cause**: Lack of explicit examples and standards regarding visibility and attribute application in the documentation. -5. **Root cause**: Lacking examples for nullable types or option types in the given context. - -6. **Recommendation**: Address nullable type usage in the documentation, emphasizing how to correctly check for value presence. +**Recommendation**: Incorporate detailed examples that differentiate between public and default access levels, especially in C# attributes. --- -### Final Thoughts - -A thorough review of generator patterns and failure analysis indicates that clarifying visibility, return types, syntax for scheduling, and handling nullable types are crucial improvements for development efficiency and error avoidance in SpacetimeDB. Documenting common patterns and providing clear guidelines will enhance user experience and reduce test failures. +## Conclusion +The analysis reveals consistent issues to be addressed across various tests. The primary points of failure involve: +- Visibility modifiers. +- Correct API usage of structure and attributes. +- Suggest clarifying the conditions and requirements for successful compilation in the documentation to enhance developer understanding and reduce confusion. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index 6a2bcda9a11..164647c309a 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 22/27 | 74.3% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% ⬆️ +10.0% | -| Rust | rustdoc_json | **total** | 48/61 | **74.8%** ⬆️ +4.5% | +| Rust | rustdoc_json | basics | 26/27 | 91.7% ⬆️ +17.4% | +| Rust | rustdoc_json | schema | 22/34 | 62.0% ⬇️ -13.3% | +| Rust | rustdoc_json | **total** | 48/61 | **78.2%** ⬆️ +3.4% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | -| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | -| C# | docs | basics | 24/27 | 91.7% ⬇️ -8.3% | -| C# | docs | schema | 22/34 | 63.7% ⬇️ -10.0% | -| C# | docs | **total** | 46/61 | **78.9%** ⬇️ -9.1% | +| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | +| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | +| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | +| C# | docs | schema | 22/34 | 63.7% | +| C# | docs | **total** | 49/61 | **83.5%** ⬆️ +4.5% | _Compared against master branch baseline_ -Generated at: 2026-01-23T20:54:53.246Z +Generated at: 2026-01-25T02:39:25.598Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 69aaf946ba3..6470d95191e 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,14 +5,14 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, @@ -35,25 +35,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319507359Z", - "finished_at": "2026-01-23T20:46:21.669318285Z" + "started_at": "2026-01-25T02:31:03.346418371Z", + "finished_at": "2026-01-25T02:31:28.816274281Z" }, "t_001_basic_tables": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -69,25 +69,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.318770844Z", - "finished_at": "2026-01-23T20:46:19.785596995Z" + "started_at": "2026-01-25T02:31:22.050348955Z", + "finished_at": "2026-01-25T02:31:53.323790812Z" }, "t_002_scheduled_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +99,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling zmij v1.0.16\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:44\n |\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling zerocopy v0.8.33\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.16\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling serde_json v1.0.149\n Compiling spacetimedb-lib v1.11.1\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling rand_core v0.6.4\n Compiling itertools v0.12.1\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319658080Z", - "finished_at": "2026-01-23T20:46:56.319486952Z" + "started_at": "2026-01-25T02:31:53.478926950Z", + "finished_at": "2026-01-25T02:32:31.946860665Z" }, "t_003_struct_in_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn insert_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}\n\n#[spacetimedb::reducer]\npub fn set_pos(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n if let Some(mut e) = ctx.db.entity().id().find(id) {\n e.pos.x = x;\n e.pos.y = y;\n ctx.db.entity().id().update(e);\n }\n}", + "passed_tests": 1, + "llm_output": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -125,38 +125,31 @@ "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": false, - "partial": 0.0, + "pass": true, + "partial": 1.0, "notes": { "golden_db": "basics-t-003-struct-in-table-golden", "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": { - "only_golden": [], - "only_llm": [ - "init()", - "insert_entity()", - "set_pos()" - ] - }, - "reducers_equal": false, - "server": "http://127.0.0.1:34477", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319022832Z", - "finished_at": "2026-01-23T20:46:29.838664307Z" + "started_at": "2026-01-25T02:31:53.479198674Z", + "finished_at": "2026-01-25T02:32:17.958288213Z" }, "t_004_insert": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 1, + "passed_tests": 2, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -173,33 +166,44 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } }, "data_parity_insert_user": { - "pass": false, - "partial": 0.0, + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200e9fa53e3a843d1820d588423033ed683af3525a68a4dc3f8578dabd3b437/sql)\n", - "phase": "sql_golden" + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "insert_user", + "server": "http://127.0.0.1:45345" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319345443Z", - "finished_at": "2026-01-23T20:46:19.313225077Z" + "started_at": "2026-01-25T02:31:13.441650390Z", + "finished_at": "2026-01-25T02:31:33.453015538Z" }, "t_005_update": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let new_row = User { id, name, age, active };\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(new_row);\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -223,14 +227,7 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:34477" - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "server": "http://127.0.0.1:45345" } }, "schema_parity": { @@ -241,25 +238,32 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:19.313268675Z", - "finished_at": "2026-01-23T20:46:50.997884926Z" + "started_at": "2026-01-25T02:31:03.346171101Z", + "finished_at": "2026-01-25T02:31:57.978256373Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_006_delete", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -267,13 +271,6 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -282,7 +279,7 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } @@ -295,21 +292,28 @@ "expected": 0, "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:21.669348863Z", - "finished_at": "2026-01-23T20:46:43.139558965Z" + "started_at": "2026-01-25T02:31:14.887913551Z", + "finished_at": "2026-01-25T02:31:53.478880313Z" }, "t_007_crud": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_007_crud", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n let id_to_delete = 2;\n users.id().delete(&id_to_delete);\n}", + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n ctx.db.user().id().update(updated);\n ctx.db.user().id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -326,14 +330,6 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200fecea2f4fd1e842ce9bf311fc7af775ed2f1c09d17c59d34a450538b65ed/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -342,11 +338,25 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:45345" + } + }, "crud_total_count_one": { "pass": true, "partial": 1.0, @@ -358,18 +368,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319932266Z", - "finished_at": "2026-01-23T20:46:41.862776693Z" + "started_at": "2026-01-25T02:31:53.323830823Z", + "finished_at": "2026-01-25T02:32:29.819999491Z" }, "t_008_index_lookup": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&u.id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&u.id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -377,6 +387,22 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { + "index_lookup_projection_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1 + ], + "golden_db": "basics-t-008-index-lookup-golden", + "golden_out": "id | name ----+--------- 1 | \"Alice\"", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "llm_out": "id | name ----+--------- 1 | \"Alice\"", + "query": "SELECT id, name FROM result WHERE id=1", + "reducer": "lookup_user_name", + "server": "http://127.0.0.1:45345" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -385,7 +411,7 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } @@ -396,30 +422,14 @@ "notes": { "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } - }, - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_db": "basics-t-008-index-lookup-golden", - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name", - "server": "http://127.0.0.1:34477" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:20.508322641Z", - "finished_at": "2026-01-23T20:46:56.770679153Z" + "started_at": "2026-01-25T02:31:33.453053061Z", + "finished_at": "2026-01-25T02:32:13.689948121Z" }, "t_009_init": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_009_init", "lang": "rust", "golden_published": true, @@ -451,36 +461,36 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:19.785638677Z", - "finished_at": "2026-01-23T20:46:42.338746730Z" + "started_at": "2026-01-25T02:31:28.430437681Z", + "finished_at": "2026-01-25T02:31:57.789037350Z" }, "t_010_connect": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -503,24 +513,24 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319798861Z", - "finished_at": "2026-01-23T20:46:24.449136396Z" + "started_at": "2026-01-25T02:31:28.816300522Z", + "finished_at": "2026-01-25T02:31:58.802773918Z" }, "t_011_helper_function": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, + "passed_tests": 3, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -529,6 +539,24 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM result WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:45345" + } + }, "helper_func_sum_abs": { "pass": true, "partial": 1.0, @@ -538,14 +566,6 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200a76ada80a129ab1d98b1132742dd63568f85071327226e59b33240f90ed8/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -554,24 +574,24 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319183557Z", - "finished_at": "2026-01-23T20:46:20.508270684Z" + "started_at": "2026-01-25T02:31:23.706005145Z", + "finished_at": "2026-01-25T02:31:53.232845655Z" }, "t_012_spacetime_product_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, + "passed_tests": 2, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", @@ -580,19 +600,6 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true - } - }, "product_type_row_count": { "pass": true, "partial": 1.0, @@ -603,37 +610,40 @@ } }, "product_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c200ae83d4dafbab5553f9122e0ca48ece3f49f7a81788ca7b482fbadb2bfdc6/sql)\n", + "phase": "sql_golden" + } + }, + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:34477" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45345", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:50.997932201Z", - "finished_at": "2026-01-23T20:47:18.590168533Z" + "started_at": "2026-01-25T02:30:25.755441145Z", + "finished_at": "2026-01-25T02:30:56.427439801Z" }, "t_013_spacetime_sum_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -649,7 +659,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } @@ -664,36 +674,27 @@ } }, "sum_type_row_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:34477" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c200393f375639ae660e3acad93b582d176f774b002af25510f5a6e5918dfee9/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:56.770721316Z", - "finished_at": "2026-01-23T20:47:22.846786446Z" + "started_at": "2026-01-25T02:30:25.755726622Z", + "finished_at": "2026-01-25T02:31:02.603517791Z" }, "t_014_elementary_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -701,18 +702,17 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-014-elementary-columns-golden", - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34477" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45345", + "tables_diff": null, + "tables_equal": true } }, "elementary_columns_row_count": { @@ -724,33 +724,34 @@ "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } }, - "schema_parity": { + "elementary_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:45345" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:56.319506066Z", - "finished_at": "2026-01-23T20:47:21.058802558Z" + "started_at": "2026-01-25T02:30:56.427473474Z", + "finished_at": "2026-01-25T02:31:22.050298776Z" }, "t_015_product_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -766,23 +767,17 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } }, "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34477" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c2006e1b25ff28f6ae88b2c072ea7e0bb34a9558acc6b799d903dd85649e3abc/sql)\n", + "phase": "sql_golden" } }, "product_type_columns_row_count": { @@ -796,18 +791,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:56.770961297Z", - "finished_at": "2026-01-23T20:47:32.576819834Z" + "started_at": "2026-01-25T02:30:25.756612791Z", + "finished_at": "2026-01-25T02:31:14.887824445Z" }, "t_016_sum_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let a = Shape::Circle(10);\n let b = Shape::Rectangle(Rect { width: 4, height: 6 });\n ctx.db.drawing().insert(Drawing { id: 1, a, b });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().id().find(1).is_none() {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -815,6 +810,14 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c2001e6bfa410e8407328c63f0e357b56f2983a849414cffa113aa69b61ba8c7/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -823,7 +826,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } @@ -832,32 +835,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002a23ef16d310345e1d4ce4b5d48569b614371f4967976a5819022bbc2662/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c200a4f49580f3c6577e995861d2eede035cb964ba3c499f577b033537d5c75d/sql)\n", "phase": "sql" } - }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c20040bc998fdc9ab3d5a21e438444dc0653e63e1691252800cd646cfc6722b3/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:00.920013549Z", - "finished_at": "2026-01-23T20:47:31.508612717Z" + "started_at": "2026-01-25T02:30:25.755940232Z", + "finished_at": "2026-01-25T02:31:03.164647878Z" }, "t_017_scheduled_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -869,24 +864,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.54\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.16\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling itoa v1.0.17\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling constant_time_eq v0.4.2\n Compiling itertools v0.12.1\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:21:42\n |\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n 20 | #[reducer]\n | ---------- required by a bound introduced by this call\n 21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n20 | #[reducer]\n | ---------- required by a bound introduced by this call\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:43.139606842Z", - "finished_at": "2026-01-23T20:47:30.076456163Z" + "started_at": "2026-01-25T02:30:25.756799192Z", + "finished_at": "2026-01-25T02:31:13.441629467Z" }, "t_018_constraints": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n for row in tbl.iter() {\n tbl.id().delete(&row.id);\n }\n\n tbl.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n tbl.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n\n let mut ids: Vec = Vec::new();\n for row in account.iter() {\n ids.push(row.id);\n }\n for id in ids {\n account.id().delete(&id);\n }\n\n account.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n account.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -894,19 +889,6 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true - } - }, "constraints_seed_two_rows": { "pass": true, "partial": 1.0, @@ -917,33 +899,40 @@ } }, "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20087d08bf31d4bf8fd3f3f6c645f9e2f81779454fb7e9160ec9aa5a5cb3145/sql)\n", + "phase": "sql_golden" + } + }, + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-018-constraints-golden", - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34477" - } - } - }, + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45345", + "tables_diff": null, + "tables_equal": true + } + } + }, "vendor": "openai", - "started_at": "2026-01-23T20:46:41.862819773Z", - "finished_at": "2026-01-23T20:47:07.104266828Z" + "started_at": "2026-01-25T02:30:25.756125204Z", + "finished_at": "2026-01-25T02:31:23.705962716Z" }, "t_019_many_to_many": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_019_many_to_many", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id])),\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure the tables contain exactly the requested rows\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -951,13 +940,22 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "m2m_has_2_20": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, "m2m_has_1_10": { @@ -969,13 +967,13 @@ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "m2m_has_1_20": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } }, "schema_parity": { @@ -986,34 +984,25 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:45345", "tables_diff": null, "tables_equal": true } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:42.338778389Z", - "finished_at": "2026-01-23T20:47:21.898328928Z" + "started_at": "2026-01-25T02:31:02.603568306Z", + "finished_at": "2026-01-25T02:31:59.581434159Z" }, "t_020_ecs": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n let _ = ctx.db.next_position().entity_id().update(next);\n } else {\n let _ = ctx.db.next_position().insert(next);\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n // Clear existing rows for ids 1 and 2 to make seeding idempotent\n for id in [1i32, 2i32] {\n ctx.db.next_position().entity_id().delete(&id);\n ctx.db.velocity().entity_id().delete(&id);\n ctx.db.position().entity_id().delete(&id);\n ctx.db.entity().id().delete(&id);\n }\n\n // Insert entity 1\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db\n .position()\n .insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db\n .velocity()\n .insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n // Insert entity 2\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db\n .position()\n .insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db\n .velocity()\n .insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\nfn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: pos.entity_id, x: nx, y: ny });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1021,32 +1010,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_seed_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", "phase": "sql" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_positions_count": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", "phase": "sql" } }, @@ -1054,32 +1030,45 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", "phase": "sql" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45345", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:24.449168444Z", - "finished_at": "2026-01-23T20:47:00.919947661Z" + "started_at": "2026-01-25T02:30:25.756440656Z", + "finished_at": "2026-01-25T02:31:28.430387499Z" }, "t_021_multi_column_index": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1087,50 +1076,50 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } }, - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "mcindex_lookup_u7_d1": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45345", + "tables_diff": null, + "tables_equal": true } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:29.838705690Z", - "finished_at": "2026-01-23T20:46:56.594909460Z" + "started_at": "2026-01-25T02:30:25.756281600Z", + "finished_at": "2026-01-25T02:31:03.346131676Z" } } } @@ -1170,8 +1159,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.846152158Z", - "finished_at": "2026-01-23T20:48:46.489399789Z" + "started_at": "2026-01-25T02:33:29.683293458Z", + "finished_at": "2026-01-25T02:33:45.980299464Z" }, "t_001_basic_tables": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1181,7 +1170,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb as _;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1199,18 +1188,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.833609179Z", - "finished_at": "2026-01-23T20:48:40.565099319Z" + "started_at": "2026-01-25T02:33:34.613474286Z", + "finished_at": "2026-01-25T02:34:00.375778172Z" }, "t_002_scheduled_table": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_002_scheduled_table", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1218,18 +1207,18 @@ "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n", - "phase": "describe_golden" + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling second-stack v0.3.5\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling log v0.4.29\n Compiling generic-array v0.14.7\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:5:57\n |\n5 | #[spacetimedb::table(name = tick_timer, scheduled(tick, column = scheduled_at))]\n | ^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:14:44\n |\n14 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:36\n |\n18 | ctx.db.tick_timer().try_insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:14:8\n |\n 13 | #[spacetimedb::reducer]\n | ----------------------- required by a bound introduced by this call\n 14 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:14:8\n |\n13 | #[spacetimedb::reducer]\n | ----------------------- required by a bound introduced by this call\n14 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:12\n |\n18 | ctx.db.tick_timer().try_insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 6 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.849575285Z", - "finished_at": "2026-01-23T20:48:55.497529150Z" + "started_at": "2026-01-25T02:34:00.375805875Z", + "finished_at": "2026-01-25T02:34:36.170109978Z" }, "t_003_struct_in_table": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1257,8 +1246,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.836252652Z", - "finished_at": "2026-01-23T20:48:05.833400632Z" + "started_at": "2026-01-25T02:34:00.673539976Z", + "finished_at": "2026-01-25T02:34:19.746956045Z" }, "t_004_insert": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1294,8 +1283,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.842787873Z", - "finished_at": "2026-01-23T20:48:29.038416446Z" + "started_at": "2026-01-25T02:33:31.141746367Z", + "finished_at": "2026-01-25T02:33:56.259096270Z" }, "t_005_update": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1305,7 +1294,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n Ok(())\n } else {\n Err(\"user not found\".to_string())\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -1313,6 +1302,14 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "describe_golden" + } + }, "data_parity_update_user": { "pass": false, "partial": 0.0, @@ -1329,19 +1326,11 @@ "phase": "sql_golden", "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:05.833493117Z", - "finished_at": "2026-01-23T20:48:31.203040278Z" + "started_at": "2026-01-25T02:33:23.731576942Z", + "finished_at": "2026-01-25T02:33:47.840719598Z" }, "t_006_delete": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1351,7 +1340,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -1359,12 +1348,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, "seed_users_row": { @@ -1376,19 +1366,18 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, + "schema_parity": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:31.203078568Z", - "finished_at": "2026-01-23T20:48:50.906207073Z" + "started_at": "2026-01-25T02:33:34.609970882Z", + "finished_at": "2026-01-25T02:34:04.254303613Z" }, "t_007_crud": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1398,7 +1387,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n\n if let Some(mut u1) = ctx.db.user().id().find(1) {\n u1.name = \"Alice2\".into();\n u1.age = 31;\n u1.active = false;\n ctx.db.user().id().update(u1);\n }\n\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1406,30 +1395,21 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, "notes": { "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "crud_total_count_one": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" } }, "crud_row_id1_parity": { @@ -1439,11 +1419,20 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", "phase": "call_reducer_golden" } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.856090158Z", - "finished_at": "2026-01-23T20:48:19.458976831Z" + "started_at": "2026-01-25T02:33:56.259124969Z", + "finished_at": "2026-01-25T02:34:42.423310345Z" }, "t_008_index_lookup": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1453,7 +1442,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name.clone() });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1461,15 +1450,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1485,11 +1465,20 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", "phase": "call_reducer_golden" } + }, + "seed_user_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:29.038498151Z", - "finished_at": "2026-01-23T20:48:56.042485955Z" + "started_at": "2026-01-25T02:33:47.840748081Z", + "finished_at": "2026-01-25T02:34:12.833631543Z" }, "t_009_init": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1499,7 +1488,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1515,15 +1504,6 @@ "phase": "describe_golden" } }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - }, "init_seed_alice": { "pass": true, "partial": 1.0, @@ -1541,11 +1521,20 @@ "expected": 2, "sql": "SELECT COUNT(*) AS n FROM user" } + }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:19.459032083Z", - "finished_at": "2026-01-23T20:48:44.457914124Z" + "started_at": "2026-01-25T02:33:39.734179775Z", + "finished_at": "2026-01-25T02:34:03.009233798Z" }, "t_010_connect": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1555,7 +1544,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1573,8 +1562,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.852844068Z", - "finished_at": "2026-01-23T20:48:51.666864219Z" + "started_at": "2026-01-25T02:33:45.980322799Z", + "finished_at": "2026-01-25T02:34:09.645472034Z" }, "t_011_helper_function": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1600,27 +1589,27 @@ "phase": "describe_golden" } }, - "helper_func_sum_parity": { + "helper_func_sum_abs": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, - "helper_func_sum_abs": { + "helper_func_sum_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.839497474Z", - "finished_at": "2026-01-23T20:48:51.151329754Z" + "started_at": "2026-01-25T02:33:38.451065185Z", + "finished_at": "2026-01-25T02:34:01.189264068Z" }, "t_012_spacetime_product_type": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1630,7 +1619,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n let value = Score { left, right };\n ctx.db.result().insert(ResultRow { id, value });\n}", + "llm_output": "use spacetimedb::{SpacetimeType, Table, ReducerContext};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1638,20 +1627,20 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "product_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "call_reducer_golden" } }, "product_type_row_count": { @@ -1665,8 +1654,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:51.666947087Z", - "finished_at": "2026-01-23T20:49:14.419221949Z" + "started_at": "2026-01-25T02:32:32.186581007Z", + "finished_at": "2026-01-25T02:33:31.141703042Z" }, "t_013_spacetime_sum_type": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1676,7 +1665,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1684,6 +1673,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "call_reducer_golden" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1700,19 +1697,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:56.042516286Z", - "finished_at": "2026-01-23T20:49:19.391193690Z" + "started_at": "2026-01-25T02:32:32.189565224Z", + "finished_at": "2026-01-25T02:33:23.731499392Z" }, "t_014_elementary_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1722,7 +1711,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1730,35 +1719,35 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:55.497603714Z", - "finished_at": "2026-01-23T20:49:21.590560437Z" + "started_at": "2026-01-25T02:33:05.794685418Z", + "finished_at": "2026-01-25T02:33:29.683265539Z" }, "t_015_product_type_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1768,7 +1757,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let existing_ids: Vec = ctx.db.profile().iter().map(|p| p.id).collect();\n for id in existing_ids {\n ctx.db.profile().id().delete(&id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1776,20 +1765,20 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" } }, "product_type_columns_row_count": { @@ -1803,8 +1792,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:49:09.915973191Z", - "finished_at": "2026-01-23T20:49:35.664676341Z" + "started_at": "2026-01-25T02:32:32.208388460Z", + "finished_at": "2026-01-25T02:33:34.609891050Z" }, "t_016_sum_type_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1814,7 +1803,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(&row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1826,7 +1815,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200de67ac503f1e38be9c09680d3df61e49eb3a7d66ef3c6c8c371fb66d4edf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c2002cd10e5fbdc1ac171a6579dc592276b6377c27107e3f7fe80ec794fe502b/sql)\n", "phase": "sql" } }, @@ -1848,8 +1837,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:49:10.455837761Z", - "finished_at": "2026-01-23T20:49:49.430676143Z" + "started_at": "2026-01-25T02:32:32.193184498Z", + "finished_at": "2026-01-25T02:33:20.169991629Z" }, "t_017_scheduled_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1859,7 +1848,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n }\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1867,6 +1856,14 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } + }, "scheduled_seeded_one_row": { "pass": true, "partial": 1.0, @@ -1875,19 +1872,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:51.151415261Z", - "finished_at": "2026-01-23T20:49:22.096623158Z" + "started_at": "2026-01-25T02:32:32.212187627Z", + "finished_at": "2026-01-25T02:33:38.450978070Z" }, "t_018_constraints": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1897,7 +1886,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1905,6 +1894,14 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "describe_golden" + } + }, "constraints_row_parity_after_seed": { "pass": false, "partial": 0.0, @@ -1921,48 +1918,76 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:46.489485052Z", - "finished_at": "2026-01-23T20:49:14.344881271Z" + "started_at": "2026-01-25T02:32:32.196964280Z", + "finished_at": "2026-01-25T02:33:39.734083836Z" }, "t_019_many_to_many": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 4, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + } + }, + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + } + }, + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 502 Bad Gateway: \r\n502 Bad Gateway\r\n\r\n

502 Bad Gateway

\r\n
cloudflare
\r\n\r\n\r\n", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", + "phase": "describe_golden" + } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:49:49.431161113Z", - "finished_at": "2026-01-23T20:49:49.431161113Z" + "started_at": "2026-01-25T02:33:20.170044699Z", + "finished_at": "2026-01-25T02:34:00.673511214Z" }, "t_020_ecs": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1972,7 +1997,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n ctx.db\n .next_position()\n .entity_id()\n .update(NextPosition { entity_id: pos.entity_id, x: nx, y: ny });\n } else {\n let _ = ctx.db.next_position().try_insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1980,27 +2005,27 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", + "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", "phase": "sql" } }, @@ -2008,22 +2033,22 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", - "phase": "sql" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:40.565127071Z", - "finished_at": "2026-01-23T20:49:16.253635323Z" + "started_at": "2026-01-25T02:32:32.204571288Z", + "finished_at": "2026-01-25T02:33:34.540842410Z" }, "t_021_multi_column_index": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -2041,23 +2066,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" - } - }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2075,11 +2083,28 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:44.457949164Z", - "finished_at": "2026-01-23T20:49:10.455801685Z" + "started_at": "2026-01-25T02:32:32.200776398Z", + "finished_at": "2026-01-25T02:33:05.794620033Z" } } } @@ -2300,15 +2325,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:58.516007671Z", - "finished_at": "2026-01-23T20:52:19.852644961Z" + "started_at": "2026-01-25T02:37:44.772691102Z", + "finished_at": "2026-01-25T02:38:08.769282525Z" }, "t_001_basic_tables": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2318,7 +2343,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2334,15 +2359,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.718544439Z", - "finished_at": "2026-01-23T20:51:58.515971496Z" + "started_at": "2026-01-25T02:38:08.769310649Z", + "finished_at": "2026-01-25T02:38:31.184544005Z" }, "t_002_scheduled_table": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2352,7 +2377,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer tickTimer)\n {\n // Scheduled reducer invoked by TickTimer; no-op by specification.\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2368,15 +2393,15 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:02.545010135Z", - "finished_at": "2026-01-23T20:52:47.129183717Z" + "started_at": "2026-01-25T02:38:41.870651012Z", + "finished_at": "2026-01-25T02:39:25.554760208Z" }, "t_003_struct_in_table": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2402,15 +2427,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.721057407Z", - "finished_at": "2026-01-23T20:52:05.996428958Z" + "started_at": "2026-01-25T02:38:51.058027266Z", + "finished_at": "2026-01-25T02:39:10.268923061Z" }, "t_004_insert": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2436,7 +2461,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } @@ -2457,13 +2482,13 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:46795" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.727492764Z", - "finished_at": "2026-01-23T20:52:08.554796558Z" + "started_at": "2026-01-25T02:37:49.346259164Z", + "finished_at": "2026-01-25T02:38:12.549780356Z" }, "t_005_update": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2473,7 +2498,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n\n if (ctx.Db.User.Id.Find(id) != null)\n {\n ctx.Db.User.Id.Update(updated);\n }\n else\n {\n ctx.Db.User.Insert(updated);\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing != null)\n {\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n ctx.Db.User.Id.Update(updated);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2481,19 +2506,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -2510,7 +2522,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:46795" } }, "seed_users_row": { @@ -2519,11 +2531,24 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46795", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:19.852673232Z", - "finished_at": "2026-01-23T20:52:53.389808270Z" + "started_at": "2026-01-25T02:37:35.934324581Z", + "finished_at": "2026-01-25T02:38:02.198161810Z" }, "t_006_delete": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2565,15 +2590,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:51.107488271Z", - "finished_at": "2026-01-23T20:53:13.731066286Z" + "started_at": "2026-01-25T02:38:02.198198168Z", + "finished_at": "2026-01-25T02:38:26.254084597Z" }, "t_007_crud": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2591,20 +2616,6 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "basics-t-007-crud-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "Crud", - "server": "http://127.0.0.1:46871" - } - }, "crud_total_count_one": { "pass": true, "partial": 1.0, @@ -2631,25 +2642,39 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", + "reducer": "Crud", + "server": "http://127.0.0.1:46795" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:08.554828637Z", - "finished_at": "2026-01-23T20:52:51.107450441Z" + "started_at": "2026-01-25T02:38:37.706294669Z", + "finished_at": "2026-01-25T02:39:02.219172424Z" }, "t_008_index_lookup": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_008_index_lookup", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2657,18 +2682,46 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "seed_user_row": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(32,27): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(33,29): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } + }, + "index_lookup_projection_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1 + ], + "golden_db": "basics-t-008-index-lookup-golden", + "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", + "query": "SELECT Id, Name FROM Result WHERE Id=1", + "reducer": "LookupUserName", + "server": "http://127.0.0.1:46795" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46795", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:47.129209751Z", - "finished_at": "2026-01-23T20:53:08.577581818Z" + "started_at": "2026-01-25T02:38:31.184576580Z", + "finished_at": "2026-01-25T02:39:07.239374457Z" }, "t_009_init": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2678,7 +2731,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2686,6 +2739,15 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { + "init_total_two": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2694,42 +2756,33 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:36.884011764Z", - "finished_at": "2026-01-23T20:53:00.062192214Z" + "started_at": "2026-01-25T02:38:16.264482791Z", + "finished_at": "2026-01-25T02:38:41.870615455Z" }, "t_010_connect": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2755,15 +2808,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:05.996450500Z", - "finished_at": "2026-01-23T20:52:36.883985146Z" + "started_at": "2026-01-25T02:38:26.254121165Z", + "finished_at": "2026-01-25T02:38:51.058005185Z" }, "t_011_helper_function": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2773,7 +2826,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b),\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2781,15 +2834,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2798,7 +2842,7 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } @@ -2818,13 +2862,22 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:46795" + } + }, + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.724127650Z", - "finished_at": "2026-01-23T20:52:02.544972253Z" + "started_at": "2026-01-25T02:38:12.549810724Z", + "finished_at": "2026-01-25T02:38:37.706263248Z" }, "t_012_spacetime_product_type": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2850,20 +2903,11 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } }, - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, "product_type_row_parity": { "pass": true, "partial": 1.0, @@ -2879,13 +2923,22 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:46795" + } + }, + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:40.420411949Z", - "finished_at": "2026-01-23T20:54:07.766554713Z" + "started_at": "2026-01-25T02:36:06.745029551Z", + "finished_at": "2026-01-25T02:36:54.125763993Z" }, "t_013_spacetime_sum_type": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2895,7 +2948,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", + "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Module.Circle { Radius = radius })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2913,8 +2966,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:54:05.268678013Z", - "finished_at": "2026-01-23T20:54:43.846818259Z" + "started_at": "2026-01-25T02:36:06.747511706Z", + "finished_at": "2026-01-25T02:36:55.494494204Z" }, "t_014_elementary_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2924,7 +2977,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -2932,19 +2985,11 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2003883ecec05b675876359540c06439933bae20ff47c4c334ae2cfa397c954/sql)\n", - "phase": "sql" - } - }, "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200ba25315ff7445dc543e2e42a2309cd493cfe1974181f53e50fa06c464134/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c20009af6638f1cf0fcf375f9cc76ed28165e2e2439a350311ccfc28427d6966/sql)\n", "phase": "sql_golden" } }, @@ -2956,15 +3001,23 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } + }, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200366e8eddaaa9d9fbc4186d386eb7fe6d14a6235a888d8e80e6570a63e0c4/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:43.824629255Z", - "finished_at": "2026-01-23T20:54:14.326210872Z" + "started_at": "2026-01-25T02:37:22.577850212Z", + "finished_at": "2026-01-25T02:37:49.346231061Z" }, "t_015_product_type_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2974,7 +3027,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -2991,37 +3044,37 @@ "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:46795" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:46871" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46795", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:54:07.846975920Z", - "finished_at": "2026-01-23T20:54:41.128204875Z" + "started_at": "2026-01-25T02:36:54.128552650Z", + "finished_at": "2026-01-25T02:37:22.577804857Z" }, "t_016_sum_type_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3031,7 +3084,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing { Id = 1, A = a, B = b });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3039,6 +3092,14 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c20091bdf61869c9af71e01c93d2c9e332381df02a937e4d3e056aec24264f30/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3047,31 +3108,23 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2002b630c1898fdca0d5df967295e9965d354ea89c64878bb9c71421ac210de/sql)\n", - "phase": "sql_golden" - } - }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200255f75652e874ebf03622fd9c2dfb3b65b42f575d4fa5a95cc36e4182f25/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d2f1563aa7bd84d2c9a3cffb3b1cfc12f1a874f1cbc2308e20af46e97d2a/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:54:07.849642740Z", - "finished_at": "2026-01-23T20:54:53.203074499Z" + "started_at": "2026-01-25T02:36:06.750524280Z", + "finished_at": "2026-01-25T02:36:53.948817678Z" }, "t_017_scheduled_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3081,7 +3134,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer row)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(500_000))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3097,7 +3150,7 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } @@ -3106,14 +3159,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200bcc4292b7995634fb91d48b6baa5c9096e7f85c4b73e7db1b4f1f377a348/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c2005eef72c2621b395ce9ab93b1cbfe93689e33bf4b87526400d0401b7e956a/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:28.130925948Z", - "finished_at": "2026-01-23T20:54:07.846947117Z" + "started_at": "2026-01-25T02:36:55.494511688Z", + "finished_at": "2026-01-25T02:37:33.685459289Z" }, "t_018_constraints": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3131,6 +3184,19 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46795", + "tables_diff": null, + "tables_equal": true + } + }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3142,7 +3208,7 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:46795" } }, "constraints_seed_two_rows": { @@ -3153,24 +3219,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:08.577602895Z", - "finished_at": "2026-01-23T20:53:40.420363798Z" + "started_at": "2026-01-25T02:36:06.753683987Z", + "finished_at": "2026-01-25T02:36:48.864755984Z" }, "t_019_many_to_many": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3180,7 +3233,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3196,20 +3249,11 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - }, "m2m_has_2_20": { "pass": true, "partial": 1.0, @@ -3228,6 +3272,15 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" + } + }, "m2m_has_1_20": { "pass": true, "partial": 1.0, @@ -3239,8 +3292,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:13.731095349Z", - "finished_at": "2026-01-23T20:54:05.268633334Z" + "started_at": "2026-01-25T02:37:33.685487289Z", + "finished_at": "2026-01-25T02:38:16.264442645Z" }, "t_020_ecs": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3250,7 +3303,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n int nextX = pos.X + vel.Value.VX;\n int nextY = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n var np = existing.Value;\n np.X = nextX;\n np.Y = nextY;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n if (ctx.Db.Entity.Id.Find(1) is Entity)\n {\n }\n else\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n\n if (ctx.Db.Position.EntityId.Find(1) is Position p1)\n {\n p1.X = 0;\n p1.Y = 0;\n ctx.Db.Position.EntityId.Update(p1);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n\n if (ctx.Db.Velocity.EntityId.Find(1) is Velocity v1)\n {\n v1.VX = 1;\n v1.VY = 0;\n ctx.Db.Velocity.EntityId.Update(v1);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n if (ctx.Db.Entity.Id.Find(2) is Entity)\n {\n }\n else\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n\n if (ctx.Db.Position.EntityId.Find(2) is Position p2)\n {\n p2.X = 10;\n p2.Y = 0;\n ctx.Db.Position.EntityId.Update(p2);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n\n if (ctx.Db.Velocity.EntityId.Find(2) is Velocity v2)\n {\n v2.VX = -2;\n v2.VY = 3;\n ctx.Db.Velocity.EntityId.Update(v2);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n int vx = 0;\n int vy = 0;\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n vx = vel.VX;\n vy = vel.VY;\n }\n\n int nx = pos.X + vx;\n int ny = pos.Y + vy;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3258,11 +3311,27 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", + "phase": "sql" + } + }, + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", + "phase": "sql" + } + }, "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", "phase": "sql" } }, @@ -3274,39 +3343,23 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", - "phase": "sql" - } - }, "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:53.389850738Z", - "finished_at": "2026-01-23T20:53:43.824579798Z" + "started_at": "2026-01-25T02:36:54.125794979Z", + "finished_at": "2026-01-25T02:37:44.772639848Z" }, "t_021_multi_column_index": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3324,6 +3377,15 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -3341,20 +3403,11 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:46795", "tables_diff": null, "tables_equal": true } }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - }, "mcindex_seed_count": { "pass": true, "partial": 1.0, @@ -3366,8 +3419,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:00.062219949Z", - "finished_at": "2026-01-23T20:53:28.130894765Z" + "started_at": "2026-01-25T02:36:48.864793626Z", + "finished_at": "2026-01-25T02:37:35.934293568Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 8fd73afcfd8..ddcf9423392 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-23T20:54:53.246Z", + "generated_at": "2026-01-25T02:39:25.598Z", "by_language": { "csharp": { "modes": { @@ -12,10 +12,10 @@ "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 24, - "pass_pct": 88.888885, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 + "passed_tests": 27, + "pass_pct": 100.0, + "task_pass_equiv": 12.0, + "task_pass_pct": 100.0 }, "schema": { "tasks": 10, @@ -29,10 +29,10 @@ "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 46, - "pass_pct": 75.409836, - "task_pass_equiv": 17.366667, - "task_pass_pct": 78.9394 + "passed_tests": 49, + "pass_pct": 80.327866, + "task_pass_equiv": 18.366667, + "task_pass_pct": 83.48485 } } } @@ -56,44 +56,44 @@ }, "schema": { "tasks": 10, - "total_tests": 30, - "passed_tests": 4, - "pass_pct": 13.333333, - "task_pass_equiv": 1.25, - "task_pass_pct": 12.5 + "total_tests": 34, + "passed_tests": 8, + "pass_pct": 23.529411, + "task_pass_equiv": 2.05, + "task_pass_pct": 20.5 } }, "totals": { "tasks": 22, - "total_tests": 57, - "passed_tests": 9, - "pass_pct": 15.789474, - "task_pass_equiv": 2.5833335, - "task_pass_pct": 11.742425 + "total_tests": 61, + "passed_tests": 13, + "pass_pct": 21.311476, + "task_pass_equiv": 3.3833334, + "task_pass_pct": 15.378788 } } } }, "rustdoc_json": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 8.916667, - "task_pass_pct": 74.30556 + "passed_tests": 26, + "pass_pct": 96.296295, + "task_pass_equiv": 11.0, + "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 26, - "pass_pct": 76.47059, - "task_pass_equiv": 7.5333333, - "task_pass_pct": 75.333336 + "passed_tests": 22, + "pass_pct": 64.70588, + "task_pass_equiv": 6.2, + "task_pass_pct": 62.0 } }, "totals": { @@ -101,8 +101,8 @@ "total_tests": 61, "passed_tests": 48, "pass_pct": 78.68852, - "task_pass_equiv": 16.45, - "task_pass_pct": 74.772736 + "task_pass_equiv": 17.2, + "task_pass_pct": 78.18182 } } } From a023c85df88e59b0058479ae8b34b7ceabe90b16 Mon Sep 17 00:00:00 2001 From: rekhoff Date: Fri, 23 Jan 2026 08:50:34 -0800 Subject: [PATCH 27/32] CI: pack C# NuGets in Release --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46627105cc7..6140cc90485 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -567,8 +567,8 @@ jobs: - name: Override NuGet packages run: | - dotnet pack crates/bindings-csharp/BSATN.Runtime - dotnet pack crates/bindings-csharp/Runtime + dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime + dotnet pack -c Release crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if @@ -693,8 +693,8 @@ jobs: - name: Override NuGet packages run: | - dotnet pack crates/bindings-csharp/BSATN.Runtime - dotnet pack crates/bindings-csharp/Runtime + dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime + dotnet pack -c Release crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if From 689bd9c29b6261b0dc50a7be6c58108dcbc70272 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Tue, 27 Jan 2026 05:20:04 +0000 Subject: [PATCH 28/32] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 341 +++--- docs/llms/docs-benchmark-comment.md | 16 +- docs/llms/docs-benchmark-details.json | 1428 ++++++++++++------------- docs/llms/docs-benchmark-summary.json | 54 +- 4 files changed, 941 insertions(+), 898 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 7ed34de5610..ff3159303e8 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -8,30 +8,16 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do --- -# SpacetimeDB Benchmark Test Failure Analysis +# Analysis of SpacetimeDB Benchmark Failures -## Summary of Analysis -This document organizes the test failures by language (Rust, C#, etc.) and provides detailed analysis of common issues. Each failure will include comparisons between the LLM-generated and expected code, identify the root causes, and recommend actionable changes to the documentation. +## Rust / rustdoc_json Failures ---- - -## Rust / rustdoc_json Failures (8 total) - -### Common Issues -1. **Faulty Function Signatures and Types**: Many failures stem from incorrect function signatures and types in declared reducers and tables. - -2. **Missing Expected Traits/Attributes**: LLM-generated code often lacks the required public visibility (e.g., `pub`), causing structure and attribute mismatches. - -3. **Incorrect Use of Scheduling**: The scheduling function setup was incorrect in multiple instances, failing to align with the expected API-style. - ---- - -### 1. Compile/Publish Errors +### Compile/Publish Errors (2 failures) -#### a. t_002_scheduled_table -**LLM Output**: +#### t_002_scheduled_table +**Generated Code**: ```rust -use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; +use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn, ScheduleAt}; #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] pub struct TickTimer { @@ -43,16 +29,33 @@ pub struct TickTimer { #[reducer(init)] pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { + let tbl = ctx.db.tick_timer(); + + let mut keep_id: Option = None; + for row in tbl.iter() { + if keep_id.is_none() { + let mut r = row; + r.scheduled_at = ScheduleAt::IntervalMicros(50_000); + let updated = tbl.scheduled_id().update(r); + keep_id = Some(updated.scheduled_id); + } else { + tbl.scheduled_id().delete(&row.scheduled_id); + } + } + + if keep_id.is_none() { + let _ = tbl.insert(TickTimer { scheduled_id: 0, - scheduled_at: ScheduleAt::RepeatMicros(50_000), + scheduled_at: ScheduleAt::IntervalMicros(50_000), }); } } + +#[reducer] +pub fn tick(_ctx: &ReducerContext) {} ``` -**Expected (golden)**: +**Expected (Golden Code)**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -61,8 +64,13 @@ use std::time::Duration; pub struct TickTimer { #[primary_key] #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { + Ok(()) } #[reducer(init)] @@ -74,165 +82,254 @@ pub fn init(ctx: &ReducerContext) -> Result<(), String> { Ok(()) } ``` + **Error**: `publish_error: spacetime publish failed (exit=1)` -**Difference**: -- Missing `pub` visibility on struct fields. -- Usage of `ScheduleAt::RepeatMicros` instead of `ScheduleAt::Interval(Duration::from_millis(...).into())`. +**Differences**: +- The LLM used `schedule(reducer = tick, column = scheduled_at)` instead of the expected `scheduled(tick)`. +- The `tick` reducer function lacked a return type. +- The `init` function's `scheduled_at` field was incorrectly assigned using `IntervalMicros(50_000)` instead of `Interval(Duration::from_millis(50).into())`. -**Root Cause**: -The documentation lacks clear examples of field visibility and correct usage of scheduling parameters. +**Root Cause**: Misunderstanding of how to set up scheduling and return types for reducer functions in Rust. -**Recommendation**: -Update the documentation to include visibility modifiers and examples utilizing `ScheduleAt::Interval`. +**Recommendation**: Update the documentation to clarify: +- How to define scheduling within the table attribute. +- The need for specifying return types in reducer functions. +- Correct usage examples for setting up intervals for scheduling. --- -#### b. t_017_scheduled_columns -**Similar to t_002_scheduled_table**. +#### t_017_scheduled_columns +**Generated Code**: +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; + +#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { + ctx.db + .tick_timer() + .insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat_micros(50_000), + }); + } +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {} +``` + +**Expected (Golden Code)**: +```rust +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); +} +``` -- **Action Required**: Same recommendations as above. +**Error**: `publish_error: spacetime publish failed (exit=1)` + +**Differences**: +- The LLM again used the incorrect syntax for scheduling. +- Used `ScheduleAt::repeat_micros(50_000)` while the expected syntax is using `Duration::from_millis(...)`. + +**Root Cause**: Similar to the first failure, a misunderstanding of the correct APIs for table scheduling and the desired syntax. + +**Recommendation**: Amend documentation or the code generation model to ensure the correct formatting and usage patterns are demonstrated, especially around scheduling and types. --- -### 2. Other Failures +### Other Failures (5 failures) -#### a. t_012_spacetime_product_type -**LLM Output**: +#### t_013_spacetime_sum_type (2/3 tests passed) +**Generated Code**: ```rust -use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; +use spacetimedb::{ReducerContext, Table, SpacetimeType}; #[derive(SpacetimeType)] -struct Score { - left: i32, - right: i32, +pub struct Rect { + pub width: i32, + pub height: i32, } -#[table(name = result)] -struct ResultRow { +#[derive(SpacetimeType)] +pub enum Shape { + Circle(i32), + Rectangle(Rect), +} + +#[spacetimedb::table(name = result)] +pub struct ResultRow { #[primary_key] - id: i32, - value: Score, + pub id: i32, + pub value: Shape, } -#[reducer] -fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { +#[spacetimedb::reducer] +pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { ctx.db.result().insert(ResultRow { id, - value: Score { left, right }, + value: Shape::Circle(radius), }); } ``` -**Expected (golden)**: +**Expected (Golden Code)**: ```rust use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; #[derive(SpacetimeType, Clone, Debug)] -pub struct Score { - pub left: i32, - pub right: i32, +pub struct Rect { + pub width: i32, + pub height: i32, +} + +#[derive(SpacetimeType, Clone, Debug)] +pub enum Shape { + Circle(i32), + Rectangle(Rect), } #[table(name = result)] pub struct ResultRow { #[primary_key] pub id: i32, - pub value: Score, + pub value: Shape, } #[reducer] -pub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { - ctx.db.result().insert(ResultRow { id, value: Score { left, right } }); +pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { + ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) }); } ``` -**Error**: `no such table: "result"` +**Error**: `no such table: result`. -**Difference**: -- Missing `pub` declaration on the struct and fields. +**Differences**: +- The LLM missed implementing the `Clone` and `Debug` traits for `Rect` and `Shape`. +- Notably, this could have contributed to the inability to handle implications regarding mutability and state during tests. -**Root Cause**: -The need for public access levels on types and fields is not sufficiently covered in the documentation. +**Root Cause**: The omission of `Clone` and `Debug` traits in derived types may confuse the necessary behavior for interacting with the database and running appropriate tests. -**Recommendation**: -Add clear examples demonstrating the use of `pub` in type definitions. +**Recommendation**: Include compliance with trait derivations in the documentation and examples to reflect necessary traits for database interaction. --- -### Additional Failures -- The patterns and problems arising in these similar failures (t_013, t_015, t_016, t_018) align closely with public visibility issues or missing API attributes. -- **Action Required**: Follow similar recommendations emphasizing public access in documented type structures. - ---- - -## Rust / docs Failures (22 total) +#### t_015_product_type_columns (2/3 tests passed) +**Generated Code**: +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; -The issues observed in the Rust documentation failures mirror the previous section, particularly around building, struct declarations, and public visibility. +#[derive(SpacetimeType)] +pub struct Address { + street: String, + zip: i32, +} -### Example: t_000_empty_reducers -The LLM-generated code lacks return types on various reducers, while the golden example properly uses `Result<(), String>`. +#[derive(SpacetimeType)] +pub struct Position { + x: i32, + y: i32, +} -**Actionable Insight**: Clearly document the importance of declaring function return types on reducers. +#[table(name = profile)] +pub struct Profile { + #[primary_key] + id: i32, + home: Address, + work: Address, + pos: Position, +} ---- +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.profile().insert(Profile { + id: 1, + home: Address { street: "1 Main".to_string(), zip: 11111 }, + work: Address { street: "2 Broad".to_string(), zip: 22222 }, + pos: Position { x: 7, y: 9 }, + }); +} +``` -## C# / docs Failures (5 total) +**Expected (Golden Code)**: +```rust +use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; -### Common Issues: -1. **Visibility Modifiers**: Many C# failures stem from missing or wrong visibility modifiers. -2. **Attributes and Enums**: Incorrect use or omission of expected attributes (like `[Type]`) was frequent. +#[derive(SpacetimeType, Clone, Debug)] +pub struct Address { + pub street: String, + pub zip: i32, +} ---- +#[derive(SpacetimeType, Clone, Debug)] +pub struct Position { + pub x: i32, + pub y: i32, +} -### Example: t_013_spacetime_sum_type -**LLM Output**: -```csharp -using SpacetimeDB; -using SpacetimeDB.Types; - -public static partial class Module -{ - [SpacetimeDB.Type] - public partial struct Circle { public int Radius; } - - [SpacetimeDB.Table(Name = "Result", Public = true)] - public partial struct Result - { - [SpacetimeDB.PrimaryKey] public int Id; - public Shape Value; - } +#[table(name = profile)] +pub struct Profile { + #[primary_key] + pub id: i32, + pub home: Address, + pub work: Address, + pub pos: Position, } -``` -**Expected (golden)**: -```csharp -using SpacetimeDB; - -public static partial class Module -{ - [Type] public partial struct Circle { public int Radius; } - - [Table(Name = "Result")] - public partial struct Result - { - [PrimaryKey] public int Id; - public Shape Value; - } +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.profile().insert(Profile { + id: 1, + home: Address { street: "1 Main".into(), zip: 11111 }, + work: Address { street: "2 Broad".into(), zip: 22222 }, + pos: Position { x: 7, y: 9 }, + }); } ``` -**Error**: `publish_error: spacetime build (csharp) failed` -**Difference**: Incorrectly specified `Public = true` attribute in the table declaration; missing the `[Type]` attribute. +**Error**: `no such table: profile`. + +**Differences**: +- Missing the `pub` visibility modifier on the struct fields resulted in inadequate exposure to the database layer for runtime reflection or generation. -**Root Cause**: Lack of explicit examples and standards regarding visibility and attribute application in the documentation. +**Root Cause**: The LLM lacks clarity in visibility context leading to struct field inaccessibility. -**Recommendation**: Incorporate detailed examples that differentiate between public and default access levels, especially in C# attributes. +**Recommendation**: Update documentation examples to emphasize the importance of `pub` modifiers for public API, particularly in database-related types. --- -## Conclusion -The analysis reveals consistent issues to be addressed across various tests. The primary points of failure involve: -- Visibility modifiers. -- Correct API usage of structure and attributes. -- Suggest clarifying the conditions and requirements for successful compilation in the documentation to enhance developer understanding and reduce confusion. +### Recommendation Summary +The fundamental issues in the test failures stem from: +- Incorrect syntax (particularly in `schedule` attributes). +- Missing visibility modifiers (`pub`). +- Omitted trait implementations necessary for database operations. + +Enhancing the clarity, correctness, and completeness of documentation is essential. Specific examples that match the expected outputs must illustrate successful patterns for users. This would significantly lower the occurrence of erroneous submissions, leading to successful builds and test runs. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index 164647c309a..beaddb79c57 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 26/27 | 91.7% ⬆️ +17.4% | -| Rust | rustdoc_json | schema | 22/34 | 62.0% ⬇️ -13.3% | -| Rust | rustdoc_json | **total** | 48/61 | **78.2%** ⬆️ +3.4% | +| Rust | rustdoc_json | basics | 26/27 | 91.7% ⬆️ +15.3% | +| Rust | rustdoc_json | schema | 23/34 | 65.3% ⬇️ -10.0% | +| Rust | rustdoc_json | **total** | 49/61 | **79.7%** ⬆️ +3.8% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | -| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | +| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | +| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | +| C# | docs | basics | 24/27 | 91.7% ⬇️ -8.3% | | C# | docs | schema | 22/34 | 63.7% | -| C# | docs | **total** | 49/61 | **83.5%** ⬆️ +4.5% | +| C# | docs | **total** | 46/61 | **78.9%** ⬇️ -4.5% | _Compared against master branch baseline_ -Generated at: 2026-01-25T02:39:25.598Z +Generated at: 2026-01-27T05:19:18.787Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 6470d95191e..4c1a585bbfa 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -35,15 +35,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:03.346418371Z", - "finished_at": "2026-01-25T02:31:28.816274281Z" + "started_at": "2026-01-27T05:09:39.036012266Z", + "finished_at": "2026-01-27T05:10:07.068111278Z" }, "t_001_basic_tables": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -53,7 +53,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -69,15 +69,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:22.050348955Z", - "finished_at": "2026-01-25T02:31:53.323790812Z" + "started_at": "2026-01-27T05:09:49.487854712Z", + "finished_at": "2026-01-27T05:10:12.930610649Z" }, "t_002_scheduled_table": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -87,7 +87,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let tbl = ctx.db.tick_timer();\n\n let mut keep_id: Option = None;\n for row in tbl.iter() {\n if keep_id.is_none() {\n let mut r = row;\n r.scheduled_at = ScheduleAt::IntervalMicros(50_000);\n let updated = tbl.scheduled_id().update(r);\n keep_id = Some(updated.scheduled_id);\n } else {\n tbl.scheduled_id().delete(&row.scheduled_id);\n }\n }\n\n if keep_id.is_none() {\n let _ = tbl.insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::IntervalMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,14 +99,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling zerocopy v0.8.33\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.16\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling serde_json v1.0.149\n Compiling spacetimedb-lib v1.11.1\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling rand_core v0.6.4\n Compiling itertools v0.12.1\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling zmij v1.0.17\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.4.2\n Compiling smallvec v1.15.1\n Compiling log v0.4.29\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:29:28\n |\n29 | let _ = tbl.insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nwarning: unused import: `UniqueColumn`\n --> src/lib.rs:2:58\n |\n2 | use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn, ScheduleAt};\n | ^^^^^^^^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:22\n |\n14 | let tbl = ctx.db.tick_timer();\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `IntervalMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:42\n |\n20 | r.scheduled_at = ScheduleAt::IntervalMicros(50_000);\n | ^^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0599]: no variant or associated item named `IntervalMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:31:39\n |\n31 | scheduled_at: ScheduleAt::IntervalMicros(50_000),\n | ^^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:53.478926950Z", - "finished_at": "2026-01-25T02:32:31.946860665Z" + "started_at": "2026-01-27T05:10:07.068141621Z", + "finished_at": "2026-01-27T05:11:42.265583024Z" }, "t_003_struct_in_table": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -116,7 +116,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -132,15 +132,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:53.479198674Z", - "finished_at": "2026-01-25T02:32:17.958288213Z" + "started_at": "2026-01-27T05:09:49.139992583Z", + "finished_at": "2026-01-27T05:10:16.768994056Z" }, "t_004_insert": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -158,19 +158,6 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:45345", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_insert_user": { "pass": true, "partial": 1.0, @@ -187,13 +174,26 @@ "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "insert_user", - "server": "http://127.0.0.1:45345" + "server": "http://127.0.0.1:42497" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42497", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:13.441650390Z", - "finished_at": "2026-01-25T02:31:33.453015538Z" + "started_at": "2026-01-27T05:09:53.395255126Z", + "finished_at": "2026-01-27T05:10:17.908045819Z" }, "t_005_update": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -203,7 +203,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let tbl = ctx.db.user();\n if tbl.id().find(id).is_some() {\n let user = User { id, name, age, active };\n let _ = tbl.id().update(user);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -227,7 +227,14 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:45345" + "server": "http://127.0.0.1:42497" + } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -238,22 +245,15 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:03.346171101Z", - "finished_at": "2026-01-25T02:31:57.978256373Z" + "started_at": "2026-01-27T05:09:36.767499557Z", + "finished_at": "2026-01-27T05:10:12.600469319Z" }, "t_006_delete": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -263,7 +263,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -271,6 +271,15 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -279,20 +288,11 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -302,8 +302,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:14.887913551Z", - "finished_at": "2026-01-25T02:31:53.478880313Z" + "started_at": "2026-01-27T05:09:25.210537530Z", + "finished_at": "2026-01-27T05:09:53.395220193Z" }, "t_007_crud": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -313,7 +313,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n ctx.db.user().id().update(updated);\n ctx.db.user().id().delete(&2);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let mut u1 = ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n let _u2 = ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _ = ctx.db.user().id().update(u1);\n\n ctx.db.user().id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -321,13 +321,13 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { + "crud_total_count_one": { "pass": true, "partial": 1.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "schema_parity": { @@ -338,11 +338,20 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } + }, "crud_row_id1_parity": { "pass": true, "partial": 1.0, @@ -354,22 +363,13 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "crud", - "server": "http://127.0.0.1:45345" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "server": "http://127.0.0.1:42497" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:53.323830823Z", - "finished_at": "2026-01-25T02:32:29.819999491Z" + "started_at": "2026-01-27T05:09:25.696714364Z", + "finished_at": "2026-01-27T05:10:09.073577868Z" }, "t_008_index_lookup": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -379,7 +379,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&u.id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n let result = ctx.db.result();\n let _ = result.id().delete(&user.id);\n result.insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -387,6 +387,13 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -400,7 +407,7 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:45345" + "server": "http://127.0.0.1:42497" } }, "schema_parity": { @@ -411,22 +418,15 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:33.453053061Z", - "finished_at": "2026-01-25T02:32:13.689948121Z" + "started_at": "2026-01-27T05:09:49.813403458Z", + "finished_at": "2026-01-27T05:10:34.430514387Z" }, "t_009_init": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -436,7 +436,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -453,6 +453,15 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -461,7 +470,7 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } @@ -474,20 +483,11 @@ "expected": 2, "sql": "SELECT COUNT(*) AS n FROM user" } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:28.430437681Z", - "finished_at": "2026-01-25T02:31:57.789037350Z" + "started_at": "2026-01-27T05:09:34.070638452Z", + "finished_at": "2026-01-27T05:10:02.442854152Z" }, "t_010_connect": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -513,15 +513,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:28.816300522Z", - "finished_at": "2026-01-25T02:31:58.802773918Z" + "started_at": "2026-01-27T05:09:25.470266069Z", + "finished_at": "2026-01-27T05:09:49.813377232Z" }, "t_011_helper_function": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -531,7 +531,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -554,7 +554,7 @@ "llm_out": "id | sum ----+----- 1 | 5", "query": "SELECT id, sum FROM result WHERE id=1", "reducer": "compute_sum", - "server": "http://127.0.0.1:45345" + "server": "http://127.0.0.1:42497" } }, "helper_func_sum_abs": { @@ -574,15 +574,15 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:23.706005145Z", - "finished_at": "2026-01-25T02:31:53.232845655Z" + "started_at": "2026-01-27T05:10:02.442910356Z", + "finished_at": "2026-01-27T05:10:35.887992543Z" }, "t_012_spacetime_product_type": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -591,8 +591,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -610,11 +610,21 @@ } }, "product_type_row_parity": { - "pass": false, - "partial": 0.0, + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c200ae83d4dafbab5553f9122e0ca48ece3f49f7a81788ca7b482fbadb2bfdc6/sql)\n", - "phase": "sql_golden" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_score", + "server": "http://127.0.0.1:42497" } }, "schema_parity": { @@ -625,15 +635,15 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.755441145Z", - "finished_at": "2026-01-25T02:30:56.427439801Z" + "started_at": "2026-01-27T05:09:25.210278292Z", + "finished_at": "2026-01-27T05:09:49.139940093Z" }, "t_013_spacetime_sum_type": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -643,7 +653,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -659,7 +669,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } @@ -677,14 +687,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c200393f375639ae660e3acad93b582d176f774b002af25510f5a6e5918dfee9/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200f97c09e635edf695ac26aad1c54bc9a89da51a7a604a4af5f82a11de59f6/sql)\n", "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.755726622Z", - "finished_at": "2026-01-25T02:31:02.603517791Z" + "started_at": "2026-01-27T05:08:44.005781294Z", + "finished_at": "2026-01-27T05:09:25.071436567Z" }, "t_014_elementary_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -694,7 +704,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -702,19 +712,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:45345", - "tables_diff": null, - "tables_equal": true - } - }, "elementary_columns_row_count": { "pass": true, "partial": 1.0, @@ -735,13 +732,26 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:45345" + "server": "http://127.0.0.1:42497" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42497", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:56.427473474Z", - "finished_at": "2026-01-25T02:31:22.050298776Z" + "started_at": "2026-01-27T05:09:13.710740034Z", + "finished_at": "2026-01-27T05:09:49.487808690Z" }, "t_015_product_type_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -751,7 +761,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -759,6 +769,15 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -767,7 +786,7 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } @@ -776,23 +795,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c2006e1b25ff28f6ae88b2c072ea7e0bb34a9558acc6b799d903dd85649e3abc/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c20021a9078493229a6445a6134a369b6b4e4012607c2932b27ccb84d9bed19b/sql)\n", "phase": "sql_golden" } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.756612791Z", - "finished_at": "2026-01-25T02:31:14.887824445Z" + "started_at": "2026-01-27T05:08:44.006206481Z", + "finished_at": "2026-01-27T05:09:25.470222738Z" }, "t_016_sum_type_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -802,7 +812,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().id().find(1).is_none() {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\nstruct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let a = Shape::Circle(10);\n let b = Shape::Rectangle(Rect { width: 4, height: 6 });\n ctx.db.drawing().insert(Drawing { id: 1, a, b });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -810,12 +820,12 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c2001e6bfa410e8407328c63f0e357b56f2983a849414cffa113aa69b61ba8c7/sql)\n", - "phase": "sql_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200fbf44894622c67c4741e23a23fefc9243011be0dcc61b67d7a39a6055784/sql)\n", + "phase": "sql" } }, "schema_parity": { @@ -826,23 +836,23 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c200a4f49580f3c6577e995861d2eede035cb964ba3c499f577b033537d5c75d/sql)\n", - "phase": "sql" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200af8c990f76ff6e310b60db2e5393bf8e1f0a3d6c26af8199ea36fba0cde3/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.755940232Z", - "finished_at": "2026-01-25T02:31:03.164647878Z" + "started_at": "2026-01-27T05:08:44.005628685Z", + "finished_at": "2026-01-27T05:09:13.710709192Z" }, "t_017_scheduled_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -852,7 +862,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db\n .tick_timer()\n .insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -864,14 +874,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.16\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling itoa v1.0.17\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling constant_time_eq v0.4.2\n Compiling itertools v0.12.1\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:21:42\n |\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n 20 | #[reducer]\n | ---------- required by a bound introduced by this call\n 21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n20 | #[reducer]\n | ---------- required by a bound introduced by this call\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling zmij v1.0.17\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling cc v1.2.54\n Compiling itoa v1.0.17\n Compiling spacetimedb-lib v1.11.1\n Compiling smallvec v1.15.1\n Compiling log v0.4.29\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.7\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:17:21\n |\n17 | .insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:25:42\n |\n25 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:16:14\n |\n15 | / ctx.db\n16 | | .tick_timer()\n | | -^^^^^^^^^^ method not found in `Local`\n | |_____________|\n |\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:19:43\n |\n19 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:25:8\n |\n 24 | #[reducer]\n | ---------- required by a bound introduced by this call\n 25 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:25:8\n |\n24 | #[reducer]\n | ---------- required by a bound introduced by this call\n25 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.756799192Z", - "finished_at": "2026-01-25T02:31:13.441629467Z" + "started_at": "2026-01-27T05:08:44.005307486Z", + "finished_at": "2026-01-27T05:09:36.767482084Z" }, "t_018_constraints": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -881,7 +891,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n\n let mut ids: Vec = Vec::new();\n for row in account.iter() {\n ids.push(row.id);\n }\n for id in ids {\n account.id().delete(&id);\n }\n\n account.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n account.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|row| row.id).collect();\n for id in ids {\n ctx.db.account().id().delete(&id);\n }\n\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -889,6 +899,14 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { + "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200151787d57be8f301f25c739a5cbfefc33ef1cd6af6467c963721fbe7e9a4/sql)\n", + "phase": "sql_golden" + } + }, "constraints_seed_two_rows": { "pass": true, "partial": 1.0, @@ -898,14 +916,6 @@ "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, - "constraints_row_parity_after_seed": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20087d08bf31d4bf8fd3f3f6c645f9e2f81779454fb7e9160ec9aa5a5cb3145/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -914,15 +924,15 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:45345", + "server": "http://127.0.0.1:42497", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.756125204Z", - "finished_at": "2026-01-25T02:31:23.705962716Z" + "started_at": "2026-01-27T05:08:44.005088378Z", + "finished_at": "2026-01-27T05:09:25.210239572Z" }, "t_019_many_to_many": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -932,7 +942,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id])),\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure the tables contain exactly the requested rows\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n {\n let ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in ids {\n ctx.db.membership().id().delete(&id);\n }\n }\n {\n let ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in ids {\n ctx.db.user().user_id().delete(&id);\n }\n }\n {\n let ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in ids {\n ctx.db.group().group_id().delete(&id);\n }\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -940,13 +950,17 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "m2m_has_1_20": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42497", + "tables_diff": null, + "tables_equal": true } }, "memberships_three_rows": { @@ -958,13 +972,13 @@ "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" } }, "m2m_has_2_20": { @@ -976,23 +990,19 @@ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } }, - "schema_parity": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:45345", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:31:02.603568306Z", - "finished_at": "2026-01-25T02:31:59.581434159Z" + "started_at": "2026-01-27T05:08:44.005919892Z", + "finished_at": "2026-01-27T05:09:39.035961444Z" }, "t_020_ecs": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -1002,7 +1012,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n // Clear existing rows for ids 1 and 2 to make seeding idempotent\n for id in [1i32, 2i32] {\n ctx.db.next_position().entity_id().delete(&id);\n ctx.db.velocity().entity_id().delete(&id);\n ctx.db.position().entity_id().delete(&id);\n ctx.db.entity().id().delete(&id);\n }\n\n // Insert entity 1\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db\n .position()\n .insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db\n .velocity()\n .insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n // Insert entity 2\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db\n .position()\n .insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db\n .velocity()\n .insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\nfn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: pos.entity_id, x: nx, y: ny });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let entity_tbl = ctx.db.entity();\n let position_tbl = ctx.db.position();\n let velocity_tbl = ctx.db.velocity();\n let next_position_tbl = ctx.db.next_position();\n\n for id in [1i32, 2i32] {\n entity_tbl.id().delete(&id);\n position_tbl.entity_id().delete(&id);\n velocity_tbl.entity_id().delete(&id);\n next_position_tbl.entity_id().delete(&id);\n }\n\n entity_tbl.insert(Entity { id: 1 });\n position_tbl.insert(Position { entity_id: 1, x: 0, y: 0 });\n velocity_tbl.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n entity_tbl.insert(Entity { id: 2 });\n position_tbl.insert(Position { entity_id: 2, x: 10, y: 0 });\n velocity_tbl.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let pos_tbl = ctx.db.position();\n let vel_by_eid = ctx.db.velocity().entity_id();\n let next_pos_tbl = ctx.db.next_position();\n let next_by_eid = next_pos_tbl.entity_id();\n\n for p in pos_tbl.iter() {\n if let Some(v) = vel_by_eid.find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n let row = NextPosition { entity_id: p.entity_id, x: nx, y: ny };\n if next_by_eid.find(p.entity_id).is_some() {\n next_by_eid.update(row);\n } else {\n next_pos_tbl.insert(row);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1010,27 +1020,32 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200fa2d6af091f55bd4fde28b2be1a9c4e4f4b8201739b861060bf9c546b383/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", - "phase": "sql" + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42497", + "tables_diff": null, + "tables_equal": true } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200fa2d6af091f55bd4fde28b2be1a9c4e4f4b8201739b861060bf9c546b383/sql)\n", "phase": "sql" } }, @@ -1038,27 +1053,22 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45345/v1/database/c20039746af45514fdf9e58de5ef43f025104994e05647fa6bd06ea7be673e21/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200fa2d6af091f55bd4fde28b2be1a9c4e4f4b8201739b861060bf9c546b383/sql)\n", "phase": "sql" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:45345", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42497/v1/database/c200fa2d6af091f55bd4fde28b2be1a9c4e4f4b8201739b861060bf9c546b383/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.756440656Z", - "finished_at": "2026-01-25T02:31:28.430387499Z" + "started_at": "2026-01-27T05:08:44.006061528Z", + "finished_at": "2026-01-27T05:09:34.070571106Z" }, "t_021_multi_column_index": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -1068,7 +1078,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let table = ctx.db.log();\n\n for row in table.iter() {\n table.id().delete(&row.id);\n }\n\n table.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n table.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n table.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1076,16 +1086,20 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_seed_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42497", + "tables_diff": null, + "tables_equal": true + } + }, + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { @@ -1094,17 +1108,13 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "schema_parity": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:45345", - "tables_diff": null, - "tables_equal": true + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } }, "mcindex_lookup_u7_d2": { @@ -1118,8 +1128,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:30:25.756281600Z", - "finished_at": "2026-01-25T02:31:03.346131676Z" + "started_at": "2026-01-27T05:08:44.005464852Z", + "finished_at": "2026-01-27T05:09:25.696642287Z" } } } @@ -1141,7 +1151,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1159,8 +1169,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:29.683293458Z", - "finished_at": "2026-01-25T02:33:45.980299464Z" + "started_at": "2026-01-27T05:12:53.806823291Z", + "finished_at": "2026-01-27T05:13:21.170905173Z" }, "t_001_basic_tables": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1170,7 +1180,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb as _;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product, public)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note, public)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1188,18 +1198,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:34.613474286Z", - "finished_at": "2026-01-25T02:34:00.375778172Z" + "started_at": "2026-01-27T05:12:56.718811561Z", + "finished_at": "2026-01-27T05:13:20.826920312Z" }, "t_002_scheduled_table": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_002_scheduled_table", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1207,18 +1217,18 @@ "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling second-stack v0.3.5\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling log v0.4.29\n Compiling generic-array v0.14.7\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:5:57\n |\n5 | #[spacetimedb::table(name = tick_timer, scheduled(tick, column = scheduled_at))]\n | ^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:14:44\n |\n14 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:36\n |\n18 | ctx.db.tick_timer().try_insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:14:8\n |\n 13 | #[spacetimedb::reducer]\n | ----------------------- required by a bound introduced by this call\n 14 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:14:8\n |\n13 | #[spacetimedb::reducer]\n | ----------------------- required by a bound introduced by this call\n14 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:12\n |\n18 | ctx.db.tick_timer().try_insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 6 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:34:00.375805875Z", - "finished_at": "2026-01-25T02:34:36.170109978Z" + "started_at": "2026-01-27T05:13:12.504228993Z", + "finished_at": "2026-01-27T05:13:47.964001508Z" }, "t_003_struct_in_table": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1228,7 +1238,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "use spacetimedb::{SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1246,8 +1256,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:34:00.673539976Z", - "finished_at": "2026-01-25T02:34:19.746956045Z" + "started_at": "2026-01-27T05:12:54.466865149Z", + "finished_at": "2026-01-27T05:13:22.705047898Z" }, "t_004_insert": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1265,26 +1275,26 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_insert_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "call_reducer_golden" } }, - "data_parity_insert_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:31.141746367Z", - "finished_at": "2026-01-25T02:33:56.259096270Z" + "started_at": "2026-01-27T05:13:08.941876178Z", + "finished_at": "2026-01-27T05:13:40.868532384Z" }, "t_005_update": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1294,7 +1304,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n Ok(())\n } else {\n Err(\"user not found\".to_string())\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -1302,6 +1312,15 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { + "seed_users_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1317,20 +1336,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", "phase": "call_reducer_golden" } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:23.731576942Z", - "finished_at": "2026-01-25T02:33:47.840719598Z" + "started_at": "2026-01-27T05:12:52.686095721Z", + "finished_at": "2026-01-27T05:13:21.732152560Z" }, "t_006_delete": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1340,7 +1350,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -1348,13 +1358,12 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, + "schema_parity": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "describe_golden" } }, "seed_users_row": { @@ -1366,18 +1375,19 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:34.609970882Z", - "finished_at": "2026-01-25T02:34:04.254303613Z" + "started_at": "2026-01-27T05:12:29.873730369Z", + "finished_at": "2026-01-27T05:12:52.686067526Z" }, "t_007_crud": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1387,7 +1397,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n\n if let Some(mut u1) = ctx.db.user().id().find(1) {\n u1.name = \"Alice2\".into();\n u1.age = 31;\n u1.active = false;\n ctx.db.user().id().update(u1);\n }\n\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1395,21 +1405,30 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_total_count_one": { + "schema_parity": { "pass": false, "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "schema_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "crud_row_id1_parity": { @@ -1419,20 +1438,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", "phase": "call_reducer_golden" } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:56.259124969Z", - "finished_at": "2026-01-25T02:34:42.423310345Z" + "started_at": "2026-01-27T05:12:38.866410845Z", + "finished_at": "2026-01-27T05:13:09.759360569Z" }, "t_008_index_lookup": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1442,7 +1452,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n let uid = user.id;\n let name = user.name.clone();\n ctx.db.result().id().delete(&uid);\n ctx.db.result().insert(ResultRow { id: uid, name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1450,6 +1460,15 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { + "seed_user_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1465,20 +1484,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", "phase": "call_reducer_golden" } - }, - "seed_user_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:47.840748081Z", - "finished_at": "2026-01-25T02:34:12.833631543Z" + "started_at": "2026-01-27T05:13:02.903401660Z", + "finished_at": "2026-01-27T05:13:32.048079201Z" }, "t_009_init": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1488,7 +1498,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1496,6 +1506,15 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { + "init_total_two": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1504,37 +1523,28 @@ "phase": "describe_golden" } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:39.734179775Z", - "finished_at": "2026-01-25T02:34:03.009233798Z" + "started_at": "2026-01-27T05:12:40.992327715Z", + "finished_at": "2026-01-27T05:13:02.903361231Z" }, "t_010_connect": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1544,7 +1554,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".into(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".into(),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1562,8 +1572,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:45.980322799Z", - "finished_at": "2026-01-25T02:34:09.645472034Z" + "started_at": "2026-01-27T05:12:31.039996916Z", + "finished_at": "2026-01-27T05:12:53.806797106Z" }, "t_011_helper_function": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1573,7 +1583,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1581,6 +1591,14 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", + "phase": "call_reducer_golden" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1597,19 +1615,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } - }, - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:38.451065185Z", - "finished_at": "2026-01-25T02:34:01.189264068Z" + "started_at": "2026-01-27T05:13:09.759402293Z", + "finished_at": "2026-01-27T05:13:37.844428516Z" }, "t_012_spacetime_product_type": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1619,7 +1629,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{SpacetimeType, Table, ReducerContext};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n let _ = ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1627,12 +1637,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "product_type_row_parity": { @@ -1643,19 +1654,18 @@ "phase": "call_reducer_golden" } }, - "product_type_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.186581007Z", - "finished_at": "2026-01-25T02:33:31.141703042Z" + "started_at": "2026-01-27T05:12:22.008623875Z", + "finished_at": "2026-01-27T05:12:54.466833159Z" }, "t_013_spacetime_sum_type": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1665,7 +1675,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1673,12 +1683,13 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { + "sum_type_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -1689,19 +1700,18 @@ "phase": "describe_golden" } }, - "sum_type_row_count": { + "sum_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.189565224Z", - "finished_at": "2026-01-25T02:33:23.731499392Z" + "started_at": "2026-01-27T05:11:42.499008384Z", + "finished_at": "2026-01-27T05:12:29.873679962Z" }, "t_014_elementary_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1711,7 +1721,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1719,20 +1729,20 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "elementary_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "describe_golden" } }, "elementary_columns_row_count": { @@ -1746,8 +1756,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:05.794685418Z", - "finished_at": "2026-01-25T02:33:29.683265539Z" + "started_at": "2026-01-27T05:12:16.711787129Z", + "finished_at": "2026-01-27T05:12:40.992307751Z" }, "t_015_product_type_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1757,7 +1767,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let existing_ids: Vec = ctx.db.profile().iter().map(|p| p.id).collect();\n for id in existing_ids {\n ctx.db.profile().id().delete(&id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter().collect::>() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1765,14 +1775,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1789,11 +1791,19 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } + }, + "product_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.208388460Z", - "finished_at": "2026-01-25T02:33:34.609891050Z" + "started_at": "2026-01-27T05:11:42.509029252Z", + "finished_at": "2026-01-27T05:12:56.718725922Z" }, "t_016_sum_type_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1803,7 +1813,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(&row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1811,20 +1821,20 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c2002cd10e5fbdc1ac171a6579dc592276b6377c27107e3f7fe80ec794fe502b/sql)\n", - "phase": "sql" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44577/v1/database/c20058c7c8dba2434b35d7f6bfea577d950e8671e90dc59c22ec61033c21871f/sql)\n", + "phase": "sql" } }, "sum_type_columns_row_parity": { @@ -1837,8 +1847,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.193184498Z", - "finished_at": "2026-01-25T02:33:20.169991629Z" + "started_at": "2026-01-27T05:11:42.495703085Z", + "finished_at": "2026-01-27T05:12:16.711714729Z" }, "t_017_scheduled_columns": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1856,14 +1866,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } - }, "scheduled_seeded_one_row": { "pass": true, "partial": 1.0, @@ -1872,11 +1874,19 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.212187627Z", - "finished_at": "2026-01-25T02:33:38.450978070Z" + "started_at": "2026-01-27T05:11:42.489202709Z", + "finished_at": "2026-01-27T05:12:38.866360531Z" }, "t_018_constraints": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1886,7 +1896,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n ctx.db.account().insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1894,12 +1904,13 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "constraints_seed_two_rows": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, "constraints_row_parity_after_seed": { @@ -1910,84 +1921,47 @@ "phase": "call_reducer_golden" } }, - "constraints_seed_two_rows": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.196964280Z", - "finished_at": "2026-01-25T02:33:39.734083836Z" + "started_at": "2026-01-27T05:11:42.486761960Z", + "finished_at": "2026-01-27T05:12:31.039936960Z" }, "t_019_many_to_many": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:33:20.170044699Z", - "finished_at": "2026-01-25T02:34:00.673511214Z" + "started_at": "2026-01-27T05:13:47.964056076Z", + "finished_at": "2026-01-27T05:13:47.964056076Z" }, "t_020_ecs": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -1997,7 +1971,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n ctx.db\n .next_position()\n .entity_id()\n .update(NextPosition { entity_id: pos.entity_id, x: nx, y: ny });\n } else {\n let _ = ctx.db.next_position().try_insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // entity 1\n let e1 = Entity { id: 1 };\n if ctx.db.entity().id().find(1).is_some() {\n ctx.db.entity().id().update(e1);\n } else {\n ctx.db.entity().insert(e1);\n }\n let p1 = Position { entity_id: 1, x: 0, y: 0 };\n if ctx.db.position().entity_id().find(1).is_some() {\n ctx.db.position().entity_id().update(p1);\n } else {\n ctx.db.position().insert(p1);\n }\n let v1 = Velocity { entity_id: 1, vx: 1, vy: 0 };\n if ctx.db.velocity().entity_id().find(1).is_some() {\n ctx.db.velocity().entity_id().update(v1);\n } else {\n ctx.db.velocity().insert(v1);\n }\n\n // entity 2\n let e2 = Entity { id: 2 };\n if ctx.db.entity().id().find(2).is_some() {\n ctx.db.entity().id().update(e2);\n } else {\n ctx.db.entity().insert(e2);\n }\n let p2 = Position { entity_id: 2, x: 10, y: 0 };\n if ctx.db.position().entity_id().find(2).is_some() {\n ctx.db.position().entity_id().update(p2);\n } else {\n ctx.db.position().insert(p2);\n }\n let v2 = Velocity { entity_id: 2, vx: -2, vy: 3 };\n if ctx.db.velocity().entity_id().find(2).is_some() {\n ctx.db.velocity().entity_id().update(v2);\n } else {\n ctx.db.velocity().insert(v2);\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let np = NextPosition { entity_id: pos.entity_id, x: nx, y: ny };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(np);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -2005,35 +1979,35 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44577/v1/database/c200a7fbada0fd46f3e40abdde70280002e38415abdf54cb5f4be649f8c21e56/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44577/v1/database/c200a7fbada0fd46f3e40abdde70280002e38415abdf54cb5f4be649f8c21e56/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44577/v1/database/c200a7fbada0fd46f3e40abdde70280002e38415abdf54cb5f4be649f8c21e56/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35723/v1/database/c200ed322e79eaabe434c43b71b71a346f0fcb85c3d7fc95e9a32baf7b2b04dc/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44577/v1/database/c200a7fbada0fd46f3e40abdde70280002e38415abdf54cb5f4be649f8c21e56/sql)\n", "phase": "sql" } }, @@ -2047,8 +2021,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.204571288Z", - "finished_at": "2026-01-25T02:33:34.540842410Z" + "started_at": "2026-01-27T05:11:42.505683339Z", + "finished_at": "2026-01-27T05:13:08.941800584Z" }, "t_021_multi_column_index": { "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", @@ -2075,6 +2049,15 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, @@ -2091,20 +2074,11 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", "phase": "describe_golden" } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:32:32.200776398Z", - "finished_at": "2026-01-25T02:33:05.794620033Z" + "started_at": "2026-01-27T05:11:42.492376194Z", + "finished_at": "2026-01-27T05:12:22.008559529Z" } } } @@ -2325,15 +2299,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:37:44.772691102Z", - "finished_at": "2026-01-25T02:38:08.769282525Z" + "started_at": "2026-01-27T05:17:44.627639805Z", + "finished_at": "2026-01-27T05:18:09.481335803Z" }, "t_001_basic_tables": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2343,7 +2317,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2359,15 +2333,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:08.769310649Z", - "finished_at": "2026-01-25T02:38:31.184544005Z" + "started_at": "2026-01-27T05:17:55.158383440Z", + "finished_at": "2026-01-27T05:18:25.897837968Z" }, "t_002_scheduled_table": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2377,7 +2351,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(50_000 * 10))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2393,15 +2367,15 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:41.870651012Z", - "finished_at": "2026-01-25T02:39:25.554760208Z" + "started_at": "2026-01-27T05:18:25.897877441Z", + "finished_at": "2026-01-27T05:19:18.744530486Z" }, "t_003_struct_in_table": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2427,15 +2401,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:51.058027266Z", - "finished_at": "2026-01-25T02:39:10.268923061Z" + "started_at": "2026-01-27T05:17:52.802904222Z", + "finished_at": "2026-01-27T05:18:23.634551348Z" }, "t_004_insert": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2445,7 +2419,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -2461,7 +2435,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } @@ -2482,13 +2456,13 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:46795" + "server": "http://127.0.0.1:44653" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:37:49.346259164Z", - "finished_at": "2026-01-25T02:38:12.549780356Z" + "started_at": "2026-01-27T05:18:15.980467963Z", + "finished_at": "2026-01-27T05:18:38.727126646Z" }, "t_005_update": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2498,7 +2472,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing != null)\n {\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n ctx.Db.User.Id.Update(updated);\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2506,25 +2480,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_db": "basics-t-005-update-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "UpdateUser", - "server": "http://127.0.0.1:46795" - } - }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -2540,15 +2495,34 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } + }, + "data_parity_update_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], + "golden_db": "basics-t-005-update-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-005-update-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", + "reducer": "UpdateUser", + "server": "http://127.0.0.1:44653" + } } }, "vendor": "openai", - "started_at": "2026-01-25T02:37:35.934324581Z", - "finished_at": "2026-01-25T02:38:02.198161810Z" + "started_at": "2026-01-27T05:17:38.774070282Z", + "finished_at": "2026-01-27T05:18:15.980433413Z" }, "t_006_delete": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2590,15 +2564,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:02.198198168Z", - "finished_at": "2026-01-25T02:38:26.254084597Z" + "started_at": "2026-01-27T05:17:14.353276843Z", + "finished_at": "2026-01-27T05:17:38.774035541Z" }, "t_007_crud": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2608,7 +2582,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user)\n {\n user.Name = \"Alice2\";\n user.Age = 31;\n user.Active = false;\n ctx.Db.User.Id.Update(user);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2625,6 +2599,20 @@ "sql": "SELECT COUNT(*) AS n FROM User" } }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", + "reducer": "Crud", + "server": "http://127.0.0.1:44653" + } + }, "crud_row_id2_deleted": { "pass": true, "partial": 1.0, @@ -2642,39 +2630,25 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "basics-t-007-crud-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "Crud", - "server": "http://127.0.0.1:46795" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:37.706294669Z", - "finished_at": "2026-01-25T02:39:02.219172424Z" + "started_at": "2026-01-27T05:17:19.055224012Z", + "finished_at": "2026-01-27T05:17:52.802877106Z" }, "t_008_index_lookup": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_008_index_lookup", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2682,46 +2656,18 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_db": "basics-t-008-index-lookup-golden", - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "LookupUserName", - "server": "http://127.0.0.1:46795" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46795", - "tables_diff": null, - "tables_equal": true + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(32,27): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(33,29): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:31.184576580Z", - "finished_at": "2026-01-25T02:39:07.239374457Z" + "started_at": "2026-01-27T05:18:09.481363469Z", + "finished_at": "2026-01-27T05:18:31.331453433Z" }, "t_009_init": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2756,33 +2702,33 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:16.264482791Z", - "finished_at": "2026-01-25T02:38:41.870615455Z" + "started_at": "2026-01-27T05:17:27.128266860Z", + "finished_at": "2026-01-27T05:17:55.158341637Z" }, "t_010_connect": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2792,7 +2738,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2808,15 +2754,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:26.254121165Z", - "finished_at": "2026-01-25T02:38:51.058005185Z" + "started_at": "2026-01-27T05:17:15.959992832Z", + "finished_at": "2026-01-27T05:17:44.627585170Z" }, "t_011_helper_function": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2826,7 +2772,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b),\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2834,17 +2780,13 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46795", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" } }, "helper_func_sum_parity": { @@ -2862,22 +2804,26 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:46795" + "server": "http://127.0.0.1:44653" } }, - "helper_func_sum_abs": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44653", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:38:12.549810724Z", - "finished_at": "2026-01-25T02:38:37.706263248Z" + "started_at": "2026-01-27T05:18:23.634574031Z", + "finished_at": "2026-01-27T05:18:51.982958427Z" }, "t_012_spacetime_product_type": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2903,7 +2849,7 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } @@ -2923,7 +2869,7 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:46795" + "server": "http://127.0.0.1:44653" } }, "product_type_row_count": { @@ -2937,8 +2883,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:06.745029551Z", - "finished_at": "2026-01-25T02:36:54.125763993Z" + "started_at": "2026-01-27T05:16:59.448229431Z", + "finished_at": "2026-01-27T05:17:27.128236763Z" }, "t_013_spacetime_sum_type": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2948,7 +2894,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Module.Circle { Radius = radius })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var circle = new Circle { Radius = radius };\n var shape = new Shape.Circle(circle);\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2966,8 +2912,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:06.747511706Z", - "finished_at": "2026-01-25T02:36:55.494494204Z" + "started_at": "2026-01-27T05:15:53.895789582Z", + "finished_at": "2026-01-27T05:16:41.975838933Z" }, "t_014_elementary_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -2985,12 +2931,12 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c20009af6638f1cf0fcf375f9cc76ed28165e2e2439a350311ccfc28427d6966/sql)\n", - "phase": "sql_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200f12d8dff850337b67943a5a8f67328e9d3b10f942bd23f570287d91d47c0/sql)\n", + "phase": "sql" } }, "schema_parity": { @@ -3001,23 +2947,23 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } }, - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200366e8eddaaa9d9fbc4186d386eb7fe6d14a6235a888d8e80e6570a63e0c4/sql)\n", - "phase": "sql" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200faf180e62a1f21ea5ca3f620c6008d5ec2d5129bc32b05c04dd5b75f47de/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:37:22.577850212Z", - "finished_at": "2026-01-25T02:37:49.346231061Z" + "started_at": "2026-01-27T05:16:41.975858659Z", + "finished_at": "2026-01-27T05:17:14.353242716Z" }, "t_015_product_type_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3035,15 +2981,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - }, "product_type_columns_row_parity": { "pass": true, "partial": 1.0, @@ -3055,7 +2992,16 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:46795" + "server": "http://127.0.0.1:44653" + } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" } }, "schema_parity": { @@ -3066,15 +3012,15 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:54.128552650Z", - "finished_at": "2026-01-25T02:37:22.577804857Z" + "started_at": "2026-01-27T05:16:29.391035471Z", + "finished_at": "2026-01-27T05:16:59.448195325Z" }, "t_016_sum_type_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3084,7 +3030,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3096,10 +3042,18 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c20091bdf61869c9af71e01c93d2c9e332381df02a937e4d3e056aec24264f30/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c20071d9e62a2a6c22f20a2da0ea456cbe880014c336f9d020e8cfec84adcb9e/sql)\n", "phase": "sql_golden" } }, + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200a4b0ee58447201552eba1ad36cca2929fdffee122bf1a469ee338eb6e2bf/sql)\n", + "phase": "sql" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3108,23 +3062,15 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d2f1563aa7bd84d2c9a3cffb3b1cfc12f1a874f1cbc2308e20af46e97d2a/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:06.750524280Z", - "finished_at": "2026-01-25T02:36:53.948817678Z" + "started_at": "2026-01-27T05:15:07.936666324Z", + "finished_at": "2026-01-27T05:16:24.208214847Z" }, "t_017_scheduled_columns": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3134,7 +3080,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Intentionally left blank; scheduled invocation acknowledged.\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3142,6 +3088,14 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c2006c5e4f57945fc588ce248080ec09b5beb0a00bd7ea05fdf6f2eb96f189ca/sql)\n", + "phase": "sql" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3150,23 +3104,15 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46795", + "server": "http://127.0.0.1:44653", "tables_diff": null, "tables_equal": true } - }, - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c2005eef72c2621b395ce9ab93b1cbfe93689e33bf4b87526400d0401b7e956a/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:55.494511688Z", - "finished_at": "2026-01-25T02:37:33.685459289Z" + "started_at": "2026-01-27T05:15:07.930390284Z", + "finished_at": "2026-01-27T05:16:29.391011207Z" }, "t_018_constraints": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3184,17 +3130,13 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "constraints_seed_two_rows": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46795", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" } }, "constraints_row_parity_after_seed": { @@ -3208,22 +3150,26 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:46795" + "server": "http://127.0.0.1:44653" } }, - "constraints_seed_two_rows": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44653", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:06.753683987Z", - "finished_at": "2026-01-25T02:36:48.864755984Z" + "started_at": "2026-01-27T05:15:07.927878149Z", + "finished_at": "2026-01-27T05:15:53.895758007Z" }, "t_019_many_to_many": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3233,7 +3179,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3241,17 +3187,13 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46795", - "tables_diff": null, - "tables_equal": true + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" } }, "m2m_has_2_20": { @@ -3263,37 +3205,41 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "memberships_three_rows": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, - "m2m_has_1_20": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44653", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-25T02:37:33.685487289Z", - "finished_at": "2026-01-25T02:38:16.264442645Z" + "started_at": "2026-01-27T05:16:24.208247328Z", + "finished_at": "2026-01-27T05:17:15.959939914Z" }, "t_020_ecs": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3303,7 +3249,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n if (ctx.Db.Entity.Id.Find(1) is Entity)\n {\n }\n else\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n\n if (ctx.Db.Position.EntityId.Find(1) is Position p1)\n {\n p1.X = 0;\n p1.Y = 0;\n ctx.Db.Position.EntityId.Update(p1);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n\n if (ctx.Db.Velocity.EntityId.Find(1) is Velocity v1)\n {\n v1.VX = 1;\n v1.VY = 0;\n ctx.Db.Velocity.EntityId.Update(v1);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n if (ctx.Db.Entity.Id.Find(2) is Entity)\n {\n }\n else\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n\n if (ctx.Db.Position.EntityId.Find(2) is Position p2)\n {\n p2.X = 10;\n p2.Y = 0;\n ctx.Db.Position.EntityId.Update(p2);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n\n if (ctx.Db.Velocity.EntityId.Find(2) is Velocity v2)\n {\n v2.VX = -2;\n v2.VY = 3;\n ctx.Db.Velocity.EntityId.Update(v2);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n int vx = 0;\n int vy = 0;\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n vx = vel.VX;\n vy = vel.VY;\n }\n\n int nx = pos.X + vx;\n int ny = pos.Y + vy;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n var nx = pos.X + vel.Value.VX;\n var ny = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing == null)\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n else\n {\n var updated = existing.Value;\n updated.X = nx;\n updated.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(updated);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3311,11 +3257,24 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity2": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44653", + "tables_diff": null, + "tables_equal": true + } + }, + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200e9fa3233750666c6902cce51a26d6b4f710043a8f780614b5e66e181dd47/sql)\n", "phase": "sql" } }, @@ -3323,43 +3282,30 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200e9fa3233750666c6902cce51a26d6b4f710043a8f780614b5e66e181dd47/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200e9fa3233750666c6902cce51a26d6b4f710043a8f780614b5e66e181dd47/sql)\n", "phase": "sql" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46795", - "tables_diff": null, - "tables_equal": true - } - }, "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46795/v1/database/c200d99860a4d3a82fe10fe9225d15efbbc6940efb69c3eb5375815a750c91f2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44653/v1/database/c200e9fa3233750666c6902cce51a26d6b4f710043a8f780614b5e66e181dd47/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:54.125794979Z", - "finished_at": "2026-01-25T02:37:44.772639848Z" + "started_at": "2026-01-27T05:16:24.211107372Z", + "finished_at": "2026-01-27T05:17:19.055163183Z" }, "t_021_multi_column_index": { "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", @@ -3377,6 +3323,19 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44653", + "tables_diff": null, + "tables_equal": true + } + }, "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, @@ -3395,19 +3354,6 @@ "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46795", - "tables_diff": null, - "tables_equal": true - } - }, "mcindex_seed_count": { "pass": true, "partial": 1.0, @@ -3419,8 +3365,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-25T02:36:48.864793626Z", - "finished_at": "2026-01-25T02:37:35.934293568Z" + "started_at": "2026-01-27T05:15:07.933434340Z", + "finished_at": "2026-01-27T05:16:24.062729088Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index ddcf9423392..cc7e734d8a9 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-25T02:39:25.598Z", + "generated_at": "2026-01-27T05:19:18.787Z", "by_language": { "csharp": { "modes": { @@ -12,10 +12,10 @@ "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 + "passed_tests": 24, + "pass_pct": 88.888885, + "task_pass_equiv": 11.0, + "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, @@ -29,10 +29,10 @@ "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 49, - "pass_pct": 80.327866, - "task_pass_equiv": 18.366667, - "task_pass_pct": 83.48485 + "passed_tests": 46, + "pass_pct": 75.409836, + "task_pass_equiv": 17.366667, + "task_pass_pct": 78.9394 } } } @@ -56,20 +56,20 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 8, - "pass_pct": 23.529411, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 + "total_tests": 30, + "passed_tests": 4, + "pass_pct": 13.333333, + "task_pass_equiv": 1.25, + "task_pass_pct": 12.5 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 13, - "pass_pct": 21.311476, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 + "total_tests": 57, + "passed_tests": 9, + "pass_pct": 15.789474, + "task_pass_equiv": 2.5833335, + "task_pass_pct": 11.742425 } } } @@ -90,19 +90,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 22, - "pass_pct": 64.70588, - "task_pass_equiv": 6.2, - "task_pass_pct": 62.0 + "passed_tests": 23, + "pass_pct": 67.64706, + "task_pass_equiv": 6.5333333, + "task_pass_pct": 65.33333 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 17.2, - "task_pass_pct": 78.18182 + "passed_tests": 49, + "pass_pct": 80.327866, + "task_pass_equiv": 17.533335, + "task_pass_pct": 79.696976 } } } From 9d2b9db4bb5b2971957407dec37c244bb4ab608a Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Tue, 27 Jan 2026 00:01:17 -0600 Subject: [PATCH 29/32] Up to date with master --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6140cc90485..46627105cc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -567,8 +567,8 @@ jobs: - name: Override NuGet packages run: | - dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime - dotnet pack -c Release crates/bindings-csharp/Runtime + dotnet pack crates/bindings-csharp/BSATN.Runtime + dotnet pack crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if @@ -693,8 +693,8 @@ jobs: - name: Override NuGet packages run: | - dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime - dotnet pack -c Release crates/bindings-csharp/Runtime + dotnet pack crates/bindings-csharp/BSATN.Runtime + dotnet pack crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if From 1f8cda4c73fc9d3637b3c46c8b9c961f31cd9738 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Tue, 27 Jan 2026 07:00:11 +0000 Subject: [PATCH 30/32] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 354 +++--- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1596 +++++++++++++------------ docs/llms/docs-benchmark-summary.json | 44 +- 4 files changed, 999 insertions(+), 1013 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 830c04f6c43..072b4f5bb5d 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -8,22 +8,18 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do --- -# Analysis of SpacetimeDB Benchmark Failures +# Analysis of SpacetimeDB Benchmark Test Failures -This analysis focuses on test failures within SpacetimeDB benchmarks, specifically categorized by language and mode, providing actionable insights for documentation improvements to reduce these errors. +## Rust / rustdoc_json Failures ---- - -## Rust / rustdoc_json Failures (7 total) +### Compile/Publish Errors -### Compile/Publish Errors (3 failures) - -#### 1. t_002_scheduled_table -- **The generated code**: +#### t_002_scheduled_table +1. **The generated code**: ```rust - use spacetimedb::{ReducerContext, ScheduleAt, Table}; + use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] + #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -31,21 +27,21 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical scheduled_at: ScheduleAt, } - #[spacetimedb::reducer(init)] + #[reducer(init)] pub fn init(ctx: &ReducerContext) { - ctx.db.tick_timer().insert(TickTimer { + let _ = ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: ScheduleAt::repeat_micros(50_000), + scheduled_at: ScheduleAt::repeat(50_000), }); } - #[spacetimedb::reducer] + #[reducer] pub fn tick(_ctx: &ReducerContext) { - log::info!("tick"); + log::debug!("tick fired"); } ``` - -- **The golden example**: + +2. **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -54,119 +50,47 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical pub struct TickTimer { #[primary_key] #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, + scheduled_id: u64, + scheduled_at: ScheduleAt, } #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { + pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { + Ok(()) } #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + pub fn init(ctx: &ReducerContext) -> Result<(), String> { ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: every_50ms, + scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()), }); + Ok(()) } ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explain the difference**: - - The generated code incorrectly used `ScheduleAt::repeat_micros(50_000)` instead of `ScheduleAt::Interval(Duration::from_millis(50).into())`. - - The reducer function lacked parameters that were required. - -- **Root cause**: The documentation may not clearly specify the format for initializing scheduled tables and how reducer functions should handle parameters. - -- **Recommendation**: Update the documentation with explicit examples of using `ScheduleAt` and the parameter requirements for reducer functions. - ---- - -#### 2. t_007_crud -- **The generated code**: - ```rust - use spacetimedb::{reducer, table, ReducerContext}; - - #[table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, - } - - #[reducer] - pub fn crud(ctx: &ReducerContext) { - let users = ctx.db.user(); - - users.insert(User { - id: 1, - name: "Alice".to_string(), - age: 30, - active: true, - }); - - users.insert(User { - id: 2, - name: "Bob".to_string(), - age: 22, - active: false, - }); - - if let Some(mut u) = users.id().find(1) { - u.name = "Alice2".to_string(); - u.age = 31; - u.active = false; - users.id().update(u); - } - - users.id().delete(&2); - } - ``` - -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - - #[table(name = user)] - pub struct User { - #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, - } - - #[reducer] - pub fn crud(ctx: &ReducerContext) { - ctx.db.user().insert(User { id: 1, name: "Alice".into(), age: 30, active: true }); - ctx.db.user().insert(User { id: 2, name: "Bob".into(), age: 22, active: false }); - ctx.db.user().id().update(User { id: 1, name: "Alice2".into(), age: 31, active: false }); - ctx.db.user().id().delete(2); - } - ``` - -- **The error**: `publish_error: spacetime publish failed (exit=1)` +3. **The error**: + ``` + publish_error: spacetime publish failed (exit=1) + ``` -- **Explain the difference**: - - The generated code did not declare the fields of the `User` struct as public, which is a requirement for fields of structs used with SpacetimeDB. - - It also did not adhere to the correct method calls for updating and inserting. +4. **Explain the difference**: + - The generated code used an incorrect `ScheduleAt` value and omitted returning a `Result<()>` in both the `init` and `tick` reducers. The golden example specifies the `schedule` attribute properly and uses `Duration`. -- **Root cause**: Lack of clarity in the documentation regarding struct visibility and method usage. +5. **Root cause**: + - The documentation is unclear about the proper usage of `ScheduleAt` and return types in reducer functions. -- **Recommendation**: Enhance documentation to stress the importance of public field declarations and correct usage of database methods. +6. **Recommendation**: + - Clarify the types for `ScheduleAt` in the documentation, emphasizing the need to handle the function results, especially for reducers. --- -#### 3. t_017_scheduled_columns -- **The generated code**: +#### t_017_scheduled_columns +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; + use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] + #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -174,22 +98,19 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical scheduled_at: ScheduleAt, } + #[reducer(scheduled)] + pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {} + #[reducer(init)] pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(50_000), - }); - } - } - - #[reducer(scheduled)] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::RepeatMicros(50_000), + }); } ``` - -- **The golden example**: + +2. **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -203,8 +124,7 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical } #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { - } + pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} #[reducer(init)] pub fn init(ctx: &ReducerContext) { @@ -216,23 +136,28 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical } ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` +3. **The error**: + ``` + publish_error: spacetime publish failed (exit=1) + ``` + +4. **Explain the difference**: + - The generated code incorrectly uses `ScheduleAt::RepeatMicros`. The golden example uses `Duration` properly, and the reducer method signatures are defined accurately. -- **Explain the difference**: - - The LLM incorrectly specified the scheduled column and did not update the initialization logic for accurate type conversion. - -- **Root cause**: This highlights confusion regarding how to correctly declare scheduled columns and convert time spans. +5. **Root cause**: + - The documentation lacks clear usage examples for handling scheduled columns correctly. -- **Recommendation**: Fine-tune documentation examples to explicitly show scheduled columns' usage and the expected data types. +6. **Recommendation**: + - Update the documentation to emphasize accurate definitions and expected types in reducer functions and their parameters. --- -### Other Failures (4 failures) +### Other Failures -#### t_004_insert -- **The generated code**: +#### t_001_basic_tables +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::{table, reducer, ReducerContext}; #[table(name = user)] pub struct User { @@ -243,15 +168,31 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical active: bool, } - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { - ctx.db.user().insert(User { id, name, age, active }); + #[table(name = product)] + pub struct Product { + #[primary_key] + id: i32, + title: String, + price: f32, + in_stock: bool, } + + #[table(name = note)] + pub struct Note { + #[primary_key] + id: i32, + body: String, + rating: i64, + pinned: bool, + } + + #[reducer(init)] + pub fn init(_ctx: &ReducerContext) {} ``` -- **The golden example**: +2. **The golden example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::table; #[table(name = user)] pub struct User { @@ -262,79 +203,113 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical pub active: bool, } - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { - ctx.db.user().insert(User { id, name, age, active }); - Ok(()) + #[table(name = product)] + pub struct Product { + #[primary_key] + pub id: i32, + pub title: String, + pub price: f32, + pub in_stock: bool, + } + + #[table(name = note)] + pub struct Note { + #[primary_key] + pub id: i32, + pub body: String, + pub rating: i64, + pub pinned: bool, } ``` -- **The error**: `data_parity_insert_user: spacetime sql failed: no such table: user` +3. **The error**: + ``` + schema_parity: reducers differ - expected [], got ["init()"] + ``` + +4. **Explain the difference**: + - The primary key fields don't have `pub` visibility in the generated code which results in schema mismatch. -- **Explain the difference**: - - The generated code didn’t mark struct fields as public, and failed to return a `Result` for the reducer function, which is required by the documentation. +5. **Root cause**: + - The documentation does not specify visibility modifiers adequately. -- **Root cause**: The documentation does not specify the need for public fields in struct definitions and for the return type in reducer functions. +6. **Recommendation**: + - Explicitly mention the need for public visibility on fields in table definitions in the documentation. -- **Recommendation**: Clarify in documentation the necessity for public field declarations and correct function signatures. +--- -#### t_011_helper_function -- **The generated code**: +#### t_012_spacetime_product_type (similar to t_013 and t_014) +1. **The generated code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; + use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + + #[derive(SpacetimeType, Clone)] + struct Score { + left: i32, + right: i32, + } #[table(name = result)] - pub struct ResultRow { + struct ResultRow { #[primary_key] id: i32, - sum: i32, - } - - fn add(a: i32, b: i32) -> i32 { - a + b + value: Score, } #[reducer] - fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - let sum = add(a, b); - ctx.db.result().insert(ResultRow { id, sum }); + fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { + ctx.db.result().insert(ResultRow { + id, + value: Score { left, right }, + }); } ``` -- **The golden example**: +2. **The golden example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + + #[derive(SpacetimeType, Clone, Debug)] + pub struct Score { + pub left: i32, + pub right: i32, + } #[table(name = result)] pub struct ResultRow { #[primary_key] pub id: i32, - pub sum: i32, + pub value: Score, } - fn add(a: i32, b: i32) -> i32 { a + b } - #[reducer] - pub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - ctx.db.result().insert(ResultRow { id, sum: add(a, b) }); + pub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { + ctx.db.result().insert(ResultRow { id, value: Score { left, right } }); } ``` -- **The error**: `helper_func_sum_parity: spacetime sql failed: no such table: result` +3. **The error**: + ``` + product_type_row_parity: spacetime sql failed: ... no such table: result + ``` -- **Explain the difference**: - - Missing public modifiers for struct fields and incorrect reducer function signature. +4. **Explain the difference**: + - Missing `pub` visibility on struct fields in the generated code prevents successful table creation and querying. -- **Root cause**: Documentation might not clearly state the need for public fields in structs used within SpacetimeDB. +5. **Root cause**: + - The importance of public visibility for struct fields in SpacetimeDB is not emphasized. -- **Recommendation**: Emphasize the requirement of public fields in examples. +6. **Recommendation**: + - Include specific examples in the documentation highlighting struct field visibility requirements. --- -### C# / docs Failures (5 total) +## C# / docs Failures + +### Other Failures -#### 1. t_014_elementary_columns -- **The generated code**: +#### t_014_elementary_columns +1. **The generated code**: ```csharp using SpacetimeDB; @@ -370,7 +345,7 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical } ``` -- **The golden example**: +2. **The golden example**: ```csharp using SpacetimeDB; @@ -391,8 +366,7 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical [Reducer] public static void Seed(ReducerContext ctx) { - ctx.Db.Primitive.Insert(new Primitive - { + ctx.Db.Primitive.Insert(new Primitive { Id = 1, Count = 2, Total = 3000000000, @@ -405,20 +379,20 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical } ``` -- **The error**: `no such table: primitive` - -- **Explain the difference**: Field visibility was not explicitly made public in the generated code, which is a requirement for SpacetimeDB. +3. **The error**: + ``` + no such table: primitive + ``` -- **Root cause**: The documentation may lack clarity regarding field visibility and access modifiers. +4. **Explain the difference**: + - The usage of attributes in the generated code was inconsistent (e.g., missing `public` for the `Id` field), causing schema definition issues. -- **Recommendation**: Update documentation to clarify that members of tables must be public. +5. **Root cause**: + - Insufficient clarity on the correct usage of access modifiers and attributes in class definitions. ---- - -(Continue this format for the remaining C# failures...) +6. **Recommendation**: + - Provide a detailed section in the documentation on defining tables and reducers with clear examples of access modifiers and parameter attributes. --- -### Conclusion - -This comprehensive analysis of SpacetimeDB benchmark test failures highlights key areas where the documentation can improve self-guidance for developers. Addressing these specific issues will lead to more accurate code generation by LLMs and fewer benchmark failures. +This analysis highlights several key areas in the documentation that need improvement, especially regarding visibility modifiers, function returns, and proper formatting to assist users in avoiding common pitfalls during coding. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index de9b098fefa..516b0c6d518 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 20/27 | 76.4% ⬆️ +2.1% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% | -| Rust | rustdoc_json | **total** | 46/61 | **75.9%** ⬆️ +1.1% | +| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +6.9% | +| Rust | rustdoc_json | schema | 23/34 | 65.3% ⬇️ -10.0% | +| Rust | rustdoc_json | **total** | 48/61 | **75.2%** ⬇️ -0.8% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | -| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | -| C# | docs | schema | 21/34 | 63.7% | -| C# | docs | **total** | 48/61 | **83.5%** ⬆️ +4.5% | +| Rust | docs | schema | 8/34 | 20.5% | +| Rust | docs | **total** | 13/61 | **15.4%** | +| C# | docs | basics | 27/27 | 100.0% | +| C# | docs | schema | 23/34 | 68.3% ⬆️ +4.7% | +| C# | docs | **total** | 50/61 | **85.6%** ⬆️ +2.1% | _Compared against master branch baseline_ -Generated at: 2026-01-26T15:42:12.228Z +Generated at: 2026-01-27T06:59:15.063Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 51adf7934c1..723fe1ec519 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,21 +5,21 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n let _ = count;\n}\n\n#[reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n let _ = name;\n}\n\n#[reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n let _ = (count, name);\n}\n\n#[reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n let _ = (active, ratio, label);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,25 +35,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975549166Z", - "finished_at": "2026-01-26T15:31:47.073747240Z" + "started_at": "2026-01-27T06:50:51.888522281Z", + "finished_at": "2026-01-27T06:51:13.665986517Z" }, "t_001_basic_tables": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}\n\n#[reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -62,32 +62,37 @@ "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { "golden_db": "basics-t-001-basic-tables-golden", "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "reducers_diff": { + "only_golden": [], + "only_llm": [ + "init()" + ] + }, + "reducers_equal": false, + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.974869540Z", - "finished_at": "2026-01-26T15:31:51.422745193Z" + "started_at": "2026-01-27T06:50:53.584708769Z", + "finished_at": "2026-01-27T06:51:23.711742366Z" }, "t_002_scheduled_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::info!(\"tick\");\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n let _ = ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::debug!(\"tick fired\");\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +104,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling serde_json v1.0.149\n Compiling cc v1.2.54\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:44\n |\n15 | let _ = ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:24\n |\n15 | let _ = ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975688871Z", - "finished_at": "2026-01-26T15:32:30.863567064Z" + "started_at": "2026-01-27T06:51:23.712041725Z", + "finished_at": "2026-01-27T06:52:02.618334739Z" }, "t_003_struct_in_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -132,25 +137,25 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975098187Z", - "finished_at": "2026-01-26T15:31:55.504039476Z" + "started_at": "2026-01-27T06:51:28.338644417Z", + "finished_at": "2026-01-27T06:52:09.912075445Z" }, "t_004_insert": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -166,33 +171,44 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } }, "data_parity_insert_user": { - "pass": false, - "partial": 0.0, + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002173ebf56f08f752bb2ca55d0547329e19fb14624413a7089817795d01ab/sql)\n", - "phase": "sql_golden" + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "insert_user", + "server": "http://127.0.0.1:41979" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975407164Z", - "finished_at": "2026-01-26T15:31:42.241214465Z" + "started_at": "2026-01-27T06:50:51.888753023Z", + "finished_at": "2026-01-27T06:51:12.353481558Z" }, "t_005_update": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -200,23 +216,17 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:34081" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41979", + "tables_diff": null, + "tables_equal": true } }, "seed_users_row": { @@ -226,26 +236,32 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "schema_parity": { + "data_parity_update_user": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:41979" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:42.241280124Z", - "finished_at": "2026-01-26T15:32:22.818349314Z" + "started_at": "2026-01-27T06:50:48.158462685Z", + "finished_at": "2026-01-27T06:51:23.208722152Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -260,15 +276,6 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -284,25 +291,34 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } + }, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:49.227905993Z", - "finished_at": "2026-01-26T15:32:24.290487857Z" + "started_at": "2026-01-27T06:50:53.346121053Z", + "finished_at": "2026-01-27T06:51:23.650848109Z" }, "t_007_crud": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_007_crud", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n if let Some(mut u) = users.id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n users.id().update(u);\n }\n\n users.id().delete(&2);\n}", + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false });\n ctx.db.user().id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -310,28 +326,65 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling generic-array v0.14.7\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:17:11\n |\n17 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | users.try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:24:11\n |\n24 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | users.try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } + }, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41979", + "tables_diff": null, + "tables_equal": true + } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:41979" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975950122Z", - "finished_at": "2026-01-26T15:32:16.012153929Z" + "started_at": "2026-01-27T06:51:23.711778229Z", + "finished_at": "2026-01-27T06:52:03.144704983Z" }, "t_008_index_lookup": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -339,13 +392,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -359,7 +405,7 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:41979" } }, "schema_parity": { @@ -370,25 +416,32 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } + }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.511640110Z", - "finished_at": "2026-01-26T15:32:25.220589761Z" + "started_at": "2026-01-27T06:51:23.208769752Z", + "finished_at": "2026-01-27T06:52:11.362807201Z" }, "t_009_init": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let t = ctx.db.user();\n t.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n t.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -396,26 +449,22 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, - "schema_parity": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "init_seed_alice": { @@ -427,22 +476,26 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "init_seed_bob": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41979", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.073777338Z", - "finished_at": "2026-01-26T15:32:23.492928074Z" + "started_at": "2026-01-27T06:51:12.353525885Z", + "finished_at": "2026-01-27T06:51:33.962752131Z" }, "t_010_connect": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -465,24 +518,24 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975818475Z", - "finished_at": "2026-01-26T15:31:49.227873175Z" + "started_at": "2026-01-27T06:51:13.666166477Z", + "finished_at": "2026-01-27T06:51:46.771488089Z" }, "t_011_helper_function": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, + "passed_tests": 3, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -491,6 +544,15 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -499,42 +561,43 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } }, "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20018816d0c78b41d0a1ebdd62b788e682bdd9fd17555d6ba26e941ae120e74/sql)\n", - "phase": "sql_golden" - } - }, - "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM result WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:41979" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975246967Z", - "finished_at": "2026-01-26T15:31:47.511599186Z" + "started_at": "2026-01-27T06:51:03.641953394Z", + "finished_at": "2026-01-27T06:51:30.134179123Z" }, "t_012_spacetime_product_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -542,22 +605,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { + "product_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:34081" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -568,33 +622,32 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } }, - "product_type_row_count": { - "pass": true, - "partial": 1.0, + "product_type_row_parity": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c2002a065914af17b5096ed2a6a604025ae45aba6ea5fa1c01dff1217a37e25a/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:24.290533494Z", - "finished_at": "2026-01-26T15:32:53.440043766Z" + "started_at": "2026-01-27T06:50:17.584691629Z", + "finished_at": "2026-01-27T06:50:51.888486341Z" }, "t_013_spacetime_sum_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, + "passed_tests": 2, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", @@ -603,21 +656,21 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { + "sum_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:34081" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + } + }, + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c2002487eb975b79e550304ea5a58e47940b9602e656a41c574b126959eb89f2/sql)\n", + "phase": "sql_golden" } }, "schema_parity": { @@ -628,34 +681,25 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:29.610801727Z", - "finished_at": "2026-01-26T15:33:11.012437922Z" + "started_at": "2026-01-27T06:50:17.584911361Z", + "finished_at": "2026-01-27T06:50:41.249915826Z" }, "t_014_elementary_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -663,56 +707,56 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, + "args": [], + "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:41979" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { "golden_db": "schema-t-014-elementary-columns-golden", "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } }, - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": true, "partial": 1.0, "notes": { - "args": [], - "golden_db": "schema-t-014-elementary-columns-golden", - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:25.220640456Z", - "finished_at": "2026-01-26T15:32:53.137088661Z" + "started_at": "2026-01-27T06:50:39.733763327Z", + "finished_at": "2026-01-27T06:51:03.641921076Z" }, "t_015_product_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -720,17 +764,13 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_columns_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, "product_type_columns_row_parity": { @@ -744,32 +784,36 @@ "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", "query": "SELECT id, home, work, pos FROM profile WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:41979" } }, - "product_type_columns_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41979", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:30.863583165Z", - "finished_at": "2026-01-26T15:33:07.055207496Z" + "started_at": "2026-01-27T06:50:17.585633119Z", + "finished_at": "2026-01-27T06:50:53.346086537Z" }, "t_016_sum_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawing = ctx.db.drawing();\n if drawing.count() == 0 {\n drawing.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -777,49 +821,49 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20084ed1de024b608d5d036674102ffea5602feb56e936989c5737d1fa347f1/sql)\n", - "phase": "sql_golden" + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41979", + "tables_diff": null, + "tables_equal": true } }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c200f0f8cf7ce43e0b888fecfda5572f02aac26af6bea4ac6867cab46bd5e36f/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200bcd11c4bfa72c0645f09b101b36234eebc9756ade44cf450379833ce5fbe/sql)\n", "phase": "sql" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c2004a471e90b0846f7b645bf82b0db14dde1be711b8790034374d168a4650c4/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:47.540914487Z", - "finished_at": "2026-01-26T15:33:39.147225272Z" + "started_at": "2026-01-27T06:50:17.585067459Z", + "finished_at": "2026-01-27T06:50:39.733721161Z" }, "t_017_scheduled_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -831,24 +875,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:22:11\n |\n22 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.34\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling smallvec v1.15.1\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:17:32\n |\n17 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:17:12\n |\n17 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:19:35\n |\n19 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 6 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:23.493005867Z", - "finished_at": "2026-01-26T15:33:38.753749775Z" + "started_at": "2026-01-27T06:50:17.585775367Z", + "finished_at": "2026-01-27T06:51:28.338625411Z" }, "t_018_constraints": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n let ids: Vec = account.iter().map(|r| r.id).collect();\n for id in ids {\n account.id().delete(&id);\n }\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db\n .account()\n .insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n ctx.db\n .account()\n .insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -856,20 +900,6 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-018-constraints-golden", - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" - } - }, "constraints_seed_two_rows": { "pass": true, "partial": 1.0, @@ -879,6 +909,14 @@ "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, + "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200a1d038e9530cfe7f807e664c1a64ca88402354328dba67878cae3bbe988d/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -887,25 +925,25 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:16.012174316Z", - "finished_at": "2026-01-26T15:32:54.091817951Z" + "started_at": "2026-01-27T06:50:17.585216166Z", + "finished_at": "2026-01-27T06:50:53.584680110Z" }, "t_019_many_to_many": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_019_many_to_many", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exact dataset\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -921,61 +959,61 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" } }, - "m2m_has_2_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "memberships_three_rows": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } }, - "m2m_has_1_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:22.818452395Z", - "finished_at": "2026-01-26T15:33:12.862647549Z" + "started_at": "2026-01-27T06:50:41.249961607Z", + "finished_at": "2026-01-27T06:51:39.656649059Z" }, "t_020_ecs": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(p.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: p.entity_id, x: nx, y: ny });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let entities = ctx.db.entity();\n entities.id().delete(&1);\n entities.id().delete(&2);\n entities.insert(Entity { id: 1 });\n entities.insert(Entity { id: 2 });\n\n let positions = ctx.db.position();\n positions.entity_id().delete(&1);\n positions.entity_id().delete(&2);\n positions.insert(Position { entity_id: 1, x: 0, y: 0 });\n positions.insert(Position { entity_id: 2, x: 10, y: 0 });\n\n let velocities = ctx.db.velocity();\n velocities.entity_id().delete(&1);\n velocities.entity_id().delete(&2);\n velocities.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n velocities.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let next = ctx.db.next_position();\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let np = NextPosition {\n entity_id: pos.entity_id,\n x: pos.x + vel.vx,\n y: pos.y + vel.vy,\n };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n next.entity_id().update(np);\n } else {\n next.insert(np);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -983,27 +1021,35 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", "phase": "sql" } }, @@ -1015,33 +1061,25 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:51.422780227Z", - "finished_at": "2026-01-26T15:32:47.540737141Z" + "started_at": "2026-01-27T06:50:17.585495939Z", + "finished_at": "2026-01-27T06:50:48.158397528Z" }, "t_021_multi_column_index": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.log();\n\n // Clear existing rows to ensure the table contains exactly the seeded rows\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let logs = ctx.db.log();\n logs.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n logs.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n logs.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1049,13 +1087,13 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, "schema_parity": { @@ -1066,7 +1104,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:41979", "tables_diff": null, "tables_equal": true } @@ -1080,19 +1118,19 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, - "mcindex_lookup_u7_d1": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:55.504076208Z", - "finished_at": "2026-01-26T15:32:29.610771248Z" + "started_at": "2026-01-27T06:50:17.585358828Z", + "finished_at": "2026-01-27T06:50:51.755543802Z" } } } @@ -1114,7 +1152,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1132,8 +1170,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.378312906Z", - "finished_at": "2026-01-26T15:34:06.312973975Z" + "started_at": "2026-01-27T06:52:56.984072629Z", + "finished_at": "2026-01-27T06:53:17.546073572Z" }, "t_001_basic_tables": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1143,7 +1181,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1161,8 +1199,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.366156833Z", - "finished_at": "2026-01-26T15:34:21.141281364Z" + "started_at": "2026-01-27T06:53:10.563502758Z", + "finished_at": "2026-01-27T06:53:33.840901265Z" }, "t_002_scheduled_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1172,7 +1210,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(std::time::Duration::from_micros(50_000).into()),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1190,8 +1228,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.381646420Z", - "finished_at": "2026-01-26T15:35:08.176674329Z" + "started_at": "2026-01-27T06:53:22.577942919Z", + "finished_at": "2026-01-27T06:53:54.913192801Z" }, "t_003_struct_in_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1201,7 +1239,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1219,8 +1257,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.368608920Z", - "finished_at": "2026-01-26T15:34:21.997549378Z" + "started_at": "2026-01-27T06:53:30.387408627Z", + "finished_at": "2026-01-27T06:54:01.833607799Z" }, "t_004_insert": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1256,8 +1294,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.375009734Z", - "finished_at": "2026-01-26T15:35:05.975054748Z" + "started_at": "2026-01-27T06:52:57.589191552Z", + "finished_at": "2026-01-27T06:53:14.941899668Z" }, "t_005_update": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1275,14 +1313,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1299,11 +1329,19 @@ "phase": "sql_golden", "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } + }, + "data_parity_update_user": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:06.313026786Z", - "finished_at": "2026-01-26T15:34:38.434768448Z" + "started_at": "2026-01-27T06:52:52.073327194Z", + "finished_at": "2026-01-27T06:53:20.830286213Z" }, "t_006_delete": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1313,7 +1351,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -1321,13 +1359,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, + "seed_users_row": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -1338,19 +1376,19 @@ "phase": "describe_golden" } }, - "seed_users_row": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:38.434794482Z", - "finished_at": "2026-01-26T15:35:08.669675470Z" + "started_at": "2026-01-27T06:52:59.414714306Z", + "finished_at": "2026-01-27T06:53:22.577916155Z" }, "t_007_crud": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1360,7 +1398,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1368,12 +1406,12 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id1_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" } }, "crud_row_id2_deleted": { @@ -1385,27 +1423,27 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "schema_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "crud_total_count_one": { + "crud_row_id1_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.388241938Z", - "finished_at": "2026-01-26T15:35:08.142795838Z" + "started_at": "2026-01-27T06:53:20.830310256Z", + "finished_at": "2026-01-27T06:53:46.790076084Z" }, "t_008_index_lookup": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1415,7 +1453,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1423,14 +1461,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "describe_golden" - } - }, "seed_user_row": { "pass": false, "partial": 0.0, @@ -1447,11 +1477,19 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", "phase": "call_reducer_golden" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.997572287Z", - "finished_at": "2026-01-26T15:34:53.278840948Z" + "started_at": "2026-01-27T06:53:17.546096858Z", + "finished_at": "2026-01-27T06:54:27.343881340Z" }, "t_009_init": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1461,7 +1499,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1506,8 +1544,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.141305493Z", - "finished_at": "2026-01-26T15:34:43.189086037Z" + "started_at": "2026-01-27T06:53:15.501368558Z", + "finished_at": "2026-01-27T06:53:36.936217110Z" }, "t_010_connect": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1517,7 +1555,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".into(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".into(),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1535,8 +1573,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.384904874Z", - "finished_at": "2026-01-26T15:35:08.049424018Z" + "started_at": "2026-01-27T06:53:16.023598601Z", + "finished_at": "2026-01-27T06:53:36.037355016Z" }, "t_011_helper_function": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1546,7 +1584,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1581,8 +1619,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.371740140Z", - "finished_at": "2026-01-26T15:35:03.413658925Z" + "started_at": "2026-01-27T06:53:14.941927553Z", + "finished_at": "2026-01-27T06:53:34.278454621Z" }, "t_012_spacetime_product_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1592,7 +1630,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1600,13 +1638,12 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } }, "product_type_row_parity": { @@ -1617,18 +1654,19 @@ "phase": "call_reducer_golden" } }, - "schema_parity": { + "product_type_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.176748927Z", - "finished_at": "2026-01-26T15:35:33.331707923Z" + "started_at": "2026-01-27T06:52:11.583745571Z", + "finished_at": "2026-01-27T06:52:38.781371876Z" }, "t_013_spacetime_sum_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1638,7 +1676,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1673,8 +1711,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.669696395Z", - "finished_at": "2026-01-26T15:35:36.590766489Z" + "started_at": "2026-01-27T06:52:11.586224627Z", + "finished_at": "2026-01-27T06:52:59.414653680Z" }, "t_014_elementary_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1684,7 +1722,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1692,6 +1730,15 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1707,20 +1754,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "call_reducer_golden" } - }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.180313713Z", - "finished_at": "2026-01-26T15:35:31.695987649Z" + "started_at": "2026-01-27T06:52:38.781403919Z", + "finished_at": "2026-01-27T06:52:56.984044747Z" }, "t_015_product_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1730,7 +1768,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let row = Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n };\n if ctx.db.profile().id().find(1).is_some() {\n ctx.db.profile().id().update(row);\n } else {\n ctx.db.profile().insert(row);\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1747,26 +1785,26 @@ "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" - } + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" + } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:17.784622487Z", - "finished_at": "2026-01-26T15:35:56.892022012Z" + "started_at": "2026-01-27T06:52:11.602521729Z", + "finished_at": "2026-01-27T06:52:52.073252976Z" }, "t_016_sum_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1776,7 +1814,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.drawing().try_insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().count() == 0 {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1784,34 +1822,34 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c200e332ee39012f68748e8e0058b1df01bfd4bd42fa50c7430f6e03a52a13d7/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200b059e9edd74835a85326a453facd15b98fd18c99a63c3220e05de170a610/sql)\n", "phase": "sql" } }, - "sum_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:20.422565504Z", - "finished_at": "2026-01-26T15:35:53.522136009Z" + "started_at": "2026-01-27T06:52:11.589253371Z", + "finished_at": "2026-01-27T06:52:43.647842952Z" }, "t_017_scheduled_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1821,7 +1859,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1848,8 +1886,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.142877410Z", - "finished_at": "2026-01-26T15:35:53.117891027Z" + "started_at": "2026-01-27T06:52:11.605807485Z", + "finished_at": "2026-01-27T06:52:57.589117985Z" }, "t_018_constraints": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1859,7 +1897,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().id().delete(&1);\n ctx.db.account().id().delete(&2);\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.account().iter() {\n ctx.db.account().id().delete(&row.id);\n }\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1867,13 +1905,12 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "constraints_seed_two_rows": { + "constraints_row_parity_after_seed": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "call_reducer_golden" } }, "schema_parity": { @@ -1884,18 +1921,19 @@ "phase": "describe_golden" } }, - "constraints_row_parity_after_seed": { + "constraints_seed_two_rows": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:03.413773673Z", - "finished_at": "2026-01-26T15:35:40.991202532Z" + "started_at": "2026-01-27T06:52:11.592579007Z", + "finished_at": "2026-01-27T06:53:15.501219686Z" }, "t_019_many_to_many": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1905,7 +1943,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exactly these rows are present after seeding.\n {\n let keys: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for k in keys {\n ctx.db.membership().id().delete(&k);\n }\n }\n {\n let keys: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for k in keys {\n ctx.db.user().user_id().delete(&k);\n }\n }\n {\n let keys: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for k in keys {\n ctx.db.group().group_id().delete(&k);\n }\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -1913,31 +1951,22 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "m2m_has_2_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "m2m_has_1_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, "schema_parity": { @@ -1948,19 +1977,28 @@ "phase": "describe_golden" } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:05.975134623Z", - "finished_at": "2026-01-26T15:35:55.889702970Z" + "started_at": "2026-01-27T06:52:43.647889605Z", + "finished_at": "2026-01-27T06:53:30.387369943Z" }, "t_020_ecs": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1970,7 +2008,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n let id1 = 1;\n if ctx.db.entity().id().find(id1).is_none() {\n ctx.db.entity().insert(Entity { id: id1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n let id2 = 2;\n if ctx.db.entity().id().find(id2).is_none() {\n ctx.db.entity().insert(Entity { id: id2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut pos) = ctx.db.position().entity_id().find(&1) {\n pos.x = 0;\n pos.y = 0;\n ctx.db.position().entity_id().update(pos);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut vel) = ctx.db.velocity().entity_id().find(&1) {\n vel.vx = 1;\n vel.vy = 0;\n ctx.db.velocity().entity_id().update(vel);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut pos) = ctx.db.position().entity_id().find(&2) {\n pos.x = 10;\n pos.y = 0;\n ctx.db.position().entity_id().update(pos);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut vel) = ctx.db.velocity().entity_id().find(&2) {\n vel.vx = -2;\n vel.vy = 3;\n ctx.db.velocity().entity_id().update(vel);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(&pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(&pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1982,46 +2020,46 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", "phase": "sql" } }, - "schema_parity": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", + "phase": "sql" } }, - "ecs_next_pos_entity2": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", - "phase": "sql" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", + "phase": "describe_golden" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:43.189115873Z", - "finished_at": "2026-01-26T15:35:20.422539334Z" + "started_at": "2026-01-27T06:52:11.599170875Z", + "finished_at": "2026-01-27T06:53:10.563412215Z" }, "t_021_multi_column_index": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -2031,7 +2069,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2039,6 +2077,15 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -2047,15 +2094,6 @@ "phase": "describe_golden" } }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2065,19 +2103,19 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "mcindex_lookup_u7_d2": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:53.278868473Z", - "finished_at": "2026-01-26T15:35:17.784590880Z" + "started_at": "2026-01-27T06:52:11.595885336Z", + "finished_at": "2026-01-27T06:53:16.023495614Z" } } } @@ -2298,15 +2336,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:48.351519518Z", - "finished_at": "2026-01-26T15:39:15.916317553Z" + "started_at": "2026-01-27T06:57:18.004134655Z", + "finished_at": "2026-01-27T06:57:38.475437844Z" }, "t_001_basic_tables": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2316,7 +2354,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2332,15 +2370,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.611034643Z", - "finished_at": "2026-01-26T15:39:10.630797919Z" + "started_at": "2026-01-27T06:57:38.475553797Z", + "finished_at": "2026-01-27T06:58:11.613580259Z" }, "t_002_scheduled_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2350,7 +2388,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2366,15 +2404,15 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:55.763350829Z", - "finished_at": "2026-01-26T15:39:35.117748962Z" + "started_at": "2026-01-27T06:58:26.083760618Z", + "finished_at": "2026-01-27T06:59:15.017759462Z" }, "t_003_struct_in_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2384,7 +2422,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -2400,15 +2438,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.614826701Z", - "finished_at": "2026-01-26T15:38:48.351485744Z" + "started_at": "2026-01-27T06:58:26.088195068Z", + "finished_at": "2026-01-27T06:58:52.521561886Z" }, "t_004_insert": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2434,7 +2472,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } @@ -2455,13 +2493,13 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.624534968Z", - "finished_at": "2026-01-26T15:38:55.763316545Z" + "started_at": "2026-01-27T06:57:35.297807470Z", + "finished_at": "2026-01-27T06:57:57.734182007Z" }, "t_005_update": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2471,7 +2509,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n throw new Exception(\"User not found\");\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2479,19 +2517,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -2508,7 +2533,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" } }, "seed_users_row": { @@ -2517,11 +2542,24 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:15.916349271Z", - "finished_at": "2026-01-26T15:39:57.685388275Z" + "started_at": "2026-01-27T06:57:03.966445912Z", + "finished_at": "2026-01-27T06:57:36.559661464Z" }, "t_006_delete": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2531,7 +2569,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2563,15 +2601,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:51.394765047Z", - "finished_at": "2026-01-26T15:40:20.535132716Z" + "started_at": "2026-01-27T06:57:36.559691624Z", + "finished_at": "2026-01-27T06:57:59.521935866Z" }, "t_007_crud": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2581,7 +2619,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n // delete Id=2\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u)\n {\n u.Name = \"Alice2\";\n u.Age = 31;\n u.Active = false;\n ctx.Db.User.Id.Update(u);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2589,6 +2627,15 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2597,7 +2644,7 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } @@ -2622,22 +2669,13 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:41793" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + "server": "http://127.0.0.1:34133" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:10.630849129Z", - "finished_at": "2026-01-26T15:39:51.394722435Z" + "started_at": "2026-01-27T06:58:21.941006948Z", + "finished_at": "2026-01-27T06:58:55.852358903Z" }, "t_008_index_lookup": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2647,7 +2685,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Id.Delete(id);\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n else\n {\n throw new System.Exception(\"User not found\");\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2655,19 +2693,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "seed_user_row": { "pass": true, "partial": 1.0, @@ -2688,13 +2713,26 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:35.117796353Z", - "finished_at": "2026-01-26T15:40:10.865109962Z" + "started_at": "2026-01-27T06:58:11.613650056Z", + "finished_at": "2026-01-27T06:58:41.445956326Z" }, "t_009_init": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2704,7 +2742,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2712,22 +2750,26 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, - "init_seed_bob": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true } }, "init_seed_alice": { @@ -2739,23 +2781,19 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "schema_parity": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:33.038774271Z", - "finished_at": "2026-01-26T15:40:09.097960805Z" + "started_at": "2026-01-27T06:57:57.734214775Z", + "finished_at": "2026-01-27T06:58:26.083715734Z" }, "t_010_connect": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2765,7 +2803,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2781,15 +2819,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:07.129948444Z", - "finished_at": "2026-01-26T15:39:33.038733830Z" + "started_at": "2026-01-27T06:57:59.521963264Z", + "finished_at": "2026-01-27T06:58:25.875525433Z" }, "t_011_helper_function": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2799,7 +2837,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2807,19 +2845,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "helper_func_sum_parity": { "pass": true, "partial": 1.0, @@ -2835,7 +2860,7 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" } }, "helper_func_sum_abs": { @@ -2846,11 +2871,24 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.619502537Z", - "finished_at": "2026-01-26T15:39:07.129901962Z" + "started_at": "2026-01-27T06:57:46.952002442Z", + "finished_at": "2026-01-27T06:58:21.940968232Z" }, "t_012_spacetime_product_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2860,7 +2898,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right },\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(SpacetimeDB.ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2868,17 +2906,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, "product_type_row_parity": { @@ -2896,22 +2930,26 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" } }, - "product_type_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:45.264170257Z", - "finished_at": "2026-01-26T15:41:10.281722543Z" + "started_at": "2026-01-27T06:55:44.828454843Z", + "finished_at": "2026-01-27T06:56:33.700343298Z" }, "t_013_spacetime_sum_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2921,7 +2959,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2929,6 +2967,23 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetCircle", + "server": "http://127.0.0.1:34133" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2937,7 +2992,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } @@ -2950,28 +3005,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:41793" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.281769568Z", - "finished_at": "2026-01-26T15:42:04.928512771Z" + "started_at": "2026-01-27T06:55:44.831383815Z", + "finished_at": "2026-01-27T06:56:29.034824544Z" }, "t_014_elementary_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2989,39 +3027,39 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20015ce3d6d6c97d4d92483e7212bbbdc2017cebf40569b0de2758a8c06d8c9/sql)\n", - "phase": "sql" + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true } }, "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2002dc67d9e1fd4e2f7be96328c9cadfe424ab52e14114ffe15bf0808847523/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34133/v1/database/c200056226dd3531b2838b8fd62f46de0e039bae5698b7d3f872d2876e715c0b/sql)\n", "phase": "sql_golden" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34133/v1/database/c20027d7901c8b2fd9264fe39a4fe2d0dd61f89e9f587473081630bdfad52cf6/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:59.748869772Z", - "finished_at": "2026-01-26T15:41:30.294907546Z" + "started_at": "2026-01-27T06:56:55.213478403Z", + "finished_at": "2026-01-27T06:57:18.004106999Z" }, "t_015_product_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3031,7 +3069,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n var home = new Address { Street = \"1 Main\", Zip = 11111 };\n var work = new Address { Street = \"2 Broad\", Zip = 22222 };\n var pos = new Position { X = 7, Y = 9 };\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = home,\n Work = work,\n Pos = pos\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3056,7 +3094,7 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } @@ -3072,23 +3110,23 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.284990504Z", - "finished_at": "2026-01-26T15:41:51.914201320Z" + "started_at": "2026-01-27T06:56:37.102294805Z", + "finished_at": "2026-01-27T06:57:01.039671444Z" }, "t_016_sum_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_016_sum_type_columns", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3096,39 +3134,18 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2007921ce30371e511ce337c700f912b2dc1842029ec51095bcee487a1fb5a2/sql)\n", - "phase": "sql_golden" - } - }, - "sum_type_columns_row_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20008b060dbbbe2ff941855e14b04bb0c88b907fca414a2d82874c574b1cf60/sql)\n", - "phase": "sql" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:12.893136011Z", - "finished_at": "2026-01-26T15:42:12.178253367Z" + "started_at": "2026-01-27T06:55:44.834901411Z", + "finished_at": "2026-01-27T06:56:37.102276639Z" }, "t_017_scheduled_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3138,7 +3155,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3146,14 +3163,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20077e1b6738482c8dd0108623bedd0c70f80fb40fb09fda1f12bbe2c6972bf/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3162,15 +3171,23 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } + }, + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34133/v1/database/c200f990cf75c7ddd2f366f0b63fda9167e82ca5207621982daab4a066ddde2e/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:27.963020126Z", - "finished_at": "2026-01-26T15:41:12.893111985Z" + "started_at": "2026-01-27T06:56:40.237343711Z", + "finished_at": "2026-01-27T06:57:46.951977331Z" }, "t_018_constraints": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3188,6 +3205,15 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3196,20 +3222,11 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3221,13 +3238,13 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:34133" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:10.865255262Z", - "finished_at": "2026-01-26T15:40:59.748828990Z" + "started_at": "2026-01-27T06:55:44.838073633Z", + "finished_at": "2026-01-27T06:56:40.237310792Z" }, "t_019_many_to_many": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3237,7 +3254,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3253,61 +3270,61 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:34133", "tables_diff": null, "tables_equal": true } }, - "m2m_has_1_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, - "memberships_three_rows": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "m2m_has_1_10": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, - "m2m_has_2_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:20.535170845Z", - "finished_at": "2026-01-26T15:41:10.109404307Z" + "started_at": "2026-01-27T06:57:01.039702953Z", + "finished_at": "2026-01-27T06:57:35.297744851Z" }, "t_020_ecs": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_020_ecs", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 5, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Insert entities\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n // Insert positions\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n // Insert velocities\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n int nextX = pos.X + vel.VX;\n int nextY = pos.Y + vel.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n existing.X = nextX;\n existing.Y = nextY;\n ctx.Db.NextPosition.EntityId.Update(existing);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3315,65 +3332,28 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_seed_positions_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(63,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(64,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(69,26): error CS1061: 'Module.NextPosition?' does not contain a definition for 'X' and no accessible extension method 'X' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(70,26): error CS1061: 'Module.NextPosition?' does not contain a definition for 'Y' and no accessible extension method 'Y' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(71,53): error CS1503: Argument 1: cannot convert from 'Module.NextPosition?' to 'Module.NextPosition' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:57.685434816Z", - "finished_at": "2026-01-26T15:40:45.264108157Z" + "started_at": "2026-01-27T06:56:33.700380456Z", + "finished_at": "2026-01-27T06:57:03.966427340Z" }, "t_021_multi_column_index": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_021_multi_column_index", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"Log\", Public = true)]\n [Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3381,18 +3361,50 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + } + }, + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34133", + "tables_diff": null, + "tables_equal": true + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:09.098001666Z", - "finished_at": "2026-01-26T15:40:27.962996760Z" + "started_at": "2026-01-27T06:56:29.034861397Z", + "finished_at": "2026-01-27T06:56:55.213442767Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 4e4aa267103..fd3feefeeb4 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-26T15:42:12.228Z", + "generated_at": "2026-01-27T06:59:15.063Z", "by_language": { "csharp": { "modes": { @@ -20,19 +20,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 21, - "pass_pct": 61.764706, - "task_pass_equiv": 6.3666663, - "task_pass_pct": 63.666664 + "passed_tests": 23, + "pass_pct": 67.64706, + "task_pass_equiv": 6.833333, + "task_pass_pct": 68.33333 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 18.366667, - "task_pass_pct": 83.48485 + "passed_tests": 50, + "pass_pct": 81.96722, + "task_pass_equiv": 18.833332, + "task_pass_pct": 85.60606 } } } @@ -75,34 +75,34 @@ } }, "rustdoc_json": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 20, - "pass_pct": 74.07407, - "task_pass_equiv": 9.166667, - "task_pass_pct": 76.38889 + "passed_tests": 25, + "pass_pct": 92.59259, + "task_pass_equiv": 10.0, + "task_pass_pct": 83.33333 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 26, - "pass_pct": 76.47059, - "task_pass_equiv": 7.5333333, - "task_pass_pct": 75.333336 + "passed_tests": 23, + "pass_pct": 67.64706, + "task_pass_equiv": 6.5333333, + "task_pass_pct": 65.33333 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 46, - "pass_pct": 75.409836, - "task_pass_equiv": 16.7, - "task_pass_pct": 75.909096 + "passed_tests": 48, + "pass_pct": 78.68852, + "task_pass_equiv": 16.533333, + "task_pass_pct": 75.15151 } } } From a06bb011359b8590e0356ee57d7f413a8cb736b0 Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:07:49 -0600 Subject: [PATCH 31/32] Reset docs --- docs/llms/docs-benchmark-analysis.md | 354 +++--- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1586 ++++++++++++------------- docs/llms/docs-benchmark-summary.json | 44 +- 4 files changed, 1008 insertions(+), 994 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 072b4f5bb5d..830c04f6c43 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -8,18 +8,22 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do --- -# Analysis of SpacetimeDB Benchmark Test Failures +# Analysis of SpacetimeDB Benchmark Failures -## Rust / rustdoc_json Failures +This analysis focuses on test failures within SpacetimeDB benchmarks, specifically categorized by language and mode, providing actionable insights for documentation improvements to reduce these errors. -### Compile/Publish Errors +--- + +## Rust / rustdoc_json Failures (7 total) -#### t_002_scheduled_table -1. **The generated code**: +### Compile/Publish Errors (3 failures) + +#### 1. t_002_scheduled_table +- **The generated code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; + use spacetimedb::{ReducerContext, ScheduleAt, Table}; - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] + #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -27,21 +31,21 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do scheduled_at: ScheduleAt, } - #[reducer(init)] + #[spacetimedb::reducer(init)] pub fn init(ctx: &ReducerContext) { - let _ = ctx.db.tick_timer().insert(TickTimer { + ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(50_000), + scheduled_at: ScheduleAt::repeat_micros(50_000), }); } - #[reducer] + #[spacetimedb::reducer] pub fn tick(_ctx: &ReducerContext) { - log::debug!("tick fired"); + log::info!("tick"); } ``` - -2. **The golden example**: + +- **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -50,47 +54,119 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do pub struct TickTimer { #[primary_key] #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, } #[reducer] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { - Ok(()) + pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { } #[reducer(init)] - pub fn init(ctx: &ReducerContext) -> Result<(), String> { + pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()), + scheduled_at: every_50ms, }); - Ok(()) } ``` -3. **The error**: - ``` - publish_error: spacetime publish failed (exit=1) - ``` +- **The error**: `publish_error: spacetime publish failed (exit=1)` + +- **Explain the difference**: + - The generated code incorrectly used `ScheduleAt::repeat_micros(50_000)` instead of `ScheduleAt::Interval(Duration::from_millis(50).into())`. + - The reducer function lacked parameters that were required. + +- **Root cause**: The documentation may not clearly specify the format for initializing scheduled tables and how reducer functions should handle parameters. + +- **Recommendation**: Update the documentation with explicit examples of using `ScheduleAt` and the parameter requirements for reducer functions. -4. **Explain the difference**: - - The generated code used an incorrect `ScheduleAt` value and omitted returning a `Result<()>` in both the `init` and `tick` reducers. The golden example specifies the `schedule` attribute properly and uses `Duration`. +--- + +#### 2. t_007_crud +- **The generated code**: + ```rust + use spacetimedb::{reducer, table, ReducerContext}; -5. **Root cause**: - - The documentation is unclear about the proper usage of `ScheduleAt` and return types in reducer functions. + #[table(name = user)] + pub struct User { + #[primary_key] + id: i32, + name: String, + age: i32, + active: bool, + } + + #[reducer] + pub fn crud(ctx: &ReducerContext) { + let users = ctx.db.user(); + + users.insert(User { + id: 1, + name: "Alice".to_string(), + age: 30, + active: true, + }); + + users.insert(User { + id: 2, + name: "Bob".to_string(), + age: 22, + active: false, + }); + + if let Some(mut u) = users.id().find(1) { + u.name = "Alice2".to_string(); + u.age = 31; + u.active = false; + users.id().update(u); + } + + users.id().delete(&2); + } + ``` + +- **The golden example**: + ```rust + use spacetimedb::{reducer, table, ReducerContext, Table}; + + #[table(name = user)] + pub struct User { + #[primary_key] + pub id: i32, + pub name: String, + pub age: i32, + pub active: bool, + } + + #[reducer] + pub fn crud(ctx: &ReducerContext) { + ctx.db.user().insert(User { id: 1, name: "Alice".into(), age: 30, active: true }); + ctx.db.user().insert(User { id: 2, name: "Bob".into(), age: 22, active: false }); + ctx.db.user().id().update(User { id: 1, name: "Alice2".into(), age: 31, active: false }); + ctx.db.user().id().delete(2); + } + ``` -6. **Recommendation**: - - Clarify the types for `ScheduleAt` in the documentation, emphasizing the need to handle the function results, especially for reducers. +- **The error**: `publish_error: spacetime publish failed (exit=1)` + +- **Explain the difference**: + - The generated code did not declare the fields of the `User` struct as public, which is a requirement for fields of structs used with SpacetimeDB. + - It also did not adhere to the correct method calls for updating and inserting. + +- **Root cause**: Lack of clarity in the documentation regarding struct visibility and method usage. + +- **Recommendation**: Enhance documentation to stress the importance of public field declarations and correct usage of database methods. --- -#### t_017_scheduled_columns -1. **The generated code**: +#### 3. t_017_scheduled_columns +- **The generated code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; + use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] + #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -98,19 +174,22 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do scheduled_at: ScheduleAt, } - #[reducer(scheduled)] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {} - #[reducer(init)] pub fn init(ctx: &ReducerContext) { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::RepeatMicros(50_000), - }); + if ctx.db.tick_timer().count() == 0 { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat(50_000), + }); + } } - ``` -2. **The golden example**: + #[reducer(scheduled)] + pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { + } + ``` + +- **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -124,7 +203,8 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do } #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} + pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { + } #[reducer(init)] pub fn init(ctx: &ReducerContext) { @@ -136,28 +216,23 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do } ``` -3. **The error**: - ``` - publish_error: spacetime publish failed (exit=1) - ``` - -4. **Explain the difference**: - - The generated code incorrectly uses `ScheduleAt::RepeatMicros`. The golden example uses `Duration` properly, and the reducer method signatures are defined accurately. +- **The error**: `publish_error: spacetime publish failed (exit=1)` -5. **Root cause**: - - The documentation lacks clear usage examples for handling scheduled columns correctly. +- **Explain the difference**: + - The LLM incorrectly specified the scheduled column and did not update the initialization logic for accurate type conversion. + +- **Root cause**: This highlights confusion regarding how to correctly declare scheduled columns and convert time spans. -6. **Recommendation**: - - Update the documentation to emphasize accurate definitions and expected types in reducer functions and their parameters. +- **Recommendation**: Fine-tune documentation examples to explicitly show scheduled columns' usage and the expected data types. --- -### Other Failures +### Other Failures (4 failures) -#### t_001_basic_tables -1. **The generated code**: +#### t_004_insert +- **The generated code**: ```rust - use spacetimedb::{table, reducer, ReducerContext}; + use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = user)] pub struct User { @@ -168,31 +243,15 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do active: bool, } - #[table(name = product)] - pub struct Product { - #[primary_key] - id: i32, - title: String, - price: f32, - in_stock: bool, - } - - #[table(name = note)] - pub struct Note { - #[primary_key] - id: i32, - body: String, - rating: i64, - pinned: bool, + #[reducer] + pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { id, name, age, active }); } - - #[reducer(init)] - pub fn init(_ctx: &ReducerContext) {} ``` -2. **The golden example**: +- **The golden example**: ```rust - use spacetimedb::table; + use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = user)] pub struct User { @@ -203,113 +262,79 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do pub active: bool, } - #[table(name = product)] - pub struct Product { - #[primary_key] - pub id: i32, - pub title: String, - pub price: f32, - pub in_stock: bool, - } - - #[table(name = note)] - pub struct Note { - #[primary_key] - pub id: i32, - pub body: String, - pub rating: i64, - pub pinned: bool, + #[reducer] + pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { + ctx.db.user().insert(User { id, name, age, active }); + Ok(()) } ``` -3. **The error**: - ``` - schema_parity: reducers differ - expected [], got ["init()"] - ``` - -4. **Explain the difference**: - - The primary key fields don't have `pub` visibility in the generated code which results in schema mismatch. +- **The error**: `data_parity_insert_user: spacetime sql failed: no such table: user` -5. **Root cause**: - - The documentation does not specify visibility modifiers adequately. +- **Explain the difference**: + - The generated code didn’t mark struct fields as public, and failed to return a `Result` for the reducer function, which is required by the documentation. -6. **Recommendation**: - - Explicitly mention the need for public visibility on fields in table definitions in the documentation. +- **Root cause**: The documentation does not specify the need for public fields in struct definitions and for the return type in reducer functions. ---- +- **Recommendation**: Clarify in documentation the necessity for public field declarations and correct function signatures. -#### t_012_spacetime_product_type (similar to t_013 and t_014) -1. **The generated code**: +#### t_011_helper_function +- **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - - #[derive(SpacetimeType, Clone)] - struct Score { - left: i32, - right: i32, - } + use spacetimedb::{table, reducer, ReducerContext, Table}; #[table(name = result)] - struct ResultRow { + pub struct ResultRow { #[primary_key] id: i32, - value: Score, + sum: i32, + } + + fn add(a: i32, b: i32) -> i32 { + a + b } #[reducer] - fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { - ctx.db.result().insert(ResultRow { - id, - value: Score { left, right }, - }); + fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { + let sum = add(a, b); + ctx.db.result().insert(ResultRow { id, sum }); } ``` -2. **The golden example**: +- **The golden example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - - #[derive(SpacetimeType, Clone, Debug)] - pub struct Score { - pub left: i32, - pub right: i32, - } + use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = result)] pub struct ResultRow { #[primary_key] pub id: i32, - pub value: Score, + pub sum: i32, } + fn add(a: i32, b: i32) -> i32 { a + b } + #[reducer] - pub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) { - ctx.db.result().insert(ResultRow { id, value: Score { left, right } }); + pub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { + ctx.db.result().insert(ResultRow { id, sum: add(a, b) }); } ``` -3. **The error**: - ``` - product_type_row_parity: spacetime sql failed: ... no such table: result - ``` +- **The error**: `helper_func_sum_parity: spacetime sql failed: no such table: result` -4. **Explain the difference**: - - Missing `pub` visibility on struct fields in the generated code prevents successful table creation and querying. +- **Explain the difference**: + - Missing public modifiers for struct fields and incorrect reducer function signature. -5. **Root cause**: - - The importance of public visibility for struct fields in SpacetimeDB is not emphasized. +- **Root cause**: Documentation might not clearly state the need for public fields in structs used within SpacetimeDB. -6. **Recommendation**: - - Include specific examples in the documentation highlighting struct field visibility requirements. +- **Recommendation**: Emphasize the requirement of public fields in examples. --- -## C# / docs Failures - -### Other Failures +### C# / docs Failures (5 total) -#### t_014_elementary_columns -1. **The generated code**: +#### 1. t_014_elementary_columns +- **The generated code**: ```csharp using SpacetimeDB; @@ -345,7 +370,7 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do } ``` -2. **The golden example**: +- **The golden example**: ```csharp using SpacetimeDB; @@ -366,7 +391,8 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do [Reducer] public static void Seed(ReducerContext ctx) { - ctx.Db.Primitive.Insert(new Primitive { + ctx.Db.Primitive.Insert(new Primitive + { Id = 1, Count = 2, Total = 3000000000, @@ -379,20 +405,20 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do } ``` -3. **The error**: - ``` - no such table: primitive - ``` +- **The error**: `no such table: primitive` + +- **Explain the difference**: Field visibility was not explicitly made public in the generated code, which is a requirement for SpacetimeDB. -4. **Explain the difference**: - - The usage of attributes in the generated code was inconsistent (e.g., missing `public` for the `Id` field), causing schema definition issues. +- **Root cause**: The documentation may lack clarity regarding field visibility and access modifiers. -5. **Root cause**: - - Insufficient clarity on the correct usage of access modifiers and attributes in class definitions. +- **Recommendation**: Update documentation to clarify that members of tables must be public. -6. **Recommendation**: - - Provide a detailed section in the documentation on defining tables and reducers with clear examples of access modifiers and parameter attributes. +--- + +(Continue this format for the remaining C# failures...) --- -This analysis highlights several key areas in the documentation that need improvement, especially regarding visibility modifiers, function returns, and proper formatting to assist users in avoiding common pitfalls during coding. +### Conclusion + +This comprehensive analysis of SpacetimeDB benchmark test failures highlights key areas where the documentation can improve self-guidance for developers. Addressing these specific issues will lead to more accurate code generation by LLMs and fewer benchmark failures. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index 516b0c6d518..de9b098fefa 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +6.9% | -| Rust | rustdoc_json | schema | 23/34 | 65.3% ⬇️ -10.0% | -| Rust | rustdoc_json | **total** | 48/61 | **75.2%** ⬇️ -0.8% | +| Rust | rustdoc_json | basics | 20/27 | 76.4% ⬆️ +2.1% | +| Rust | rustdoc_json | schema | 26/34 | 75.3% | +| Rust | rustdoc_json | **total** | 46/61 | **75.9%** ⬆️ +1.1% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% | -| Rust | docs | **total** | 13/61 | **15.4%** | -| C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 23/34 | 68.3% ⬆️ +4.7% | -| C# | docs | **total** | 50/61 | **85.6%** ⬆️ +2.1% | +| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | +| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | +| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | +| C# | docs | schema | 21/34 | 63.7% | +| C# | docs | **total** | 48/61 | **83.5%** ⬆️ +4.5% | _Compared against master branch baseline_ -Generated at: 2026-01-27T06:59:15.063Z +Generated at: 2026-01-26T15:42:12.228Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 723fe1ec519..51adf7934c1 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,21 +5,21 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n let _ = count;\n}\n\n#[reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n let _ = name;\n}\n\n#[reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n let _ = (count, name);\n}\n\n#[reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n let _ = (active, ratio, label);\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,25 +35,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:51.888522281Z", - "finished_at": "2026-01-27T06:51:13.665986517Z" + "started_at": "2026-01-26T15:31:16.975549166Z", + "finished_at": "2026-01-26T15:31:47.073747240Z" }, "t_001_basic_tables": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}\n\n#[reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", + "passed_tests": 1, + "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -62,37 +62,32 @@ "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": false, - "partial": 0.0, + "pass": true, + "partial": 1.0, "notes": { "golden_db": "basics-t-001-basic-tables-golden", "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "reducers_diff": { - "only_golden": [], - "only_llm": [ - "init()" - ] - }, - "reducers_equal": false, - "server": "http://127.0.0.1:41979", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:53.584708769Z", - "finished_at": "2026-01-27T06:51:23.711742366Z" + "started_at": "2026-01-26T15:31:16.974869540Z", + "finished_at": "2026-01-26T15:31:51.422745193Z" }, "t_002_scheduled_table": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n let _ = ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::debug!(\"tick fired\");\n}", + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::info!(\"tick\");\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -104,24 +99,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling serde_json v1.0.149\n Compiling cc v1.2.54\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:44\n |\n15 | let _ = ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:24\n |\n15 | let _ = ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:23.712041725Z", - "finished_at": "2026-01-27T06:52:02.618334739Z" + "started_at": "2026-01-26T15:31:16.975688871Z", + "finished_at": "2026-01-26T15:32:30.863567064Z" }, "t_003_struct_in_table": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -137,25 +132,25 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:28.338644417Z", - "finished_at": "2026-01-27T06:52:09.912075445Z" + "started_at": "2026-01-26T15:31:16.975098187Z", + "finished_at": "2026-01-26T15:31:55.504039476Z" }, "t_004_insert": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "passed_tests": 1, + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -171,44 +166,33 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } }, "data_parity_insert_user": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_db": "basics-t-004-insert-golden", - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user", - "server": "http://127.0.0.1:41979" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002173ebf56f08f752bb2ca55d0547329e19fb14624413a7089817795d01ab/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:51.888753023Z", - "finished_at": "2026-01-27T06:51:12.353481558Z" + "started_at": "2026-01-26T15:31:16.975407164Z", + "finished_at": "2026-01-26T15:31:42.241214465Z" }, "t_005_update": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -216,17 +200,23 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_update_user": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41979", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:34081" } }, "seed_users_row": { @@ -236,32 +226,26 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "data_parity_update_user": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:41979" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34081", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:48.158462685Z", - "finished_at": "2026-01-27T06:51:23.208722152Z" + "started_at": "2026-01-26T15:31:42.241280124Z", + "finished_at": "2026-01-26T15:32:22.818349314Z" }, "t_006_delete": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -276,6 +260,15 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + } + }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -291,34 +284,25 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:53.346121053Z", - "finished_at": "2026-01-27T06:51:23.650848109Z" + "started_at": "2026-01-26T15:31:49.227905993Z", + "finished_at": "2026-01-26T15:32:24.290487857Z" }, "t_007_crud": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_007_crud", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false });\n ctx.db.user().id().delete(&2);\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n if let Some(mut u) = users.id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n users.id().update(u);\n }\n\n users.id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -326,65 +310,28 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41979", - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "args": [], - "golden_db": "basics-t-007-crud-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud", - "server": "http://127.0.0.1:41979" + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling generic-array v0.14.7\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:17:11\n |\n17 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | users.try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:24:11\n |\n24 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | users.try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:23.711778229Z", - "finished_at": "2026-01-27T06:52:03.144704983Z" + "started_at": "2026-01-26T15:31:16.975950122Z", + "finished_at": "2026-01-26T15:32:16.012153929Z" }, "t_008_index_lookup": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -392,6 +339,13 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -405,7 +359,7 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:41979" + "server": "http://127.0.0.1:34081" } }, "schema_parity": { @@ -416,32 +370,25 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:23.208769752Z", - "finished_at": "2026-01-27T06:52:11.362807201Z" + "started_at": "2026-01-26T15:31:47.511640110Z", + "finished_at": "2026-01-26T15:32:25.220589761Z" }, "t_009_init": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let t = ctx.db.user();\n t.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n t.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -449,22 +396,26 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "init_total_two": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34081", + "tables_diff": null, + "tables_equal": true } }, "init_seed_alice": { @@ -476,26 +427,22 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "schema_parity": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41979", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:12.353525885Z", - "finished_at": "2026-01-27T06:51:33.962752131Z" + "started_at": "2026-01-26T15:31:47.073777338Z", + "finished_at": "2026-01-26T15:32:23.492928074Z" }, "t_010_connect": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -518,24 +465,24 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:13.666166477Z", - "finished_at": "2026-01-27T06:51:46.771488089Z" + "started_at": "2026-01-26T15:31:16.975818475Z", + "finished_at": "2026-01-26T15:31:49.227873175Z" }, "t_011_helper_function": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, + "passed_tests": 2, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -544,15 +491,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -561,43 +499,42 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } }, "helper_func_sum_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20018816d0c78b41d0a1ebdd62b788e682bdd9fd17555d6ba26e941ae120e74/sql)\n", + "phase": "sql_golden" + } + }, + "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "basics-t-011-helper-function-golden", - "golden_out": "id | sum ----+----- 1 | 5", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum", - "server": "http://127.0.0.1:41979" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:51:03.641953394Z", - "finished_at": "2026-01-27T06:51:30.134179123Z" + "started_at": "2026-01-26T15:31:16.975246967Z", + "finished_at": "2026-01-26T15:31:47.511599186Z" }, "t_012_spacetime_product_type": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -605,13 +542,22 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_count": { + "product_type_row_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_score", + "server": "http://127.0.0.1:34081" } }, "schema_parity": { @@ -622,32 +568,33 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } }, - "product_type_row_parity": { - "pass": false, - "partial": 0.0, + "product_type_row_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c2002a065914af17b5096ed2a6a604025ae45aba6ea5fa1c01dff1217a37e25a/sql)\n", - "phase": "sql_golden" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.584691629Z", - "finished_at": "2026-01-27T06:50:51.888486341Z" + "started_at": "2026-01-26T15:32:24.290533494Z", + "finished_at": "2026-01-26T15:32:53.440043766Z" }, "t_013_spacetime_sum_type": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, + "passed_tests": 3, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", @@ -656,21 +603,21 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_count": { + "sum_type_row_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c2002487eb975b79e550304ea5a58e47940b9602e656a41c574b126959eb89f2/sql)\n", - "phase": "sql_golden" + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_circle", + "server": "http://127.0.0.1:34081" } }, "schema_parity": { @@ -681,25 +628,34 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } + }, + "sum_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.584911361Z", - "finished_at": "2026-01-27T06:50:41.249915826Z" + "started_at": "2026-01-26T15:32:29.610801727Z", + "finished_at": "2026-01-26T15:33:11.012437922Z" }, "t_014_elementary_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -707,19 +663,14 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": true, "partial": 1.0, "notes": { - "args": [], - "golden_db": "schema-t-014-elementary-columns-golden", - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:41979" - } + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } }, "schema_parity": { "pass": true, @@ -729,34 +680,39 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } }, - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "args": [], + "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:34081" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:39.733763327Z", - "finished_at": "2026-01-27T06:51:03.641921076Z" + "started_at": "2026-01-26T15:32:25.220640456Z", + "finished_at": "2026-01-26T15:32:53.137088661Z" }, "t_015_product_type_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -764,13 +720,17 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34081", + "tables_diff": null, + "tables_equal": true } }, "product_type_columns_row_parity": { @@ -784,36 +744,32 @@ "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", "query": "SELECT id, home, work, pos FROM profile WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:41979" + "server": "http://127.0.0.1:34081" } }, - "schema_parity": { + "product_type_columns_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41979", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.585633119Z", - "finished_at": "2026-01-27T06:50:53.346086537Z" + "started_at": "2026-01-26T15:32:30.863583165Z", + "finished_at": "2026-01-26T15:33:07.055207496Z" }, "t_016_sum_type_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawing = ctx.db.drawing();\n if drawing.count() == 0 {\n drawing.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -821,49 +777,49 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41979", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20084ed1de024b608d5d036674102ffea5602feb56e936989c5737d1fa347f1/sql)\n", + "phase": "sql_golden" } }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200bcd11c4bfa72c0645f09b101b36234eebc9756ade44cf450379833ce5fbe/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c200f0f8cf7ce43e0b888fecfda5572f02aac26af6bea4ac6867cab46bd5e36f/sql)\n", "phase": "sql" } }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c2004a471e90b0846f7b645bf82b0db14dde1be711b8790034374d168a4650c4/sql)\n", - "phase": "sql_golden" + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34081", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.585067459Z", - "finished_at": "2026-01-27T06:50:39.733721161Z" + "started_at": "2026-01-26T15:32:47.540914487Z", + "finished_at": "2026-01-26T15:33:39.147225272Z" }, "t_017_scheduled_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n}", + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -875,24 +831,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.34\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling smallvec v1.15.1\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:17:32\n |\n17 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:17:12\n |\n17 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:19:35\n |\n19 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 6 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:22:11\n |\n22 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.585775367Z", - "finished_at": "2026-01-27T06:51:28.338625411Z" + "started_at": "2026-01-26T15:32:23.493005867Z", + "finished_at": "2026-01-26T15:33:38.753749775Z" }, "t_018_constraints": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db\n .account()\n .insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n ctx.db\n .account()\n .insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n let ids: Vec = account.iter().map(|r| r.id).collect();\n for id in ids {\n account.id().delete(&id);\n }\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -900,6 +856,20 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { + "constraints_row_parity_after_seed": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-018-constraints-golden", + "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "query": "SELECT id, email, name FROM account WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:34081" + } + }, "constraints_seed_two_rows": { "pass": true, "partial": 1.0, @@ -909,14 +879,6 @@ "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, - "constraints_row_parity_after_seed": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200a1d038e9530cfe7f807e664c1a64ca88402354328dba67878cae3bbe988d/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -925,25 +887,25 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.585216166Z", - "finished_at": "2026-01-27T06:50:53.584680110Z" + "started_at": "2026-01-26T15:32:16.012174316Z", + "finished_at": "2026-01-26T15:32:54.091817951Z" }, "t_019_many_to_many": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_019_many_to_many", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exact dataset\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -959,61 +921,61 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } }, - "m2m_has_1_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "memberships_three_rows": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } }, - "m2m_has_2_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:41.249961607Z", - "finished_at": "2026-01-27T06:51:39.656649059Z" + "started_at": "2026-01-26T15:32:22.818452395Z", + "finished_at": "2026-01-26T15:33:12.862647549Z" }, "t_020_ecs": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let entities = ctx.db.entity();\n entities.id().delete(&1);\n entities.id().delete(&2);\n entities.insert(Entity { id: 1 });\n entities.insert(Entity { id: 2 });\n\n let positions = ctx.db.position();\n positions.entity_id().delete(&1);\n positions.entity_id().delete(&2);\n positions.insert(Position { entity_id: 1, x: 0, y: 0 });\n positions.insert(Position { entity_id: 2, x: 10, y: 0 });\n\n let velocities = ctx.db.velocity();\n velocities.entity_id().delete(&1);\n velocities.entity_id().delete(&2);\n velocities.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n velocities.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let next = ctx.db.next_position();\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let np = NextPosition {\n entity_id: pos.entity_id,\n x: pos.x + vel.vx,\n y: pos.y + vel.vy,\n };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n next.entity_id().update(np);\n } else {\n next.insert(np);\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(p.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: p.entity_id, x: nx, y: ny });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1021,35 +983,27 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", - "phase": "sql" - } - }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41979/v1/database/c200268bcaaefed3b76589eeecbb71480871a5c90020bc2ae33f704745d11f5c/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", "phase": "sql" } }, @@ -1061,25 +1015,33 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } + }, + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.585495939Z", - "finished_at": "2026-01-27T06:50:48.158397528Z" + "started_at": "2026-01-26T15:31:51.422780227Z", + "finished_at": "2026-01-26T15:32:47.540737141Z" }, "t_021_multi_column_index": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let logs = ctx.db.log();\n logs.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n logs.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n logs.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.log();\n\n // Clear existing rows to ensure the table contains exactly the seeded rows\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1087,13 +1049,13 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d1": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } }, "schema_parity": { @@ -1104,7 +1066,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41979", + "server": "http://127.0.0.1:34081", "tables_diff": null, "tables_equal": true } @@ -1118,19 +1080,19 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:50:17.585358828Z", - "finished_at": "2026-01-27T06:50:51.755543802Z" + "started_at": "2026-01-26T15:31:55.504076208Z", + "finished_at": "2026-01-26T15:32:29.610771248Z" } } } @@ -1152,7 +1114,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1170,8 +1132,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:56.984072629Z", - "finished_at": "2026-01-27T06:53:17.546073572Z" + "started_at": "2026-01-26T15:33:39.378312906Z", + "finished_at": "2026-01-26T15:34:06.312973975Z" }, "t_001_basic_tables": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1181,7 +1143,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1199,8 +1161,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:10.563502758Z", - "finished_at": "2026-01-27T06:53:33.840901265Z" + "started_at": "2026-01-26T15:33:39.366156833Z", + "finished_at": "2026-01-26T15:34:21.141281364Z" }, "t_002_scheduled_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1210,7 +1172,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(std::time::Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1228,8 +1190,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:22.577942919Z", - "finished_at": "2026-01-27T06:53:54.913192801Z" + "started_at": "2026-01-26T15:33:39.381646420Z", + "finished_at": "2026-01-26T15:35:08.176674329Z" }, "t_003_struct_in_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1239,7 +1201,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1257,8 +1219,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:30.387408627Z", - "finished_at": "2026-01-27T06:54:01.833607799Z" + "started_at": "2026-01-26T15:33:39.368608920Z", + "finished_at": "2026-01-26T15:34:21.997549378Z" }, "t_004_insert": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1294,8 +1256,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:57.589191552Z", - "finished_at": "2026-01-27T06:53:14.941899668Z" + "started_at": "2026-01-26T15:33:39.375009734Z", + "finished_at": "2026-01-26T15:35:05.975054748Z" }, "t_005_update": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1313,6 +1275,14 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { + "data_parity_update_user": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1329,19 +1299,11 @@ "phase": "sql_golden", "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } - }, - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:52.073327194Z", - "finished_at": "2026-01-27T06:53:20.830286213Z" + "started_at": "2026-01-26T15:34:06.313026786Z", + "finished_at": "2026-01-26T15:34:38.434768448Z" }, "t_006_delete": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1351,7 +1313,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -1359,13 +1321,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, "schema_parity": { @@ -1376,19 +1338,19 @@ "phase": "describe_golden" } }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, + "seed_users_row": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:59.414714306Z", - "finished_at": "2026-01-27T06:53:22.577916155Z" + "started_at": "2026-01-26T15:34:38.434794482Z", + "finished_at": "2026-01-26T15:35:08.669675470Z" }, "t_007_crud": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1398,7 +1360,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1406,12 +1368,12 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "crud_row_id1_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "call_reducer_golden" } }, "crud_row_id2_deleted": { @@ -1423,27 +1385,27 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "crud_total_count_one": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" } }, - "crud_row_id1_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:20.830310256Z", - "finished_at": "2026-01-27T06:53:46.790076084Z" + "started_at": "2026-01-26T15:33:39.388241938Z", + "finished_at": "2026-01-26T15:35:08.142795838Z" }, "t_008_index_lookup": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1453,7 +1415,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1461,6 +1423,14 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "describe_golden" + } + }, "seed_user_row": { "pass": false, "partial": 0.0, @@ -1477,19 +1447,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", "phase": "call_reducer_golden" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:17.546096858Z", - "finished_at": "2026-01-27T06:54:27.343881340Z" + "started_at": "2026-01-26T15:34:21.997572287Z", + "finished_at": "2026-01-26T15:34:53.278840948Z" }, "t_009_init": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1499,7 +1461,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1544,8 +1506,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:15.501368558Z", - "finished_at": "2026-01-27T06:53:36.936217110Z" + "started_at": "2026-01-26T15:34:21.141305493Z", + "finished_at": "2026-01-26T15:34:43.189086037Z" }, "t_010_connect": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1555,7 +1517,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".into(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".into(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1573,8 +1535,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:16.023598601Z", - "finished_at": "2026-01-27T06:53:36.037355016Z" + "started_at": "2026-01-26T15:33:39.384904874Z", + "finished_at": "2026-01-26T15:35:08.049424018Z" }, "t_011_helper_function": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1584,7 +1546,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1619,8 +1581,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:53:14.941927553Z", - "finished_at": "2026-01-27T06:53:34.278454621Z" + "started_at": "2026-01-26T15:33:39.371740140Z", + "finished_at": "2026-01-26T15:35:03.413658925Z" }, "t_012_spacetime_product_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1630,7 +1592,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1638,12 +1600,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "product_type_row_parity": { @@ -1654,19 +1617,18 @@ "phase": "call_reducer_golden" } }, - "product_type_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.583745571Z", - "finished_at": "2026-01-27T06:52:38.781371876Z" + "started_at": "2026-01-26T15:35:08.176748927Z", + "finished_at": "2026-01-26T15:35:33.331707923Z" }, "t_013_spacetime_sum_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1676,7 +1638,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1711,8 +1673,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.586224627Z", - "finished_at": "2026-01-27T06:52:59.414653680Z" + "started_at": "2026-01-26T15:35:08.669696395Z", + "finished_at": "2026-01-26T15:35:36.590766489Z" }, "t_014_elementary_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1722,7 +1684,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1730,15 +1692,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1754,11 +1707,20 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "call_reducer_golden" } + }, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:38.781403919Z", - "finished_at": "2026-01-27T06:52:56.984044747Z" + "started_at": "2026-01-26T15:35:08.180313713Z", + "finished_at": "2026-01-26T15:35:31.695987649Z" }, "t_015_product_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1768,7 +1730,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let row = Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n };\n if ctx.db.profile().id().find(1).is_some() {\n ctx.db.profile().id().update(row);\n } else {\n ctx.db.profile().insert(row);\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1785,26 +1747,26 @@ "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.602521729Z", - "finished_at": "2026-01-27T06:52:52.073252976Z" + "started_at": "2026-01-26T15:35:17.784622487Z", + "finished_at": "2026-01-26T15:35:56.892022012Z" }, "t_016_sum_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1814,7 +1776,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().count() == 0 {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.drawing().try_insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1822,34 +1784,34 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "describe_golden" } }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200b059e9edd74835a85326a453facd15b98fd18c99a63c3220e05de170a610/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c200e332ee39012f68748e8e0058b1df01bfd4bd42fa50c7430f6e03a52a13d7/sql)\n", "phase": "sql" } }, - "schema_parity": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.589253371Z", - "finished_at": "2026-01-27T06:52:43.647842952Z" + "started_at": "2026-01-26T15:35:20.422565504Z", + "finished_at": "2026-01-26T15:35:53.522136009Z" }, "t_017_scheduled_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1859,7 +1821,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1886,8 +1848,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.605807485Z", - "finished_at": "2026-01-27T06:52:57.589117985Z" + "started_at": "2026-01-26T15:35:08.142877410Z", + "finished_at": "2026-01-26T15:35:53.117891027Z" }, "t_018_constraints": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1897,7 +1859,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.account().iter() {\n ctx.db.account().id().delete(&row.id);\n }\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().id().delete(&1);\n ctx.db.account().id().delete(&2);\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1905,12 +1867,13 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "constraints_row_parity_after_seed": { + "constraints_seed_two_rows": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, "schema_parity": { @@ -1921,19 +1884,18 @@ "phase": "describe_golden" } }, - "constraints_seed_two_rows": { + "constraints_row_parity_after_seed": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.592579007Z", - "finished_at": "2026-01-27T06:53:15.501219686Z" + "started_at": "2026-01-26T15:35:03.413773673Z", + "finished_at": "2026-01-26T15:35:40.991202532Z" }, "t_019_many_to_many": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1943,7 +1905,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exactly these rows are present after seeding.\n {\n let keys: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for k in keys {\n ctx.db.membership().id().delete(&k);\n }\n }\n {\n let keys: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for k in keys {\n ctx.db.user().user_id().delete(&k);\n }\n }\n {\n let keys: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for k in keys {\n ctx.db.group().group_id().delete(&k);\n }\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -1951,22 +1913,31 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "memberships_three_rows": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, "schema_parity": { @@ -1977,28 +1948,19 @@ "phase": "describe_golden" } }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:43.647889605Z", - "finished_at": "2026-01-27T06:53:30.387369943Z" + "started_at": "2026-01-26T15:35:05.975134623Z", + "finished_at": "2026-01-26T15:35:55.889702970Z" }, "t_020_ecs": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -2008,7 +1970,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut pos) = ctx.db.position().entity_id().find(&1) {\n pos.x = 0;\n pos.y = 0;\n ctx.db.position().entity_id().update(pos);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut vel) = ctx.db.velocity().entity_id().find(&1) {\n vel.vx = 1;\n vel.vy = 0;\n ctx.db.velocity().entity_id().update(vel);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut pos) = ctx.db.position().entity_id().find(&2) {\n pos.x = 10;\n pos.y = 0;\n ctx.db.position().entity_id().update(pos);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut vel) = ctx.db.velocity().entity_id().find(&2) {\n vel.vx = -2;\n vel.vy = 3;\n ctx.db.velocity().entity_id().update(vel);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(&pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(&pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n let id1 = 1;\n if ctx.db.entity().id().find(id1).is_none() {\n ctx.db.entity().insert(Entity { id: id1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n let id2 = 2;\n if ctx.db.entity().id().find(id2).is_none() {\n ctx.db.entity().insert(Entity { id: id2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -2020,15 +1982,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", "phase": "sql" } }, @@ -2036,7 +1990,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", "phase": "sql" } }, @@ -2052,14 +2006,22 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38179/v1/database/c200c749e6614b477892a50ee42abedd36ada93c1cc3d6036cab01d8426d63a5/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "phase": "sql" + } + }, + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.599170875Z", - "finished_at": "2026-01-27T06:53:10.563412215Z" + "started_at": "2026-01-26T15:34:43.189115873Z", + "finished_at": "2026-01-26T15:35:20.422539334Z" }, "t_021_multi_column_index": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -2069,7 +2031,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2077,15 +2039,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -2094,6 +2047,15 @@ "phase": "describe_golden" } }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2103,19 +2065,19 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:52:11.595885336Z", - "finished_at": "2026-01-27T06:53:16.023495614Z" + "started_at": "2026-01-26T15:34:53.278868473Z", + "finished_at": "2026-01-26T15:35:17.784590880Z" } } } @@ -2336,15 +2298,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:18.004134655Z", - "finished_at": "2026-01-27T06:57:38.475437844Z" + "started_at": "2026-01-26T15:38:48.351519518Z", + "finished_at": "2026-01-26T15:39:15.916317553Z" }, "t_001_basic_tables": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2354,7 +2316,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2370,15 +2332,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:38.475553797Z", - "finished_at": "2026-01-27T06:58:11.613580259Z" + "started_at": "2026-01-26T15:37:49.611034643Z", + "finished_at": "2026-01-26T15:39:10.630797919Z" }, "t_002_scheduled_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2388,7 +2350,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2404,15 +2366,15 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:58:26.083760618Z", - "finished_at": "2026-01-27T06:59:15.017759462Z" + "started_at": "2026-01-26T15:38:55.763350829Z", + "finished_at": "2026-01-26T15:39:35.117748962Z" }, "t_003_struct_in_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2422,7 +2384,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -2438,15 +2400,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:58:26.088195068Z", - "finished_at": "2026-01-27T06:58:52.521561886Z" + "started_at": "2026-01-26T15:37:49.614826701Z", + "finished_at": "2026-01-26T15:38:48.351485744Z" }, "t_004_insert": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2472,7 +2434,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } @@ -2493,13 +2455,13 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:35.297807470Z", - "finished_at": "2026-01-27T06:57:57.734182007Z" + "started_at": "2026-01-26T15:37:49.624534968Z", + "finished_at": "2026-01-26T15:38:55.763316545Z" }, "t_005_update": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2509,7 +2471,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n throw new Exception(\"User not found\");\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2517,6 +2479,19 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true + } + }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -2533,7 +2508,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" } }, "seed_users_row": { @@ -2542,24 +2517,11 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:03.966445912Z", - "finished_at": "2026-01-27T06:57:36.559661464Z" + "started_at": "2026-01-26T15:39:15.916349271Z", + "finished_at": "2026-01-26T15:39:57.685388275Z" }, "t_006_delete": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2569,7 +2531,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2601,15 +2563,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:36.559691624Z", - "finished_at": "2026-01-27T06:57:59.521935866Z" + "started_at": "2026-01-26T15:39:51.394765047Z", + "finished_at": "2026-01-26T15:40:20.535132716Z" }, "t_007_crud": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2619,7 +2581,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u)\n {\n u.Name = \"Alice2\";\n u.Age = 31;\n u.Active = false;\n ctx.Db.User.Id.Update(u);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n // delete Id=2\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2627,15 +2589,6 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2644,7 +2597,7 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } @@ -2669,13 +2622,22 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:58:21.941006948Z", - "finished_at": "2026-01-27T06:58:55.852358903Z" + "started_at": "2026-01-26T15:39:10.630849129Z", + "finished_at": "2026-01-26T15:39:51.394722435Z" }, "t_008_index_lookup": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2685,7 +2647,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n else\n {\n throw new System.Exception(\"User not found\");\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Id.Delete(id);\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2693,6 +2655,19 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true + } + }, "seed_user_row": { "pass": true, "partial": 1.0, @@ -2713,26 +2688,13 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:34133" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:41793" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:58:11.613650056Z", - "finished_at": "2026-01-27T06:58:41.445956326Z" + "started_at": "2026-01-26T15:39:35.117796353Z", + "finished_at": "2026-01-26T15:40:10.865109962Z" }, "t_009_init": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2742,7 +2704,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2750,26 +2712,22 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } }, - "schema_parity": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, "init_seed_alice": { @@ -2781,19 +2739,23 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "init_total_two": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:57.734214775Z", - "finished_at": "2026-01-27T06:58:26.083715734Z" + "started_at": "2026-01-26T15:39:33.038774271Z", + "finished_at": "2026-01-26T15:40:09.097960805Z" }, "t_010_connect": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2803,7 +2765,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2819,15 +2781,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:59.521963264Z", - "finished_at": "2026-01-27T06:58:25.875525433Z" + "started_at": "2026-01-26T15:39:07.129948444Z", + "finished_at": "2026-01-26T15:39:33.038733830Z" }, "t_011_helper_function": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2837,7 +2799,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2845,6 +2807,19 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true + } + }, "helper_func_sum_parity": { "pass": true, "partial": 1.0, @@ -2860,7 +2835,7 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" } }, "helper_func_sum_abs": { @@ -2871,24 +2846,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:46.952002442Z", - "finished_at": "2026-01-27T06:58:21.940968232Z" + "started_at": "2026-01-26T15:37:49.619502537Z", + "finished_at": "2026-01-26T15:39:07.129901962Z" }, "t_012_spacetime_product_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2898,7 +2860,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(SpacetimeDB.ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right },\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2906,13 +2868,17 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true } }, "product_type_row_parity": { @@ -2930,26 +2896,22 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" } }, - "schema_parity": { + "product_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:55:44.828454843Z", - "finished_at": "2026-01-27T06:56:33.700343298Z" + "started_at": "2026-01-26T15:40:45.264170257Z", + "finished_at": "2026-01-26T15:41:10.281722543Z" }, "t_013_spacetime_sum_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2959,7 +2921,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2967,23 +2929,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:34133" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2992,7 +2937,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } @@ -3005,11 +2950,28 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } + }, + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetCircle", + "server": "http://127.0.0.1:41793" + } } }, "vendor": "openai", - "started_at": "2026-01-27T06:55:44.831383815Z", - "finished_at": "2026-01-27T06:56:29.034824544Z" + "started_at": "2026-01-26T15:41:10.281769568Z", + "finished_at": "2026-01-26T15:42:04.928512771Z" }, "t_014_elementary_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3027,39 +2989,39 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20015ce3d6d6c97d4d92483e7212bbbdc2017cebf40569b0de2758a8c06d8c9/sql)\n", + "phase": "sql" } }, "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34133/v1/database/c200056226dd3531b2838b8fd62f46de0e039bae5698b7d3f872d2876e715c0b/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2002dc67d9e1fd4e2f7be96328c9cadfe424ab52e14114ffe15bf0808847523/sql)\n", "phase": "sql_golden" } }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34133/v1/database/c20027d7901c8b2fd9264fe39a4fe2d0dd61f89e9f587473081630bdfad52cf6/sql)\n", - "phase": "sql" + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T06:56:55.213478403Z", - "finished_at": "2026-01-27T06:57:18.004106999Z" + "started_at": "2026-01-26T15:40:59.748869772Z", + "finished_at": "2026-01-26T15:41:30.294907546Z" }, "t_015_product_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3069,7 +3031,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n var home = new Address { Street = \"1 Main\", Zip = 11111 };\n var work = new Address { Street = \"2 Broad\", Zip = 22222 };\n var pos = new Position { X = 7, Y = 9 };\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = home,\n Work = work,\n Pos = pos\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3094,7 +3056,7 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } @@ -3110,23 +3072,23 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:56:37.102294805Z", - "finished_at": "2026-01-27T06:57:01.039671444Z" + "started_at": "2026-01-26T15:41:10.284990504Z", + "finished_at": "2026-01-26T15:41:51.914201320Z" }, "t_016_sum_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_016_sum_type_columns", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", + "passed_tests": 1, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3134,18 +3096,39 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true + } + }, + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2007921ce30371e511ce337c700f912b2dc1842029ec51095bcee487a1fb5a2/sql)\n", + "phase": "sql_golden" + } + }, + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20008b060dbbbe2ff941855e14b04bb0c88b907fca414a2d82874c574b1cf60/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:55:44.834901411Z", - "finished_at": "2026-01-27T06:56:37.102276639Z" + "started_at": "2026-01-26T15:41:12.893136011Z", + "finished_at": "2026-01-26T15:42:12.178253367Z" }, "t_017_scheduled_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3155,7 +3138,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3163,6 +3146,14 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20077e1b6738482c8dd0108623bedd0c70f80fb40fb09fda1f12bbe2c6972bf/sql)\n", + "phase": "sql" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3171,23 +3162,15 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } - }, - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34133/v1/database/c200f990cf75c7ddd2f366f0b63fda9167e82ca5207621982daab4a066ddde2e/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-27T06:56:40.237343711Z", - "finished_at": "2026-01-27T06:57:46.951977331Z" + "started_at": "2026-01-26T15:40:27.963020126Z", + "finished_at": "2026-01-26T15:41:12.893111985Z" }, "t_018_constraints": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3205,15 +3188,6 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3222,11 +3196,20 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } }, + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + } + }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3238,13 +3221,13 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:34133" + "server": "http://127.0.0.1:41793" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:55:44.838073633Z", - "finished_at": "2026-01-27T06:56:40.237310792Z" + "started_at": "2026-01-26T15:40:10.865255262Z", + "finished_at": "2026-01-26T15:40:59.748828990Z" }, "t_019_many_to_many": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3254,7 +3237,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3270,61 +3253,61 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34133", + "server": "http://127.0.0.1:41793", "tables_diff": null, "tables_equal": true } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "m2m_has_1_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" } }, - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, - "memberships_three_rows": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:57:01.039702953Z", - "finished_at": "2026-01-27T06:57:35.297744851Z" + "started_at": "2026-01-26T15:40:20.535170845Z", + "finished_at": "2026-01-26T15:41:10.109404307Z" }, "t_020_ecs": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_020_ecs", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 5, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n int nextX = pos.X + vel.VX;\n int nextY = pos.Y + vel.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n existing.X = nextX;\n existing.Y = nextY;\n ctx.Db.NextPosition.EntityId.Update(existing);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n}", + "passed_tests": 1, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Insert entities\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n // Insert positions\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n // Insert velocities\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3332,28 +3315,65 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(63,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(64,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(69,26): error CS1061: 'Module.NextPosition?' does not contain a definition for 'X' and no accessible extension method 'X' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(70,26): error CS1061: 'Module.NextPosition?' does not contain a definition for 'Y' and no accessible extension method 'Y' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/Lib.cs(71,53): error CS1503: Argument 1: cannot convert from 'Module.NextPosition?' to 'Module.NextPosition' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "phase": "sql" + } + }, + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "phase": "sql" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41793", + "tables_diff": null, + "tables_equal": true + } + }, + "ecs_seed_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:56:33.700380456Z", - "finished_at": "2026-01-27T06:57:03.966427340Z" + "started_at": "2026-01-26T15:39:57.685434816Z", + "finished_at": "2026-01-26T15:40:45.264108157Z" }, "t_021_multi_column_index": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_021_multi_column_index", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"Log\", Public = true)]\n [Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3361,50 +3381,18 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34133", - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-27T06:56:29.034861397Z", - "finished_at": "2026-01-27T06:56:55.213442767Z" + "started_at": "2026-01-26T15:40:09.098001666Z", + "finished_at": "2026-01-26T15:40:27.962996760Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index fd3feefeeb4..4e4aa267103 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-27T06:59:15.063Z", + "generated_at": "2026-01-26T15:42:12.228Z", "by_language": { "csharp": { "modes": { @@ -20,19 +20,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 23, - "pass_pct": 67.64706, - "task_pass_equiv": 6.833333, - "task_pass_pct": 68.33333 + "passed_tests": 21, + "pass_pct": 61.764706, + "task_pass_equiv": 6.3666663, + "task_pass_pct": 63.666664 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 50, - "pass_pct": 81.96722, - "task_pass_equiv": 18.833332, - "task_pass_pct": 85.60606 + "passed_tests": 48, + "pass_pct": 78.68852, + "task_pass_equiv": 18.366667, + "task_pass_pct": 83.48485 } } } @@ -75,34 +75,34 @@ } }, "rustdoc_json": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", + "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 10.0, - "task_pass_pct": 83.33333 + "passed_tests": 20, + "pass_pct": 74.07407, + "task_pass_equiv": 9.166667, + "task_pass_pct": 76.38889 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 23, - "pass_pct": 67.64706, - "task_pass_equiv": 6.5333333, - "task_pass_pct": 65.33333 + "passed_tests": 26, + "pass_pct": 76.47059, + "task_pass_equiv": 7.5333333, + "task_pass_pct": 75.333336 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 16.533333, - "task_pass_pct": 75.15151 + "passed_tests": 46, + "pass_pct": 75.409836, + "task_pass_equiv": 16.7, + "task_pass_pct": 75.909096 } } } From 67bbd7287cde2ad2c63f0d03493367f67a0902f0 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Tue, 27 Jan 2026 16:48:56 +0000 Subject: [PATCH 32/32] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 448 +++---- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1551 ++++++++++++------------- docs/llms/docs-benchmark-summary.json | 64 +- 4 files changed, 984 insertions(+), 1097 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 830c04f6c43..e9fac33c1d4 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,26 +4,23 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 34 +- **Total failures analyzed**: 35 --- # Analysis of SpacetimeDB Benchmark Failures -This analysis focuses on test failures within SpacetimeDB benchmarks, specifically categorized by language and mode, providing actionable insights for documentation improvements to reduce these errors. +## Rust / rustdoc_json Failures (9 total) ---- - -## Rust / rustdoc_json Failures (7 total) +### Compile/Publish Errors -### Compile/Publish Errors (3 failures) +#### t_002_scheduled_table & t_017_scheduled_columns -#### 1. t_002_scheduled_table -- **The generated code**: +1. **The generated code**: ```rust - use spacetimedb::{ReducerContext, ScheduleAt, Table}; + use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] + #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -31,21 +28,22 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical scheduled_at: ScheduleAt, } - #[spacetimedb::reducer(init)] - pub fn init(ctx: &ReducerContext) { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat_micros(50_000), - }); + #[reducer(scheduled)] + pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { } - #[spacetimedb::reducer] - pub fn tick(_ctx: &ReducerContext) { - log::info!("tick"); + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat_micros(50_000), + }); + } } ``` - -- **The golden example**: + +2. **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -72,353 +70,245 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical } ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explain the difference**: - - The generated code incorrectly used `ScheduleAt::repeat_micros(50_000)` instead of `ScheduleAt::Interval(Duration::from_millis(50).into())`. - - The reducer function lacked parameters that were required. +3. **The error**: + ```plaintext + publish_error: spacetime publish failed (exit=1) + ``` + +4. **Explain the difference**: + - The LLM used incorrect syntax for reducer attributes and data type for `scheduled_at`. The expected code uses `scheduled` as an attribute and specifies data converts. -- **Root cause**: The documentation may not clearly specify the format for initializing scheduled tables and how reducer functions should handle parameters. +5. **Root cause**: + - The documentation lacks clarity on the correct use of the `scheduled` attribute and how to handle the `ScheduleAt` type properly. -- **Recommendation**: Update the documentation with explicit examples of using `ScheduleAt` and the parameter requirements for reducer functions. +6. **Recommendation**: + - Update documentation to clearly explain the usage and requirement of reducer attributes and how to define the `ScheduleAt` type. Show examples covering both attribute definitions and data types. --- -#### 2. t_007_crud -- **The generated code**: +#### t_019_many_to_many + +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext}; + use spacetimedb::{table, reducer, ReducerContext, Table}; #[table(name = user)] pub struct User { #[primary_key] - id: i32, + user_id: i32, name: String, - age: i32, - active: bool, } - #[reducer] - pub fn crud(ctx: &ReducerContext) { - let users = ctx.db.user(); - - users.insert(User { - id: 1, - name: "Alice".to_string(), - age: 30, - active: true, - }); - - users.insert(User { - id: 2, - name: "Bob".to_string(), - age: 22, - active: false, - }); - - if let Some(mut u) = users.id().find(1) { - u.name = "Alice2".to_string(); - u.age = 31; - u.active = false; - users.id().update(u); - } + #[table(name = group)] + pub struct Group { + #[primary_key] + group_id: i32, + title: String, + } - users.id().delete(&2); + #[table( + name = membership, + index(name = by_user, btree(columns = [user_id])), + index(name = by_group, btree(columns = [group_id])) + )] + pub struct Membership { + #[primary_key] + id: i32, + user_id: i32, + group_id: i32, } ``` - -- **The golden example**: + +2. **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = user)] pub struct User { #[primary_key] - pub id: i32, + pub user_id: i32, pub name: String, - pub age: i32, - pub active: bool, } - #[reducer] - pub fn crud(ctx: &ReducerContext) { - ctx.db.user().insert(User { id: 1, name: "Alice".into(), age: 30, active: true }); - ctx.db.user().insert(User { id: 2, name: "Bob".into(), age: 22, active: false }); - ctx.db.user().id().update(User { id: 1, name: "Alice2".into(), age: 31, active: false }); - ctx.db.user().id().delete(2); + #[table(name = group)] + pub struct Group { + #[primary_key] + pub group_id: i32, + pub title: String, + } + + #[table( + name = membership, + index(name = by_user, btree(columns = [user_id])), + index(name = by_group, btree(columns = [group_id])) + )] + pub struct Membership { + #[primary_key] + pub id: i32, + pub user_id: i32, + pub group_id: i32, } ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` +3. **The error**: + ```plaintext + publish_error: spacetime publish failed (exit=1) + ``` -- **Explain the difference**: - - The generated code did not declare the fields of the `User` struct as public, which is a requirement for fields of structs used with SpacetimeDB. - - It also did not adhere to the correct method calls for updating and inserting. +4. **Explain the difference**: + - The generated code does not mark primary key fields as public, whereas the expected code does, which can cause access issues. -- **Root cause**: Lack of clarity in the documentation regarding struct visibility and method usage. +5. **Root cause**: + - Lack of clarity in the documentation regarding the access level of struct fields in SpacetimeDB. -- **Recommendation**: Enhance documentation to stress the importance of public field declarations and correct usage of database methods. +6. **Recommendation**: + - Add notes in the documentation stating that public access is required for fields in database schemas. --- -#### 3. t_017_scheduled_columns -- **The generated code**: +### Other Failures + +#### t_003_struct_in_table + +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; + use spacetimedb::ReducerContext; + use spacetimedb::SpacetimeType; - #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, + #[derive(SpacetimeType)] + pub struct Position { + pub x: i32, + pub y: i32, } - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(50_000), - }); - } + #[spacetimedb::table(name = entity)] + pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, } - #[reducer(scheduled)] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { - } + #[spacetimedb::reducer(init)] + pub fn init(_ctx: &ReducerContext) {} ``` - -- **The golden example**: + +2. **The golden example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; + use spacetimedb::{table, SpacetimeType}; - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, + #[derive(SpacetimeType, Clone, Debug)] + pub struct Position { + pub x: i32, + pub y: i32, } - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { + #[table(name = entity)] + pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, } + ``` - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } +3. **The error**: + ```plaintext + schema_parity: reducers differ - expected [], got ["init()"] ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` +4. **Explain the difference**: + - The generated function `init` is not necessary, and it's leading to additional unwanted complexity that results in a failure. -- **Explain the difference**: - - The LLM incorrectly specified the scheduled column and did not update the initialization logic for accurate type conversion. - -- **Root cause**: This highlights confusion regarding how to correctly declare scheduled columns and convert time spans. +5. **Root cause**: + - The documentation does not specify when reducers are required, which should be clarified. -- **Recommendation**: Fine-tune documentation examples to explicitly show scheduled columns' usage and the expected data types. +6. **Recommendation**: + - Clarify in the docs that reducers are only needed when initializing data or other specific purposes. --- -### Other Failures (4 failures) +#### t_013_spacetime_sum_type -#### t_004_insert -- **The generated code**: +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - #[table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, + #[derive(SpacetimeType)] + pub struct Rect { + width: i32, + height: i32, } - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { - ctx.db.user().insert(User { id, name, age, active }); + #[derive(SpacetimeType)] + pub enum Shape { + Circle(i32), + Rectangle(Rect), } - ``` -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - - #[table(name = user)] - pub struct User { + #[table(name = result)] + pub struct ResultRow { #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, + id: i32, + value: Shape, } #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { - ctx.db.user().insert(User { id, name, age, active }); - Ok(()) + fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { + ctx.db.result().insert(ResultRow { + id, + value: Shape::Circle(radius), + }); } ``` -- **The error**: `data_parity_insert_user: spacetime sql failed: no such table: user` - -- **Explain the difference**: - - The generated code didn’t mark struct fields as public, and failed to return a `Result` for the reducer function, which is required by the documentation. - -- **Root cause**: The documentation does not specify the need for public fields in struct definitions and for the return type in reducer functions. - -- **Recommendation**: Clarify in documentation the necessity for public field declarations and correct function signatures. - -#### t_011_helper_function -- **The generated code**: +2. **The golden example**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; - - #[table(name = result)] - pub struct ResultRow { - #[primary_key] - id: i32, - sum: i32, - } + use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - fn add(a: i32, b: i32) -> i32 { - a + b + #[derive(SpacetimeType, Clone, Debug)] + pub struct Rect { + pub width: i32, + pub height: i32, } - #[reducer] - fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - let sum = add(a, b); - ctx.db.result().insert(ResultRow { id, sum }); + #[derive(SpacetimeType, Clone, Debug)] + pub enum Shape { + Circle(i32), + Rectangle(Rect), } - ``` - -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = result)] pub struct ResultRow { #[primary_key] pub id: i32, - pub sum: i32, + pub value: Shape, } - fn add(a: i32, b: i32) -> i32 { a + b } - #[reducer] - pub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - ctx.db.result().insert(ResultRow { id, sum: add(a, b) }); + pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { + ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) }); } ``` -- **The error**: `helper_func_sum_parity: spacetime sql failed: no such table: result` - -- **Explain the difference**: - - Missing public modifiers for struct fields and incorrect reducer function signature. - -- **Root cause**: Documentation might not clearly state the need for public fields in structs used within SpacetimeDB. - -- **Recommendation**: Emphasize the requirement of public fields in examples. - ---- - -### C# / docs Failures (5 total) - -#### 1. t_014_elementary_columns -- **The generated code**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [SpacetimeDB.Table(Name = "Primitive")] - public partial struct Primitive - { - [SpacetimeDB.PrimaryKey] - public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } - ``` - -- **The golden example**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [Table(Name = "Primitive")] - public partial struct Primitive - { - [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } +3. **The error**: + ```plaintext + sum_type_row_parity: spacetime sql failed: no such table: `result`. ``` -- **The error**: `no such table: primitive` - -- **Explain the difference**: Field visibility was not explicitly made public in the generated code, which is a requirement for SpacetimeDB. +4. **Explain the difference**: + - The LLM did not make the necessary fields public, which can lead to visibility issues with database inserts. -- **Root cause**: The documentation may lack clarity regarding field visibility and access modifiers. +5. **Root cause**: + - The documentation may not be clear about the visibility of struct fields. -- **Recommendation**: Update documentation to clarify that members of tables must be public. +6. **Recommendation**: + - Update the documentation to stress the importance of public access for all fields that interact with the database. --- -(Continue this format for the remaining C# failures...) +### Summary Recommendations ---- - -### Conclusion - -This comprehensive analysis of SpacetimeDB benchmark test failures highlights key areas where the documentation can improve self-guidance for developers. Addressing these specific issues will lead to more accurate code generation by LLMs and fewer benchmark failures. +- **Visibility**: The documentation should consistently emphasize that public access is necessary for struct fields used in SpacetimeDB. +- **Reducer Necessity**: Clarify when reducers should be defined in accordance with specific use cases (e.g., initial data setup). +- **Attribute Usage**: Provide specific examples of attribute usage, including trait bounds for types like `ScheduleAt` and when/how to annotate functions within data models. + +By addressing the above gaps in the documentation, future errors can be mitigated, making it easier for developers to utilize SpacetimeDB effectively. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index de9b098fefa..02fff94c87c 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 20/27 | 76.4% ⬆️ +2.1% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% | -| Rust | rustdoc_json | **total** | 46/61 | **75.9%** ⬆️ +1.1% | +| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +6.9% | +| Rust | rustdoc_json | schema | 18/34 | 55.3% ⬇️ -20.0% | +| Rust | rustdoc_json | **total** | 43/61 | **70.6%** ⬇️ -5.3% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | -| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | -| C# | docs | schema | 21/34 | 63.7% | -| C# | docs | **total** | 48/61 | **83.5%** ⬆️ +4.5% | +| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | +| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | +| C# | docs | basics | 27/27 | 100.0% | +| C# | docs | schema | 25/34 | 73.7% ⬆️ +10.0% | +| C# | docs | **total** | 52/61 | **88.0%** ⬆️ +4.5% | _Compared against master branch baseline_ -Generated at: 2026-01-26T15:42:12.228Z +Generated at: 2026-01-27T16:47:52.904Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 51adf7934c1..1f525cb5996 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,21 +5,21 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,25 +35,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975549166Z", - "finished_at": "2026-01-26T15:31:47.073747240Z" + "started_at": "2026-01-27T16:32:33.607423222Z", + "finished_at": "2026-01-27T16:34:08.828085880Z" }, "t_001_basic_tables": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -69,25 +69,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.974869540Z", - "finished_at": "2026-01-26T15:31:51.422745193Z" + "started_at": "2026-01-27T16:34:07.172274692Z", + "finished_at": "2026-01-27T16:35:04.564067732Z" }, "t_002_scheduled_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::info!(\"tick\");\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +99,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling constant_time_eq v0.4.2\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling syn v2.0.114\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:19:36\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:15\n |\n18 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:16\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:21:39\n |\n21 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975688871Z", - "finished_at": "2026-01-26T15:32:30.863567064Z" + "started_at": "2026-01-27T16:34:08.828209290Z", + "finished_at": "2026-01-27T16:35:26.918284244Z" }, "t_003_struct_in_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::ReducerContext;\nuse spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -125,32 +125,37 @@ "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { "golden_db": "basics-t-003-struct-in-table-golden", "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "reducers_diff": { + "only_golden": [], + "only_llm": [ + "init()" + ] + }, + "reducers_equal": false, + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975098187Z", - "finished_at": "2026-01-26T15:31:55.504039476Z" + "started_at": "2026-01-27T16:34:07.169845549Z", + "finished_at": "2026-01-27T16:35:07.116018289Z" }, "t_004_insert": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &spacetimedb::ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -158,6 +163,25 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { + "data_parity_insert_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "insert_user", + "server": "http://127.0.0.1:35791" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -166,33 +190,25 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } - }, - "data_parity_insert_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002173ebf56f08f752bb2ca55d0547329e19fb14624413a7089817795d01ab/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975407164Z", - "finished_at": "2026-01-26T15:31:42.241214465Z" + "started_at": "2026-01-27T16:34:08.208097234Z", + "finished_at": "2026-01-27T16:35:13.309948498Z" }, "t_005_update": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user = User { id, name, age, active };\n ctx.db.user().id().update(user);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -216,7 +232,7 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" } }, "seed_users_row": { @@ -234,18 +250,18 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:42.241280124Z", - "finished_at": "2026-01-26T15:32:22.818349314Z" + "started_at": "2026-01-27T16:32:30.244848815Z", + "finished_at": "2026-01-27T16:34:12.499371947Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -269,13 +285,6 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -284,25 +293,32 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:49.227905993Z", - "finished_at": "2026-01-26T15:32:24.290487857Z" + "started_at": "2026-01-27T16:32:20.195987848Z", + "finished_at": "2026-01-27T16:34:08.193093663Z" }, "t_007_crud": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_007_crud", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n if let Some(mut u) = users.id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n users.id().update(u);\n }\n\n users.id().delete(&2);\n}", + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n let _ = users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n let _ = users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n users.id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -310,28 +326,65 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling generic-array v0.14.7\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:17:11\n |\n17 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | users.try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:24:11\n |\n24 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | users.try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } + }, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true + } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:35791" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975950122Z", - "finished_at": "2026-01-26T15:32:16.012153929Z" + "started_at": "2026-01-27T16:32:23.303951905Z", + "finished_at": "2026-01-27T16:34:10.621743269Z" }, "t_008_index_lookup": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let _ = ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -359,7 +412,7 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" } }, "schema_parity": { @@ -370,25 +423,25 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.511640110Z", - "finished_at": "2026-01-26T15:32:25.220589761Z" + "started_at": "2026-01-27T16:34:08.199928363Z", + "finished_at": "2026-01-27T16:35:12.331943993Z" }, "t_009_init": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let _ = ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n let _ = ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -396,13 +449,22 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, "schema_parity": { @@ -413,36 +475,27 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.073777338Z", - "finished_at": "2026-01-26T15:32:23.492928074Z" + "started_at": "2026-01-27T16:32:30.244175665Z", + "finished_at": "2026-01-27T16:34:05.774125311Z" }, "t_010_connect": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -465,24 +518,24 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975818475Z", - "finished_at": "2026-01-26T15:31:49.227873175Z" + "started_at": "2026-01-27T16:32:22.434743282Z", + "finished_at": "2026-01-27T16:34:04.716053382Z" }, "t_011_helper_function": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, + "passed_tests": 3, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -491,25 +544,22 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "helper_func_sum_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + 2, + 3 + ], "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20018816d0c78b41d0a1ebdd62b788e682bdd9fd17555d6ba26e941ae120e74/sql)\n", - "phase": "sql_golden" + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM result WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:35791" } }, "helper_func_sum_abs": { @@ -520,21 +570,34 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975246967Z", - "finished_at": "2026-01-26T15:31:47.511599186Z" + "started_at": "2026-01-27T16:34:08.208709446Z", + "finished_at": "2026-01-27T16:35:04.173989578Z" }, "t_012_spacetime_product_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -557,7 +620,16 @@ "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_score", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" + } + }, + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -568,34 +640,25 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } - }, - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:24.290533494Z", - "finished_at": "2026-01-26T15:32:53.440043766Z" + "started_at": "2026-01-27T16:32:19.074381549Z", + "finished_at": "2026-01-27T16:34:04.285926069Z" }, "t_013_spacetime_sum_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -603,23 +666,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:34081" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -628,11 +674,19 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } }, + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20002a07d9e6caa3aff0cce8d70357fc0dab19fc7bec72594d9469cfd9e0d34/sql)\n", + "phase": "sql_golden" + } + }, "sum_type_row_count": { "pass": true, "partial": 1.0, @@ -644,18 +698,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:29.610801727Z", - "finished_at": "2026-01-26T15:33:11.012437922Z" + "started_at": "2026-01-27T16:30:59.526974942Z", + "finished_at": "2026-01-27T16:32:33.607356316Z" }, "t_014_elementary_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -663,15 +717,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -680,7 +725,7 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } @@ -696,23 +741,32 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" + } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:25.220640456Z", - "finished_at": "2026-01-26T15:32:53.137088661Z" + "started_at": "2026-01-27T16:32:12.570231067Z", + "finished_at": "2026-01-27T16:34:07.169774438Z" }, "t_015_product_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.profile().id().find(1i32).is_none() {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -720,56 +774,50 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_columns_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20022cf10da13cb5686a3d982e8ce97257e0a609cd8cdbf4b75448c0605a05c/sql)\n", + "phase": "sql_golden" } }, - "product_type_columns_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:30.863583165Z", - "finished_at": "2026-01-26T15:33:07.055207496Z" + "started_at": "2026-01-27T16:30:59.529279381Z", + "finished_at": "2026-01-27T16:32:19.074245877Z" }, "t_016_sum_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawing = ctx.db.drawing();\n if drawing.count() == 0 {\n drawing.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawing();\n if drawings.count() == 0 {\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -781,18 +829,10 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20084ed1de024b608d5d036674102ffea5602feb56e936989c5737d1fa347f1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c200d48d9295216edd01ffee50fc27f7162d3c6ce2ff1442fb683d8707911992/sql)\n", "phase": "sql_golden" } }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c200f0f8cf7ce43e0b888fecfda5572f02aac26af6bea4ac6867cab46bd5e36f/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -801,25 +841,33 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } + }, + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c200ed636d1809b5fcb164f6ee86a539d4bdcbe0e1bcc7e6343341fa806bb61a/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:47.540914487Z", - "finished_at": "2026-01-26T15:33:39.147225272Z" + "started_at": "2026-01-27T16:30:59.526533974Z", + "finished_at": "2026-01-27T16:32:27.393830604Z" }, "t_017_scheduled_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -831,24 +879,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:22:11\n |\n22 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.7\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:19:36\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:15\n |\n18 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:16\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:21:39\n |\n21 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:23.493005867Z", - "finished_at": "2026-01-26T15:33:38.753749775Z" + "started_at": "2026-01-27T16:30:59.525424836Z", + "finished_at": "2026-01-27T16:32:23.303916011Z" }, "t_018_constraints": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n let ids: Vec = account.iter().map(|r| r.id).collect();\n for id in ids {\n account.id().delete(&id);\n }\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -857,17 +905,24 @@ "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c2002a3ca814e8fc4e610af01b0cba5740eac41619685f4959b780669796967a/sql)\n", + "phase": "sql_golden" + } + }, + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-018-constraints-golden", - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true } }, "constraints_seed_two_rows": { @@ -878,34 +933,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:16.012174316Z", - "finished_at": "2026-01-26T15:32:54.091817951Z" + "started_at": "2026-01-27T16:30:59.520529105Z", + "finished_at": "2026-01-27T16:32:22.433999235Z" }, "t_019_many_to_many": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 5, - "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exact dataset\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().delete(&m);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().delete(&u);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().delete(&g);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -913,69 +955,28 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.3\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling ppv-lite86 v0.2.21\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm)\nerror[E0308]: mismatched types\n --> src/lib.rs:33:36\n |\n 33 | ctx.db.membership().delete(&m);\n | ------ ^^ expected `Membership`, found `&Membership`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 33 - ctx.db.membership().delete(&m);\n 33 + ctx.db.membership().delete(m);\n |\n\nerror[E0308]: mismatched types\n --> src/lib.rs:36:30\n |\n 36 | ctx.db.user().delete(&u);\n | ------ ^^ expected `User`, found `&User`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 36 - ctx.db.user().delete(&u);\n 36 + ctx.db.user().delete(u);\n |\n\nerror[E0308]: mismatched types\n --> src/lib.rs:39:31\n |\n 39 | ctx.db.group().delete(&g);\n | ------ ^^ expected `Group`, found `&Group`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 39 - ctx.db.group().delete(&g);\n 39 + ctx.db.group().delete(g);\n |\n\nFor more information about this error, try `rustc --explain E0308`.\nerror: could not compile `spacetime-module` (lib) due to 3 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:22.818452395Z", - "finished_at": "2026-01-26T15:33:12.862647549Z" + "started_at": "2026-01-27T16:30:59.527880905Z", + "finished_at": "2026-01-27T16:32:20.195931017Z" }, "t_020_ecs": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(p.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: p.entity_id, x: nx, y: ny });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n let entity_tbl = ctx.db.entity();\n entity_tbl.id().delete(&1);\n entity_tbl.id().delete(&2);\n entity_tbl.insert(Entity { id: 1 });\n entity_tbl.insert(Entity { id: 2 });\n\n // Positions\n let pos_tbl = ctx.db.position();\n pos_tbl.entity_id().delete(&1);\n pos_tbl.entity_id().delete(&2);\n pos_tbl.insert(Position { entity_id: 1, x: 0, y: 0 });\n pos_tbl.insert(Position { entity_id: 2, x: 10, y: 0 });\n\n // Velocities\n let vel_tbl = ctx.db.velocity();\n vel_tbl.entity_id().delete(&1);\n vel_tbl.entity_id().delete(&2);\n vel_tbl.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n vel_tbl.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n let pos_tbl = ctx.db.position();\n let vel_idx = ctx.db.velocity().entity_id();\n let next_tbl = ctx.db.next_position();\n\n for p in pos_tbl.iter() {\n if let Some(v) = vel_idx.find(p.entity_id) {\n let new_x = p.x + v.vx;\n let new_y = p.y + v.vy;\n\n if let Some(mut existing) = next_tbl.entity_id().find(p.entity_id) {\n existing.x = new_x;\n existing.y = new_y;\n let _updated = next_tbl.entity_id().update(existing);\n } else {\n next_tbl.insert(NextPosition {\n entity_id: p.entity_id,\n x: new_x,\n y: new_y,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -983,27 +984,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", - "phase": "sql" - } - }, "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", "phase": "sql" } }, @@ -1015,33 +1008,41 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } }, - "ecs_step_next_positions_count": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:51.422780227Z", - "finished_at": "2026-01-26T15:32:47.540737141Z" + "started_at": "2026-01-27T16:30:59.528443334Z", + "finished_at": "2026-01-27T16:32:30.244044278Z" }, "t_021_multi_column_index": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.log();\n\n // Clear existing rows to ensure the table contains exactly the seeded rows\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1049,13 +1050,13 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, "schema_parity": { @@ -1066,7 +1067,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } @@ -1080,19 +1081,19 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, - "mcindex_lookup_u7_d1": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:55.504076208Z", - "finished_at": "2026-01-26T15:32:29.610771248Z" + "started_at": "2026-01-27T16:30:59.525967187Z", + "finished_at": "2026-01-27T16:32:12.570150592Z" } } } @@ -1114,7 +1115,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1132,8 +1133,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.378312906Z", - "finished_at": "2026-01-26T15:34:06.312973975Z" + "started_at": "2026-01-27T16:37:07.698626526Z", + "finished_at": "2026-01-27T16:38:22.803963062Z" }, "t_001_basic_tables": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1161,8 +1162,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.366156833Z", - "finished_at": "2026-01-26T15:34:21.141281364Z" + "started_at": "2026-01-27T16:37:11.008540155Z", + "finished_at": "2026-01-27T16:38:33.131965123Z" }, "t_002_scheduled_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1172,7 +1173,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) { }\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1190,8 +1191,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.381646420Z", - "finished_at": "2026-01-26T15:35:08.176674329Z" + "started_at": "2026-01-27T16:37:40.463756751Z", + "finished_at": "2026-01-27T16:38:46.674647986Z" }, "t_003_struct_in_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1201,7 +1202,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1219,8 +1220,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.368608920Z", - "finished_at": "2026-01-26T15:34:21.997549378Z" + "started_at": "2026-01-27T16:37:10.913641809Z", + "finished_at": "2026-01-27T16:38:28.540002548Z" }, "t_004_insert": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1256,8 +1257,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.375009734Z", - "finished_at": "2026-01-26T15:35:05.975054748Z" + "started_at": "2026-01-27T16:37:12.398406350Z", + "finished_at": "2026-01-27T16:38:29.957852183Z" }, "t_005_update": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1275,20 +1276,20 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "data_parity_update_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" } }, "seed_users_row": { @@ -1302,8 +1303,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:06.313026786Z", - "finished_at": "2026-01-26T15:34:38.434768448Z" + "started_at": "2026-01-27T16:37:05.840143086Z", + "finished_at": "2026-01-27T16:38:27.673339086Z" }, "t_006_delete": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1321,13 +1322,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, + "seed_users_row": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -1338,19 +1339,19 @@ "phase": "describe_golden" } }, - "seed_users_row": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:38.434794482Z", - "finished_at": "2026-01-26T15:35:08.669675470Z" + "started_at": "2026-01-27T16:36:39.493868942Z", + "finished_at": "2026-01-27T16:37:10.913606401Z" }, "t_007_crud": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1360,7 +1361,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1368,21 +1369,13 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id1_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, "notes": { "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "schema_parity": { @@ -1393,19 +1386,27 @@ "phase": "describe_golden" } }, - "crud_total_count_one": { + "crud_row_id1_parity": { "pass": false, "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "call_reducer_golden" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.388241938Z", - "finished_at": "2026-01-26T15:35:08.142795838Z" + "started_at": "2026-01-27T16:36:46.682301461Z", + "finished_at": "2026-01-27T16:37:30.046001557Z" }, "t_008_index_lookup": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1415,7 +1416,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let row = ResultRow { id: u.id, name: u.name.clone() };\n if ctx.db.result().id().find(u.id).is_some() {\n ctx.db.result().id().update(row);\n } else {\n ctx.db.result().insert(row);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1450,8 +1451,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.997572287Z", - "finished_at": "2026-01-26T15:34:53.278840948Z" + "started_at": "2026-01-27T16:37:11.012380843Z", + "finished_at": "2026-01-27T16:38:41.409499176Z" }, "t_009_init": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1461,7 +1462,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1478,22 +1479,22 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, - "init_total_two": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "init_seed_alice": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "schema_parity": { @@ -1506,8 +1507,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.141305493Z", - "finished_at": "2026-01-26T15:34:43.189086037Z" + "started_at": "2026-01-27T16:36:57.771441165Z", + "finished_at": "2026-01-27T16:37:40.463630543Z" }, "t_010_connect": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1517,7 +1518,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1535,8 +1536,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.384904874Z", - "finished_at": "2026-01-26T15:35:08.049424018Z" + "started_at": "2026-01-27T16:36:39.717206337Z", + "finished_at": "2026-01-27T16:37:11.008504607Z" }, "t_011_helper_function": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1546,7 +1547,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1581,8 +1582,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.371740140Z", - "finished_at": "2026-01-26T15:35:03.413658925Z" + "started_at": "2026-01-27T16:37:30.046112807Z", + "finished_at": "2026-01-27T16:38:42.478505901Z" }, "t_012_spacetime_product_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1592,7 +1593,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1627,8 +1628,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.176748927Z", - "finished_at": "2026-01-26T15:35:33.331707923Z" + "started_at": "2026-01-27T16:36:36.282196616Z", + "finished_at": "2026-01-27T16:37:12.398375835Z" }, "t_013_spacetime_sum_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1638,7 +1639,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1646,6 +1647,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "call_reducer_golden" + } + }, "sum_type_row_count": { "pass": false, "partial": 0.0, @@ -1662,19 +1671,11 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", "phase": "describe_golden" } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.669696395Z", - "finished_at": "2026-01-26T15:35:36.590766489Z" + "started_at": "2026-01-27T16:35:27.761679764Z", + "finished_at": "2026-01-27T16:36:39.717100673Z" }, "t_014_elementary_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1684,7 +1685,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000_i64,\n price: 1.5_f32,\n ratio: 2.25_f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1692,6 +1693,15 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1707,20 +1717,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "call_reducer_golden" } - }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.180313713Z", - "finished_at": "2026-01-26T15:35:31.695987649Z" + "started_at": "2026-01-27T16:36:31.329332201Z", + "finished_at": "2026-01-27T16:37:07.698588033Z" }, "t_015_product_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1730,7 +1731,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".into(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".into(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1747,26 +1748,26 @@ "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:17.784622487Z", - "finished_at": "2026-01-26T15:35:56.892022012Z" + "started_at": "2026-01-27T16:35:27.778895055Z", + "finished_at": "2026-01-27T16:36:36.282016589Z" }, "t_016_sum_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1776,7 +1777,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.drawing().try_insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1792,26 +1793,26 @@ "phase": "describe_golden" } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c200e332ee39012f68748e8e0058b1df01bfd4bd42fa50c7430f6e03a52a13d7/sql)\n", - "phase": "sql" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c2008da059ad558df05ae36956ea06f6277a53e08f40f6950324728e3d63cfb8/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:20.422565504Z", - "finished_at": "2026-01-26T15:35:53.522136009Z" + "started_at": "2026-01-27T16:35:27.755209342Z", + "finished_at": "2026-01-27T16:37:05.840061596Z" }, "t_017_scheduled_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1821,7 +1822,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let interval = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: interval,\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1829,14 +1830,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } - }, "scheduled_seeded_one_row": { "pass": true, "partial": 1.0, @@ -1845,11 +1838,19 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.142877410Z", - "finished_at": "2026-01-26T15:35:53.117891027Z" + "started_at": "2026-01-27T16:35:27.744787479Z", + "finished_at": "2026-01-27T16:37:10.715377293Z" }, "t_018_constraints": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1859,7 +1860,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().id().delete(&1);\n ctx.db.account().id().delete(&2);\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1894,73 +1895,37 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:03.413773673Z", - "finished_at": "2026-01-26T15:35:40.991202532Z" + "started_at": "2026-01-27T16:35:27.741095372Z", + "finished_at": "2026-01-27T16:36:39.493832562Z" }, "t_019_many_to_many": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:05.975134623Z", - "finished_at": "2026-01-26T15:35:55.889702970Z" + "started_at": "2026-01-27T16:38:46.674706560Z", + "finished_at": "2026-01-27T16:38:46.674706560Z" }, "t_020_ecs": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1970,7 +1935,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n let id1 = 1;\n if ctx.db.entity().id().find(id1).is_none() {\n ctx.db.entity().insert(Entity { id: id1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n let id2 = 2;\n if ctx.db.entity().id().find(id2).is_none() {\n ctx.db.entity().insert(Entity { id: id2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1978,19 +1943,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } }, @@ -2002,11 +1967,11 @@ "phase": "describe_golden" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } }, @@ -2014,14 +1979,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:43.189115873Z", - "finished_at": "2026-01-26T15:35:20.422539334Z" + "started_at": "2026-01-27T16:35:27.774072550Z", + "finished_at": "2026-01-27T16:36:31.328546056Z" }, "t_021_multi_column_index": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -2031,7 +1996,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2039,23 +2004,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2073,11 +2021,28 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:53.278868473Z", - "finished_at": "2026-01-26T15:35:17.784590880Z" + "started_at": "2026-01-27T16:35:27.749626445Z", + "finished_at": "2026-01-27T16:36:46.682079929Z" } } } @@ -2298,15 +2263,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:48.351519518Z", - "finished_at": "2026-01-26T15:39:15.916317553Z" + "started_at": "2026-01-27T16:45:33.624143092Z", + "finished_at": "2026-01-27T16:46:27.480022955Z" }, "t_001_basic_tables": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2332,15 +2297,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.611034643Z", - "finished_at": "2026-01-26T15:39:10.630797919Z" + "started_at": "2026-01-27T16:45:58.037479404Z", + "finished_at": "2026-01-27T16:46:46.072177909Z" }, "t_002_scheduled_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2350,7 +2315,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2366,15 +2331,15 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:55.763350829Z", - "finished_at": "2026-01-26T15:39:35.117748962Z" + "started_at": "2026-01-27T16:46:46.072324097Z", + "finished_at": "2026-01-27T16:47:52.804047987Z" }, "t_003_struct_in_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2384,7 +2349,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -2400,15 +2365,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.614826701Z", - "finished_at": "2026-01-26T15:38:48.351485744Z" + "started_at": "2026-01-27T16:45:44.449789708Z", + "finished_at": "2026-01-27T16:46:27.856049534Z" }, "t_004_insert": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2418,7 +2383,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -2426,19 +2391,6 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_insert_user": { "pass": true, "partial": 1.0, @@ -2455,13 +2407,26 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.624534968Z", - "finished_at": "2026-01-26T15:38:55.763316545Z" + "started_at": "2026-01-27T16:46:29.820748855Z", + "finished_at": "2026-01-27T16:47:18.121969583Z" }, "t_005_update": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2471,7 +2436,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2487,7 +2452,7 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -2508,7 +2473,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, "seed_users_row": { @@ -2520,8 +2485,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:15.916349271Z", - "finished_at": "2026-01-26T15:39:57.685388275Z" + "started_at": "2026-01-27T16:45:30.312651937Z", + "finished_at": "2026-01-27T16:46:29.809622619Z" }, "t_006_delete": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2531,7 +2496,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2563,15 +2528,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:51.394765047Z", - "finished_at": "2026-01-26T15:40:20.535132716Z" + "started_at": "2026-01-27T16:44:29.592100157Z", + "finished_at": "2026-01-27T16:45:30.312571408Z" }, "t_007_crud": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2581,7 +2546,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n // delete Id=2\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.User.Id.Update(u1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2589,17 +2554,13 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "crud_row_id2_deleted": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" } }, "crud_total_count_one": { @@ -2622,22 +2583,26 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, - "crud_row_id2_deleted": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:10.630849129Z", - "finished_at": "2026-01-26T15:39:51.394722435Z" + "started_at": "2026-01-27T16:44:36.788013061Z", + "finished_at": "2026-01-27T16:45:44.449720987Z" }, "t_008_index_lookup": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2647,7 +2612,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Id.Delete(id);\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2655,19 +2620,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "seed_user_row": { "pass": true, "partial": 1.0, @@ -2688,13 +2640,26 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:35.117796353Z", - "finished_at": "2026-01-26T15:40:10.865109962Z" + "started_at": "2026-01-27T16:46:29.809976029Z", + "finished_at": "2026-01-27T16:47:52.369916015Z" }, "t_009_init": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2704,7 +2669,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2712,22 +2677,26 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } }, "init_seed_alice": { @@ -2739,23 +2708,19 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "schema_parity": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:33.038774271Z", - "finished_at": "2026-01-26T15:40:09.097960805Z" + "started_at": "2026-01-27T16:45:00.485893120Z", + "finished_at": "2026-01-27T16:45:58.037338911Z" }, "t_010_connect": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2765,7 +2730,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2781,15 +2746,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:07.129948444Z", - "finished_at": "2026-01-26T15:39:33.038733830Z" + "started_at": "2026-01-27T16:44:29.599798551Z", + "finished_at": "2026-01-27T16:45:33.624088924Z" }, "t_011_helper_function": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2799,7 +2764,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2815,7 +2780,7 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -2835,7 +2800,7 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, "helper_func_sum_abs": { @@ -2849,8 +2814,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.619502537Z", - "finished_at": "2026-01-26T15:39:07.129901962Z" + "started_at": "2026-01-27T16:46:29.832685293Z", + "finished_at": "2026-01-27T16:47:21.941754653Z" }, "t_012_spacetime_product_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2860,7 +2825,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right },\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2868,17 +2833,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, "product_type_row_parity": { @@ -2896,22 +2857,26 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, - "product_type_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:45.264170257Z", - "finished_at": "2026-01-26T15:41:10.281722543Z" + "started_at": "2026-01-27T16:44:13.602642076Z", + "finished_at": "2026-01-27T16:45:00.485815025Z" }, "t_013_spacetime_sum_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2921,7 +2886,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2937,20 +2902,11 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, "sum_type_row_parity": { "pass": true, "partial": 1.0, @@ -2965,13 +2921,22 @@ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetCircle", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" + } + }, + "sum_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.281769568Z", - "finished_at": "2026-01-26T15:42:04.928512771Z" + "started_at": "2026-01-27T16:42:33.200019323Z", + "finished_at": "2026-01-27T16:43:38.115938270Z" }, "t_014_elementary_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2981,7 +2946,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -2989,11 +2954,24 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true + } + }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20015ce3d6d6c97d4d92483e7212bbbdc2017cebf40569b0de2758a8c06d8c9/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c2004f34121dd1f4ae79dcdbda66d549feb82a5d63960ba14565d5cc8b739c97/sql)\n", "phase": "sql" } }, @@ -3001,27 +2979,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2002dc67d9e1fd4e2f7be96328c9cadfe424ab52e14114ffe15bf0808847523/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200543789d2e744179bd83fb1415d87abe1ef9050fbc5d65555128998b27490/sql)\n", "phase": "sql_golden" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:59.748869772Z", - "finished_at": "2026-01-26T15:41:30.294907546Z" + "started_at": "2026-01-27T16:43:38.116010007Z", + "finished_at": "2026-01-27T16:44:36.787949888Z" }, "t_015_product_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3031,7 +2996,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n var home = new Address { Street = \"1 Main\", Zip = 11111 };\n var work = new Address { Street = \"2 Broad\", Zip = 22222 };\n var pos = new Position { X = 7, Y = 9 };\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = home,\n Work = work,\n Pos = pos\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 },\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3048,37 +3013,37 @@ "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:32995" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:41793" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.284990504Z", - "finished_at": "2026-01-26T15:41:51.914201320Z" + "started_at": "2026-01-27T16:43:18.813807927Z", + "finished_at": "2026-01-27T16:44:27.109398198Z" }, "t_016_sum_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3088,7 +3053,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3096,6 +3061,14 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200a6ce7b1295896bce91792742020e2d963fc68bfca6d8ff9804106ed485f8/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3104,31 +3077,23 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2007921ce30371e511ce337c700f912b2dc1842029ec51095bcee487a1fb5a2/sql)\n", - "phase": "sql_golden" - } - }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20008b060dbbbe2ff941855e14b04bb0c88b907fca414a2d82874c574b1cf60/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200bbb059f1df73468f61a994e99b02f150ebdc9dce5396fc2cc064f5ebb917/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:12.893136011Z", - "finished_at": "2026-01-26T15:42:12.178253367Z" + "started_at": "2026-01-27T16:41:37.626585140Z", + "finished_at": "2026-01-27T16:43:18.813740398Z" }, "t_017_scheduled_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3138,7 +3103,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled tick; no-op\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3146,14 +3111,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20077e1b6738482c8dd0108623bedd0c70f80fb40fb09fda1f12bbe2c6972bf/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3162,15 +3119,23 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } + }, + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200283fa6a6ce49e85887cd3c05f6cd55c39e71c5cdd5dc78a67d45df2c41c4/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:27.963020126Z", - "finished_at": "2026-01-26T15:41:12.893111985Z" + "started_at": "2026-01-27T16:41:37.619157633Z", + "finished_at": "2026-01-27T16:42:40.765400962Z" }, "t_018_constraints": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3180,7 +3145,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Account.Iter())\n {\n ctx.Db.Account.Id.Delete(row.Id);\n }\n\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -3188,6 +3153,20 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { + "constraints_row_parity_after_seed": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-018-constraints-golden", + "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:32995" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3196,7 +3175,7 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -3209,25 +3188,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-018-constraints-golden", - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:41793" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:10.865255262Z", - "finished_at": "2026-01-26T15:40:59.748828990Z" + "started_at": "2026-01-27T16:41:37.615926753Z", + "finished_at": "2026-01-27T16:42:33.199931492Z" }, "t_019_many_to_many": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3237,7 +3202,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3253,7 +3218,7 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -3267,15 +3232,6 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - }, "m2m_has_1_10": { "pass": true, "partial": 1.0, @@ -3293,11 +3249,20 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:20.535170845Z", - "finished_at": "2026-01-26T15:41:10.109404307Z" + "started_at": "2026-01-27T16:42:40.765467116Z", + "finished_at": "2026-01-27T16:44:13.602547935Z" }, "t_020_ecs": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3307,7 +3272,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Insert entities\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n // Insert positions\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n // Insert velocities\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(p.EntityId) is Velocity v)\n {\n int nx = p.X + v.VX;\n int ny = p.Y + v.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(p.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = p.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3315,27 +3280,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", "phase": "sql" } }, @@ -3347,7 +3304,7 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -3356,24 +3313,32 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:57.685434816Z", - "finished_at": "2026-01-26T15:40:45.264108157Z" + "started_at": "2026-01-27T16:43:06.400065322Z", + "finished_at": "2026-01-27T16:44:29.591963094Z" }, "t_021_multi_column_index": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_021_multi_column_index", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"Log\", Public = true)]\n [Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Log.Iter())\n {\n ctx.Db.Log.Id.Delete(row.Id);\n }\n\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3381,18 +3346,50 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" + } + }, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + } + }, + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:09.098001666Z", - "finished_at": "2026-01-26T15:40:27.962996760Z" + "started_at": "2026-01-27T16:41:37.622718575Z", + "finished_at": "2026-01-27T16:43:06.399960112Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 4e4aa267103..56cb6e89647 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-26T15:42:12.228Z", + "generated_at": "2026-01-27T16:47:52.904Z", "by_language": { "csharp": { "modes": { @@ -20,19 +20,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 21, - "pass_pct": 61.764706, - "task_pass_equiv": 6.3666663, - "task_pass_pct": 63.666664 + "passed_tests": 25, + "pass_pct": 73.52941, + "task_pass_equiv": 7.3666663, + "task_pass_pct": 73.666664 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 18.366667, - "task_pass_pct": 83.48485 + "passed_tests": 52, + "pass_pct": 85.2459, + "task_pass_equiv": 19.366667, + "task_pass_pct": 88.030304 } } } @@ -56,53 +56,53 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 8, - "pass_pct": 23.529411, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 + "total_tests": 30, + "passed_tests": 4, + "pass_pct": 13.333333, + "task_pass_equiv": 1.25, + "task_pass_pct": 12.5 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 13, - "pass_pct": 21.311476, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 + "total_tests": 57, + "passed_tests": 9, + "pass_pct": 15.789474, + "task_pass_equiv": 2.5833335, + "task_pass_pct": 11.742425 } } } }, "rustdoc_json": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 20, - "pass_pct": 74.07407, - "task_pass_equiv": 9.166667, - "task_pass_pct": 76.38889 + "passed_tests": 25, + "pass_pct": 92.59259, + "task_pass_equiv": 10.0, + "task_pass_pct": 83.33333 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 26, - "pass_pct": 76.47059, - "task_pass_equiv": 7.5333333, - "task_pass_pct": 75.333336 + "passed_tests": 18, + "pass_pct": 52.941177, + "task_pass_equiv": 5.5333333, + "task_pass_pct": 55.333336 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 46, - "pass_pct": 75.409836, - "task_pass_equiv": 16.7, - "task_pass_pct": 75.909096 + "passed_tests": 43, + "pass_pct": 70.491806, + "task_pass_equiv": 15.533334, + "task_pass_pct": 70.606064 } } }