Time bucketing using auto bins working

Author: matt-aitken

apps/webapp/app/components/code/tsql/tsqlCompletion.ts

@@ -123,6 +123,16 @@ function createFunctionCompletions(): Completion[] {
     });
   }
 
+  // Add special TSQL functions not in the ClickHouse function registry
+  functions.push({
+    label: "timeBucket",
+    type: "function",
+    detail: "auto time bucket (0 args)",
+    apply: "timeBucket()",
+    boost: 1.5,
+    info: "Automatically bucket by time using the table's time column. Interval is chosen based on the query's time range.",
+  });
+
   return functions;
 }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx

@@ -38,6 +38,19 @@ ORDER BY p50_duration_ms DESC
 LIMIT 20`,
     scope: "environment",
   },
+  {
+    title: "Runs over time",
+    description:
+      "Count of runs bucketed over time. The bucket size adjusts automatically to the time range.",
+    query: `SELECT
+  timeBucket(),
+  count() AS run_count
+FROM runs
+GROUP BY timeBucket
+ORDER BY timeBucket
+LIMIT 1000`,
+    scope: "environment",
+  },
   {
     title: "Most expensive 100 runs (past 7d)",
     description: "Top 100 runs by cost over the last 7 days.",

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TRQLGuideContent.tsx

@@ -488,6 +488,11 @@ ORDER BY run_count DESC`,
           <FunctionCategory
             title="Date/time functions"
             functions={[
+              {
+                name: "timeBucket()",
+                desc: "Auto-bucket by time period. Uses the table's time column with an interval based on the query's time range.",
+                example: "SELECT timeBucket(), count() FROM runs GROUP BY timeBucket",
+              },
               { name: "now()", desc: "Current date and time", example: "now()" },
               { name: "today()", desc: "Current date", example: "today()" },
               { name: "yesterday()", desc: "Yesterday's date", example: "yesterday()" },

apps/webapp/app/services/queryService.server.ts

@@ -18,7 +18,7 @@ import {
   GLOBAL_CONCURRENCY_LIMIT,
 } from "./queryConcurrencyLimiter.server";
 import { getLimit } from "./platform.v3.server";
-import { timeFilters } from "~/components/runs/v3/SharedFilters";
+import { timeFilters, timeFilterFromTo } from "~/components/runs/v3/SharedFilters";
 import parse from "parse-duration";
 import { querySchemas } from "~/v3/querySchemas";
 
@@ -205,6 +205,14 @@ export async function executeQuery<TOut extends z.ZodSchema>(
     queue: queues && queues.length > 0 ? { op: "in", values: queues } : undefined,
   } satisfies Record<string, WhereClauseCondition | undefined>;
 
+  // Compute the effective time range for timeBucket() interval calculation
+  const timeRange = timeFilterFromTo({
+    period: period ?? undefined,
+    from: from ?? undefined,
+    to: to ?? undefined,
+    defaultPeriod,
+  });
+
   try {
     // Build field mappings for project_ref → project_id and environment_id → slug translation
     const projects = await prisma.project.findMany({
@@ -232,6 +240,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
       whereClauseFallback: {
         triggered_at: triggeredAtFallback,
       },
+      timeRange,
       clickhouseSettings: {
         ...getDefaultClickhouseSettings(),
         ...baseOptions.clickhouseSettings, // Allow caller overrides if needed

apps/webapp/app/v3/querySchemas.ts

@@ -28,6 +28,7 @@ export const runsSchema: TableSchema = {
   name: "runs",
   clickhouseName: "trigger_dev.task_runs_v2",
   description: "Task runs - stores all task execution records",
+  timeConstraint: "triggered_at",
   tenantColumns: {
     organizationId: "organization_id",
     projectId: "project_id",

apps/webapp/app/v3/services/aiQueryService.server.ts

@@ -414,13 +414,28 @@ HAVING cnt > 10
 \`\`\`
 
 ### Date/Time Functions
+- timeBucket() - automatically bucket by time. Uses the table's time column and picks the best interval based on the query's time range. Use in SELECT and reference as \`timeBucket\` in GROUP BY / ORDER BY.
 - now() - current timestamp
 - today() - current date
 - toDate(datetime) - extract date
 - toStartOfDay/Hour/Minute(datetime)
 - dateDiff('unit', start, end) - difference in units (second, minute, hour, day, week, month, year)
 - INTERVAL n unit - time interval (e.g., INTERVAL 7 DAY)
 
+### Time Bucketing
+When the user wants to see data "over time", "by hour", "by day", or any time-series aggregation, prefer \`timeBucket()\` over manual \`toStartOfHour\`/\`toStartOfDay\` calls. \`timeBucket()\` automatically picks the right interval for the current time range.
+
+\`\`\`sql
+-- Runs over time (bucket size auto-selected)
+SELECT timeBucket(), count() AS run_count
+FROM runs
+GROUP BY timeBucket
+ORDER BY timeBucket
+LIMIT 1000
+\`\`\`
+
+Only use explicit \`toStartOfHour\`/\`toStartOfDay\` etc. if the user specifically requests a particular bucket size (e.g., "group by hour", "bucket by day").
+
 ### Common Patterns
 - Status filter: WHERE status = 'Failed' or WHERE status IN ('Failed', 'Crashed')
 - Time filtering: Use the \`setTimeFilter\` tool (NOT triggered_at in WHERE clause)
@@ -432,13 +447,14 @@ HAVING cnt > 10
 3. When column selection is ambiguous, use the core columns marked [CORE] in the schema
 4. **TIME FILTERING**: When the user wants to filter by time (e.g., "last 7 days", "past hour", "yesterday"), ALWAYS use the \`setTimeFilter\` tool instead of adding \`triggered_at\` conditions to the query. The UI has a time filter that will apply this automatically.
 5. Do NOT add \`triggered_at\` to WHERE clauses - use \`setTimeFilter\` tool instead. If the user doesn't specify a time period, do NOT add any time filter (the UI defaults to 7 days).
-6. ALWAYS use the validateTSQLQuery tool to check your query before returning it
-7. If validation fails, fix the issues and try again (up to 3 attempts)
-8. Use column names exactly as defined in the schema (case-sensitive)
-9. For enum columns like status, use the allowed values shown in the schema
-10. Always include a LIMIT clause (default to 100 if not specified)
-11. Use meaningful column aliases with AS for aggregations
-12. Format queries with proper indentation for readability
+6. **TIME BUCKETING**: When the user wants to see data over time or in time buckets, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
+7. ALWAYS use the validateTSQLQuery tool to check your query before returning it
+8. If validation fails, fix the issues and try again (up to 3 attempts)
+9. Use column names exactly as defined in the schema (case-sensitive)
+10. For enum columns like status, use the allowed values shown in the schema
+11. Always include a LIMIT clause (default to 100 if not specified)
+12. Use meaningful column aliases with AS for aggregations
+13. Format queries with proper indentation for readability
 
 ## Response Format
 
@@ -504,25 +520,38 @@ HAVING cnt > 10
 \`\`\`
 
 ### Date/Time Functions
+- timeBucket() - automatically bucket by time. Uses the table's time column and picks the best interval based on the query's time range. Use in SELECT and reference as \`timeBucket\` in GROUP BY / ORDER BY.
 - now() - current timestamp
 - today() - current date
 - toDate(datetime) - extract date
 - toStartOfDay/Hour/Minute(datetime)
 - dateDiff('unit', start, end) - difference in units (second, minute, hour, day, week, month, year)
 - INTERVAL n unit - time interval (e.g., INTERVAL 7 DAY)
 
+### Time Bucketing
+When the user wants to see data "over time", "by hour", "by day", or any time-series aggregation, prefer \`timeBucket()\` over manual \`toStartOfHour\`/\`toStartOfDay\` calls unless the user specifically requests a particular bucket size.
+
+\`\`\`sql
+SELECT timeBucket(), count() AS run_count
+FROM runs
+GROUP BY timeBucket
+ORDER BY timeBucket
+LIMIT 1000
+\`\`\`
+
 ## Important Rules
 
 1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
 2. If the existing query uses SELECT *, replace it with specific columns (use core columns marked [CORE] as defaults)
 3. **TIME FILTERING**: When the user wants to change time filtering (e.g., "change to last 30 days"), use the \`setTimeFilter\` tool instead of modifying \`triggered_at\` conditions. If the existing query has \`triggered_at\` in WHERE, consider removing it and using \`setTimeFilter\` instead.
-4. ALWAYS use the validateTSQLQuery tool to check your modified query before returning it
-5. If validation fails, fix the issues and try again (up to 3 attempts)
-6. Use column names exactly as defined in the schema (case-sensitive)
-7. For enum columns like status, use the allowed values shown in the schema
-8. Always include a LIMIT clause (default to 100 if not specified)
-9. Preserve the user's existing query structure and style where possible
-10. Only make the changes specifically requested by the user
+4. **TIME BUCKETING**: When adding time-series grouping, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
+5. ALWAYS use the validateTSQLQuery tool to check your modified query before returning it
+6. If validation fails, fix the issues and try again (up to 3 attempts)
+7. Use column names exactly as defined in the schema (case-sensitive)
+8. For enum columns like status, use the allowed values shown in the schema
+9. Always include a LIMIT clause (default to 100 if not specified)
+10. Preserve the user's existing query structure and style where possible
+11. Only make the changes specifically requested by the user
 
 ## Response Format
 

internal-packages/clickhouse/src/client/tsql.ts

@@ -14,6 +14,7 @@ import {
   type TableSchema,
   type QuerySettings,
   type FieldMappings,
+  type TimeRange,
   type WhereClauseCondition
 } from "@internal/tsql";
 import type { ClickhouseReader, QueryStats } from "./types.js";
@@ -25,7 +26,7 @@ const logger = new Logger("tsql", "info");
 
 export type { QueryStats };
 
-export type { TableSchema, QuerySettings, FieldMappings, WhereClauseCondition };
+export type { TableSchema, QuerySettings, FieldMappings, TimeRange, WhereClauseCondition };
 
 /**
  * Options for executing a TSQL query
@@ -101,6 +102,12 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> {
    * ```
    */
   whereClauseFallback?: Record<string, WhereClauseCondition>;
+  /**
+   * Time range for `timeBucket()` interval calculation.
+   * When provided, `timeBucket()` uses this to determine the appropriate bucket size
+   * based on the span of the time range.
+   */
+  timeRange?: TimeRange;
 }
 
 /**
@@ -183,6 +190,7 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
       settings: compiledSettings,
       fieldMappings: options.fieldMappings,
       whereClauseFallback: options.whereClauseFallback,
+      timeRange: options.timeRange,
     });
 
     generatedSql = sql;