feat: OTEL metrics pipeline for task workers

.changeset/fix-coderabbit-review-items.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/sdk": patch
+---
+
+Add OTEL metrics pipeline for task workers. Workers collect process CPU/memory, Node.js runtime metrics (event loop utilization, event loop delay, heap usage), and user-defined custom metrics via `otel.metrics.getMeter()`. Metrics are exported to ClickHouse with 10-second aggregation buckets and 1m/5m rollups, and are queryable through the dashboard query engine with typed attribute columns, `prettyFormat()` for human-readable values, and AI query support.

apps/webapp/app/components/code/QueryResultsChart.tsx

@@ -1,6 +1,9 @@
-import type { OutputColumnMetadata } from "@internal/clickhouse";
+import type { ColumnFormatType, OutputColumnMetadata } from "@internal/clickhouse";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
 import { BarChart3, LineChart } from "lucide-react";
 import { memo, useMemo } from "react";
+import { createValueFormatter } from "~/utils/columnFormat";
+import { formatCurrencyAccurate } from "~/utils/numberFormatter";
 import type { ChartConfig } from "~/components/primitives/charts/Chart";
 import { Chart } from "~/components/primitives/charts/ChartCompound";
 import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -798,8 +801,24 @@ export const QueryResultsChart = memo(function QueryResultsChart({
     };
   }, [isDateBased, timeGranularity]);
 
-  // Create dynamic Y-axis formatter based on data range
-  const yAxisFormatter = useMemo(() => createYAxisFormatter(data, series), [data, series]);
+  // Resolve the Y-axis column format for formatting
+  const yAxisFormat = useMemo(() => {
+    if (yAxisColumns.length === 0) return undefined;
+    const col = columns.find((c) => c.name === yAxisColumns[0]);
+    return (col?.format ?? col?.customRenderType) as ColumnFormatType | undefined;
+  }, [yAxisColumns, columns]);
+
+  // Create dynamic Y-axis formatter based on data range and format
+  const yAxisFormatter = useMemo(
+    () => createYAxisFormatter(data, series, yAxisFormat),
+    [data, series, yAxisFormat]
+  );
+
+  // Create value formatter for tooltips and legend based on column format
+  const tooltipValueFormatter = useMemo(
+    () => createValueFormatter(yAxisFormat),
+    [yAxisFormat]
+  );
 
   // Check if the group-by column has a runStatus customRenderType
   const groupByIsRunStatus = useMemo(() => {
@@ -1019,6 +1038,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
         showLegend={showLegend}
         maxLegendItems={fullLegend ? Infinity : 5}
         legendAggregation={config.aggregation}
+        legendValueFormatter={tooltipValueFormatter}
         minHeight="300px"
         fillContainer
         onViewAllLegendItems={onViewAllLegendItems}
@@ -1030,6 +1050,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
           yAxisProps={yAxisProps}
           stackId={stacked ? "stack" : undefined}
           tooltipLabelFormatter={tooltipLabelFormatter}
+          tooltipValueFormatter={tooltipValueFormatter}
         />
       </Chart.Root>
     );
@@ -1046,6 +1067,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
       showLegend={showLegend}
       maxLegendItems={fullLegend ? Infinity : 5}
       legendAggregation={config.aggregation}
+      legendValueFormatter={tooltipValueFormatter}
       minHeight="300px"
       fillContainer
       onViewAllLegendItems={onViewAllLegendItems}
@@ -1057,16 +1079,21 @@ export const QueryResultsChart = memo(function QueryResultsChart({
         yAxisProps={yAxisProps}
         stacked={stacked && sortedSeries.length > 1}
         tooltipLabelFormatter={tooltipLabelFormatter}
+        tooltipValueFormatter={tooltipValueFormatter}
         lineType="linear"
       />
     </Chart.Root>
   );
 });
 
 /**
- * Creates a Y-axis value formatter based on the data range
+ * Creates a Y-axis value formatter based on the data range and optional format hint
  */
-function createYAxisFormatter(data: Record<string, unknown>[], series: string[]) {
+function createYAxisFormatter(
+  data: Record<string, unknown>[],
+  series: string[],
+  format?: ColumnFormatType
+) {
   // Find min and max values across all series
   let minVal = Infinity;
   let maxVal = -Infinity;
@@ -1083,6 +1110,46 @@ function createYAxisFormatter(data: Record<string, unknown>[], series: string[])
 
   const range = maxVal - minVal;
 
+  // Format-aware formatters
+  if (format === "bytes" || format === "decimalBytes") {
+    const divisor = format === "bytes" ? 1024 : 1000;
+    const units =
+      format === "bytes"
+        ? ["B", "KiB", "MiB", "GiB", "TiB"]
+        : ["B", "KB", "MB", "GB", "TB"];
+    return (value: number): string => {
+      if (value === 0) return "0 B";
+      // Use consistent unit for all ticks based on max value
+      const i = Math.min(
+        Math.floor(Math.log(Math.abs(maxVal || 1)) / Math.log(divisor)),
+        units.length - 1
+      );
+      const scaled = value / Math.pow(divisor, i);
+      return `${scaled.toFixed(scaled < 10 ? 1 : 0)} ${units[i]}`;
+    };
+  }
+
+  if (format === "percent") {
+    return (value: number): string => `${value.toFixed(range < 1 ? 2 : 1)}%`;
+  }
+
+  if (format === "duration") {
+    return (value: number): string => formatDurationMilliseconds(value, { style: "short" });
+  }
+
+  if (format === "durationSeconds") {
+    return (value: number): string =>
+      formatDurationMilliseconds(value * 1000, { style: "short" });
+  }
+
+  if (format === "costInDollars" || format === "cost") {
+    return (value: number): string => {
+      const dollars = format === "cost" ? value / 100 : value;
+      return formatCurrencyAccurate(dollars);
+    };
+  }
+
+  // Default formatter
   return (value: number): string => {
     // Use abbreviations for large numbers
     if (Math.abs(value) >= 1_000_000) {

apps/webapp/app/components/code/TSQLResultsTable.tsx

@@ -35,6 +35,7 @@ import { useCopy } from "~/hooks/useCopy";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { cn } from "~/utils/cn";
+import { formatBytes, formatDecimalBytes, formatQuantity } from "~/utils/columnFormat";
 import { formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter";
 import { v3ProjectPath, v3RunPathFromFriendlyId } from "~/utils/pathBuilder";
 import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -66,9 +67,10 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
   if (value === null) return "NULL";
   if (value === undefined) return "";
 
-  // Handle custom render types
-  if (column.customRenderType) {
-    switch (column.customRenderType) {
+  // Handle format hints (from prettyFormat() or auto-populated from customRenderType)
+  const formatType = column.format ?? column.customRenderType;
+  if (formatType) {
+    switch (formatType) {
       case "duration":
         if (typeof value === "number") {
           return formatDurationMilliseconds(value, { style: "short" });
@@ -95,6 +97,26 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
           return value;
         }
         break;
+      case "bytes":
+        if (typeof value === "number") {
+          return formatBytes(value);
+        }
+        break;
+      case "decimalBytes":
+        if (typeof value === "number") {
+          return formatDecimalBytes(value);
+        }
+        break;
+      case "percent":
+        if (typeof value === "number") {
+          return `${value.toFixed(2)}%`;
+        }
+        break;
+      case "quantity":
+        if (typeof value === "number") {
+          return formatQuantity(value);
+        }
+        break;
     }
   }
 
@@ -222,6 +244,21 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
   if (value === null) return 4; // "NULL"
   if (value === undefined) return 9; // "UNDEFINED"
 
+  // Handle format hint types - estimate their rendered width
+  const fmt = column.format;
+  if (fmt === "bytes" || fmt === "decimalBytes") {
+    // e.g., "1.50 GiB" or "256.00 MB"
+    return 12;
+  }
+  if (fmt === "percent") {
+    // e.g., "45.23%"
+    return 8;
+  }
+  if (fmt === "quantity") {
+    // e.g., "1.50M"
+    return 8;
+  }
+
   // Handle custom render types - estimate their rendered width
   if (column.customRenderType) {
     switch (column.customRenderType) {
@@ -394,6 +431,10 @@ function isRightAlignedColumn(column: OutputColumnMetadata): boolean {
   ) {
     return true;
   }
+  const fmt = column.format;
+  if (fmt === "bytes" || fmt === "decimalBytes" || fmt === "percent" || fmt === "quantity") {
+    return true;
+  }
   return isNumericType(column.type);
 }
 
@@ -476,6 +517,32 @@ function CellValue({
     return <pre className="text-text-dimmed">UNDEFINED</pre>;
   }
 
+  // Check format hint for new format types (from prettyFormat())
+  if (column.format && !column.customRenderType) {
+    switch (column.format) {
+      case "bytes":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{formatBytes(value)}</span>;
+        }
+        break;
+      case "decimalBytes":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{formatDecimalBytes(value)}</span>;
+        }
+        break;
+      case "percent":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{value.toFixed(2)}%</span>;
+        }
+        break;
+      case "quantity":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{formatQuantity(value)}</span>;
+        }
+        break;
+    }
+  }
+
   // First check customRenderType for special rendering
   if (column.customRenderType) {
     switch (column.customRenderType) {

apps/webapp/app/components/primitives/charts/BigNumberCard.tsx

@@ -1,10 +1,11 @@
-import type { OutputColumnMetadata } from "@internal/tsql";
+import type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
 import { Hash } from "lucide-react";
 import { useMemo } from "react";
 import type {
   BigNumberAggregationType,
   BigNumberConfiguration,
 } from "~/components/metrics/QueryWidget";
+import { createValueFormatter } from "~/utils/columnFormat";
 import { AnimatedNumber } from "../AnimatedNumber";
 import { ChartBlankState } from "./ChartBlankState";
 import { Spinner } from "../Spinner";
@@ -130,6 +131,15 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
     return aggregateValues(values, aggregation);
   }, [rows, column, aggregation, sortDirection]);
 
+  // Look up column format for format-aware display
+  const columnValueFormatter = useMemo(() => {
+    const columnMeta = columns.find((c) => c.name === column);
+    const formatType = (columnMeta?.format ?? columnMeta?.customRenderType) as
+      | ColumnFormatType
+      | undefined;
+    return createValueFormatter(formatType);
+  }, [columns, column]);
+
   if (isLoading) {
     return (
       <div className="grid h-full place-items-center [container-type:size]">
@@ -142,14 +152,29 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
     return <ChartBlankState icon={Hash} message="No data to display" />;
   }
 
+  // Use format-aware formatter when available
+  if (columnValueFormatter) {
+    return (
+      <div className="h-full w-full [container-type:size]">
+        <div className="grid h-full w-full place-items-center">
+          <div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
+            {prefix && <span>{prefix}</span>}
+            <span>{columnValueFormatter(result)}</span>
+            {suffix && <span className="text-[0.4em] text-text-dimmed">{suffix}</span>}
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   const { displayValue, unitSuffix, decimalPlaces } = abbreviate
     ? abbreviateValue(result)
     : { displayValue: result, unitSuffix: undefined, decimalPlaces: getDecimalPlaces(result) };
 
   return (
     <div className="h-full w-full [container-type:size]">
       <div className="grid h-full w-full place-items-center">
-        <div className="flex items-baseline gap-[0.15em] whitespace-nowrap font-normal tabular-nums leading-none text-text-bright text-[clamp(24px,12cqw,96px)]">
+        <div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
           {prefix && <span>{prefix}</span>}
           <AnimatedNumber value={displayValue} decimalPlaces={decimalPlaces} />
           {(unitSuffix || suffix) && (

apps/webapp/app/components/primitives/charts/Chart.tsx

@@ -104,6 +104,8 @@ const ChartTooltipContent = React.forwardRef<
       indicator?: "line" | "dot" | "dashed";
       nameKey?: string;
       labelKey?: string;
+      /** Optional formatter for numeric values (e.g. bytes, duration) */
+      valueFormatter?: (value: number) => string;
     }
 >(
   (
@@ -121,6 +123,7 @@ const ChartTooltipContent = React.forwardRef<
       color,
       nameKey,
       labelKey,
+      valueFormatter,
     },
     ref
   ) => {
@@ -221,9 +224,11 @@ const ChartTooltipContent = React.forwardRef<
                           {itemConfig?.label || item.name}
                         </span>
                       </div>
-                      {item.value && (
+                      {item.value != null && (
                         <span className="text-foreground font-mono font-medium tabular-nums">
-                          {item.value.toLocaleString()}
+                          {valueFormatter && typeof item.value === "number"
+                            ? valueFormatter(item.value)
+                            : item.value.toLocaleString()}
                         </span>
                       )}
                     </div>

apps/webapp/app/components/primitives/charts/ChartBar.tsx

@@ -41,6 +41,8 @@ export type ChartBarRendererProps = {
   referenceLine?: ReferenceLineProps;
   /** Custom tooltip label formatter */
   tooltipLabelFormatter?: (label: string, payload: any[]) => string;
+  /** Optional formatter for numeric tooltip values (e.g. bytes, duration) */
+  tooltipValueFormatter?: (value: number) => string;
   /** Width injected by ResponsiveContainer */
   width?: number;
   /** Height injected by ResponsiveContainer */
@@ -65,6 +67,7 @@ export function ChartBarRenderer({
   yAxisProps: yAxisPropsProp,
   referenceLine,
   tooltipLabelFormatter,
+  tooltipValueFormatter,
   width,
   height,
 }: ChartBarRendererProps) {
@@ -163,7 +166,7 @@ export function ChartBarRenderer({
           showLegend ? (
             () => null
           ) : tooltipLabelFormatter ? (
-            <ChartTooltipContent />
+            <ChartTooltipContent valueFormatter={tooltipValueFormatter} />
           ) : (
             <ZoomTooltip
               isSelecting={zoom?.isSelecting}