From d3f0634463999e7962acde400b2a441958618b55 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 12:23:05 +0200 Subject: [PATCH 01/11] init --- apps/api/src/services/QueryEngineService.ts | 2 +- packages/query-engine/src/ch/ch.test.ts | 93 +++++++++++- packages/query-engine/src/ch/compile.ts | 10 +- packages/query-engine/src/ch/expr.ts | 138 +++++++++++++++++- packages/query-engine/src/ch/index.ts | 18 ++- packages/query-engine/src/ch/param.ts | 2 + .../query-engine/src/ch/queries/alerts.ts | 45 +++--- .../src/ch/queries/attribute-keys.ts | 12 +- .../query-engine/src/ch/queries/errors.ts | 31 +--- packages/query-engine/src/ch/queries/logs.ts | 45 +++--- .../query-engine/src/ch/queries/metrics.ts | 52 ++----- .../query-engine/src/ch/queries/services.ts | 31 ++-- .../query-engine/src/ch/queries/traces.ts | 67 +++++---- packages/query-engine/src/ch/query.ts | 90 +++++++++--- packages/query-engine/src/ch/union.ts | 24 +-- 15 files changed, 444 insertions(+), 216 deletions(-) diff --git a/apps/api/src/services/QueryEngineService.ts b/apps/api/src/services/QueryEngineService.ts index a884ded4ad..3ae0b15dd5 100644 --- a/apps/api/src/services/QueryEngineService.ts +++ b/apps/api/src/services/QueryEngineService.ts @@ -390,7 +390,7 @@ const truncateSql = (s: string, maxLen = 1000) => const executeCHQuery = , Params extends Record>( tinybird: Pick, tenant: TenantContext, - query: CH.CHQuery, + query: CH.CHQuery, params: Params, context: string, ): Effect.Effect, QueryEngineExecutionError> => { diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index 95f41e258b..e3fd7fb343 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -159,6 +159,33 @@ describe("CH.from / select / where / compile", () => { const { sql } = compileCH(q, {}) expect(sql).toContain("Name IN ('a', 'b', 'c')") }) + + it("compiles column shorthand select", () => { + const q = CH.from(TestTable).select("Id", "Name", "Value") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("Id AS Id") + expect(sql).toContain("Name AS Name") + expect(sql).toContain("Value AS Value") + expect(sql).toContain("FROM test_table") + }) + + it("compiles in_() on expressions", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Name.in_("alice", "bob", "charlie")]) + + const { sql } = compileCH(q, {}) + expect(sql).toContain("Name IN ('alice', 'bob', 'charlie')") + }) + + it("compiles arrayOf()", () => { + const q = CH.from(TestTable) + .select(($) => ({ names: CH.arrayOf($.Name) })) + + const { sql } = compileCH(q, {}) + expect(sql).toContain("[Name] AS names") + }) }) // --------------------------------------------------------------------------- @@ -208,7 +235,7 @@ describe("tracesTimeseriesQuery", () => { it("builds error_rate timeseries", () => { const q = tracesTimeseriesQuery({ metric: "error_rate", needsSampling: false }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("countIf(StatusCode = 'Error') * 100.0 / count(), 0) AS errorRate") + expect(sql).toContain("countIf(StatusCode = 'Error') * 100 / count(), 0) AS errorRate") }) it("includes sampling columns when needsSampling is true", () => { @@ -539,7 +566,6 @@ describe("unionAll", () => { })) .where(($) => [$.Id.eq("1")]) .groupBy("name") - .withParams<{}>() const q2 = CH.from(TestTable) .select(($) => ({ @@ -549,7 +575,6 @@ describe("unionAll", () => { })) .where(($) => [$.Id.eq("2")]) .groupBy("name") - .withParams<{}>() const union = unionAll(q1, q2).format("JSON") const { sql } = compileUnion(union, {}) @@ -566,12 +591,10 @@ describe("unionAll", () => { const q1 = CH.from(TestTable) .select(($) => ({ name: $.Name, count: CH.count() })) .groupBy("name") - .withParams<{}>() const q2 = CH.from(TestTable) .select(($) => ({ name: $.Id, count: CH.count() })) .groupBy("name") - .withParams<{}>() const union = unionAll(q1, q2) .orderBy(["count", "desc"]) @@ -586,6 +609,66 @@ describe("unionAll", () => { }) }) +// --------------------------------------------------------------------------- +// Subquery support +// --------------------------------------------------------------------------- + +describe("subquery support", () => { + const TestTable = CH.table("test_table", { + Id: CH.string, + Name: CH.string, + Value: CH.uint64, + }) + + it("compiles fromSubquery", () => { + const innerSql = compileCH( + CH.from(TestTable).select(($) => ({ id: $.Id })).where(($) => [$.Value.gt(100)]), + {}, + { skipFormat: true }, + ).sql + + const outer = CH.from(TestTable) + .select(($) => ({ name: $.Name })) + .where(($) => [CH.inSubquery($.Id, innerSql)]) + .format("JSON") + + const { sql } = compileCH(outer, {}) + expect(sql).toContain("Id IN (SELECT") + expect(sql).toContain("Value > 100") + }) + + it("compiles exists()", () => { + const subSql = "SELECT 1 FROM other WHERE other.Id = test_table.Id" + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(() => [CH.exists(subSql)]) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("EXISTS (SELECT 1 FROM other WHERE other.Id = test_table.Id)") + }) + + it("compiles fromSubquery as FROM source", () => { + const innerSql = compileCH( + CH.from(TestTable).select(($) => ({ id: $.Id, name: $.Name })).limit(10), + {}, + { skipFormat: true }, + ).sql + + const outer = CH.fromSubquery(innerSql, "sub") + .select(() => ({ + id: CH.dynamicColumn("id"), + name: CH.dynamicColumn("name"), + })) + .format("JSON") + + const { sql } = compileCH(outer, {}) + expect(sql).toContain("FROM (SELECT") + expect(sql).toContain(") AS sub") + expect(sql).toContain("LIMIT 10") + }) +}) + // --------------------------------------------------------------------------- // New expression functions // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/compile.ts b/packages/query-engine/src/ch/compile.ts index d7b59ede92..efbc9bd4f1 100644 --- a/packages/query-engine/src/ch/compile.ts +++ b/packages/query-engine/src/ch/compile.ts @@ -47,7 +47,7 @@ export function compileCH< Output extends Record, Params extends Record, >( - query: CHQuery, + query: CHQuery, params: Params, options?: { skipFormat?: boolean }, ): CompiledQuery { @@ -71,9 +71,13 @@ export function compileCH< .map((c) => c.toFragment()) // Resolve param placeholders in the compiled SQL + const fromFragment = state.fromSubquerySql + ? raw(`(${state.fromSubquerySql}) AS ${state.fromSubqueryAlias}`) + : ident(state.tableName) + const sqlQuery: SqlQuery = { select: selectFragments, - from: ident(state.tableName), + from: fromFragment, where: whereFragments, groupBy: state.groupByKeys.map((k) => raw(k)), orderBy: state.orderBySpecs.map(([k, dir]) => raw(`${k} ${dir.toUpperCase()}`)), @@ -105,7 +109,7 @@ export function compileUnion< Output extends Record, Params extends Record, >( - union: CHUnionQuery, + union: CHUnionQuery, params: Params, ): CompiledQuery { const state = union._state diff --git a/packages/query-engine/src/ch/expr.ts b/packages/query-engine/src/ch/expr.ts index 76a902de19..fa7bb7c06f 100644 --- a/packages/query-engine/src/ch/expr.ts +++ b/packages/query-engine/src/ch/expr.ts @@ -36,6 +36,10 @@ export interface Expr { notLike(this: Expr, pattern: string): Condition ilike(this: Expr, pattern: string): Condition + // IN / NOT IN + in_(...values: TSType[]): Condition + notIn(...values: TSType[]): Condition + // Arithmetic — only valid for number expressions div(this: Expr, n: number | Expr): Expr mul(this: Expr, n: number | Expr): Expr @@ -96,6 +100,15 @@ function makeExpr(fragment: SqlFragment): Expr { notLike: (pattern: string) => makeCond(raw(`${compile(fragment)} NOT LIKE ${compile(str(pattern))}`)), ilike: (pattern: string) => makeCond(raw(`${compile(fragment)} ILIKE ${compile(str(pattern))}`)), + in_: (...values) => { + const escaped = values.map((v) => compile(toFragment(v))).join(", ") + return makeCond(raw(`${compile(fragment)} IN (${escaped})`)) + }, + notIn: (...values) => { + const escaped = values.map((v) => compile(toFragment(v))).join(", ") + return makeCond(raw(`${compile(fragment)} NOT IN (${escaped})`)) + }, + div: (n: number | Expr) => makeExpr(raw(`${compile(fragment)} / ${compile(toFragment(n))}`)), mul: (n: number | Expr) => makeExpr(raw(`${compile(fragment)} * ${compile(toFragment(n))}`)), add: (n: number | Expr) => makeExpr(raw(`${compile(fragment)} + ${compile(toFragment(n))}`)), @@ -247,11 +260,14 @@ export function mapContains( } export function arrayStringConcat( - parts: Expr[], + parts: Expr[] | Expr>, sep: string, ): Expr { - const arr = parts.map((p) => compile(p.toFragment())).join(", ") - return makeExpr(raw(`arrayStringConcat([${arr}], ${compile(str(sep))})`)) + if (Array.isArray(parts)) { + const arr = parts.map((p: Expr) => compile(p.toFragment())).join(", ") + return makeExpr(raw(`arrayStringConcat([${arr}], ${compile(str(sep))})`)) + } + return makeExpr(raw(`arrayStringConcat(${compile(parts.toFragment())}, ${compile(str(sep))})`)) } export function arrayFilter( @@ -316,6 +332,34 @@ export function intDiv(a: Expr, b: number | Expr): Expr return makeExpr(raw(`intDiv(${compile(a.toFragment())}, ${bStr})`)) } +// --------------------------------------------------------------------------- +// Subquery expressions +// --------------------------------------------------------------------------- + +/** + * EXISTS (subquery) — for correlated subqueries. + * The subquery must be compiled separately; this wraps its SQL as a condition. + */ +export function exists(subquerySql: string): Condition { + return makeCond(raw(`EXISTS (${subquerySql})`)) +} + +/** + * expr IN (subquery) — for uncorrelated subqueries. + * The subquery must be compiled separately; this wraps its SQL as a condition. + */ +export function inSubquery(expr: Expr, subquerySql: string): Condition { + return makeCond(raw(`${compile(expr.toFragment())} IN (${subquerySql})`)) +} + +/** + * Reference an outer query's column in a correlated subquery. + * Usage: `outerRef("t.TraceId")` or `outerRef("TraceId")` + */ +export function outerRef(name: string): Expr { + return makeExpr(raw(name)) +} + // --------------------------------------------------------------------------- // Raw expression (escape hatch) // --------------------------------------------------------------------------- @@ -360,3 +404,91 @@ export function whenTrue( if (!value) return undefined return fn() } + +// --------------------------------------------------------------------------- +// Array & Map constructors +// --------------------------------------------------------------------------- + +/** Build an array literal: `[expr1, expr2, ...]` */ +export function arrayOf(...exprs: Expr[]): Expr> { + const args = exprs.map((e) => compile(e.toFragment())).join(", ") + return makeExpr>(raw(`[${args}]`)) +} + +/** Build a map literal: `map('k1', v1, 'k2', v2, ...)` */ +export function mapLiteral( + ...pairs: Array<[string, Expr]> +): Expr> { + if (pairs.length === 0) return makeExpr>(raw("map()")) + const args = pairs + .map(([k, v]) => `${compile(str(k))}, ${compile(v.toFragment())}`) + .join(", ") + return makeExpr>(raw(`map(${args})`)) +} + +// --------------------------------------------------------------------------- +// Cast functions +// --------------------------------------------------------------------------- + +export function toUInt64(expr: Expr): Expr { + return makeExpr(raw(`toUInt64(${compile(expr.toFragment())})`)) +} + +export function toInt64(expr: Expr): Expr { + return makeExpr(raw(`toInt64(${compile(expr.toFragment())})`)) +} + +// --------------------------------------------------------------------------- +// Multi-branch conditional +// --------------------------------------------------------------------------- + +/** multiIf(cond1, val1, cond2, val2, ..., else) */ +export function multiIf( + cases: Array<[Condition, Expr]>, + else_: Expr, +): Expr { + const parts = cases + .map(([cond, val]) => `${compile(cond.toFragment())}, ${compile(val.toFragment())}`) + .join(", ") + return makeExpr(raw(`multiIf(${parts}, ${compile(else_.toFragment())})`)) +} + +// --------------------------------------------------------------------------- +// String functions +// --------------------------------------------------------------------------- + +export function position_(haystack: Expr, needle: string): Expr { + return makeExpr(raw(`position(${compile(haystack.toFragment())}, ${compile(str(needle))})`)) +} + +export function left_(s: Expr, len: Expr): Expr { + return makeExpr(raw(`left(${compile(s.toFragment())}, ${compile(len.toFragment())})`)) +} + +export function length_(s: Expr): Expr { + return makeExpr(raw(`length(${compile(s.toFragment())})`)) +} + +export function replaceOne( + haystack: Expr, + pattern: string, + replacement: string, +): Expr { + return makeExpr( + raw(`replaceOne(${compile(haystack.toFragment())}, ${compile(str(pattern))}, ${compile(str(replacement))})`), + ) +} + +// --------------------------------------------------------------------------- +// Numeric functions +// --------------------------------------------------------------------------- + +export function least_(...exprs: Expr[]): Expr { + const args = exprs.map((e) => compile(e.toFragment())).join(", ") + return makeExpr(raw(`least(${args})`)) +} + +export function greatest_(...exprs: Expr[]): Expr { + const args = exprs.map((e) => compile(e.toFragment())).join(", ") + return makeExpr(raw(`greatest(${args})`)) +} diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 75f49d644f..36f02efbdd 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -70,6 +70,20 @@ export { dynamicColumn, when, whenTrue, + arrayOf, + mapLiteral, + toUInt64, + toInt64, + multiIf, + position_ as position, + left_ as left, + length_ as length, + replaceOne, + least_ as least, + greatest_ as greatest, + exists, + inSubquery, + outerRef, } from "./expr" // Params @@ -80,14 +94,16 @@ export { type CHQuery, type ColumnAccessor, type InferOutput, + type InferQueryOutput, from, + fromSubquery, } from "./query" // Compilation export { compileCH as compile, compileUnion, type CompiledQuery, QueryBuilderError } from "./compile" // Union -export { unionAll, type CHUnionQuery } from "./union" +export { unionAll, type CHUnionQuery, type InferUnionOutput } from "./union" // Tables export * as tables from "./tables" diff --git a/packages/query-engine/src/ch/param.ts b/packages/query-engine/src/ch/param.ts index b0a5300d8f..cbe1268fc4 100644 --- a/packages/query-engine/src/ch/param.ts +++ b/packages/query-engine/src/ch/param.ts @@ -38,6 +38,8 @@ function makeParamMarker(name: N, fragment: SqlFragment): P mul: () => { throw new QueryBuilderError({ code: "UnresolvedParam", message: `Param '${name}' not resolved` }) }, add: () => { throw new QueryBuilderError({ code: "UnresolvedParam", message: `Param '${name}' not resolved` }) }, sub: () => { throw new QueryBuilderError({ code: "UnresolvedParam", message: `Param '${name}' not resolved` }) }, + in_: () => { throw new QueryBuilderError({ code: "UnresolvedParam", message: `Param '${name}' not resolved` }) }, + notIn: () => { throw new QueryBuilderError({ code: "UnresolvedParam", message: `Param '${name}' not resolved` }) }, } as ParamMarker } diff --git a/packages/query-engine/src/ch/queries/alerts.ts b/packages/query-engine/src/ch/queries/alerts.ts index 2b96142f39..9f35469c8c 100644 --- a/packages/query-engine/src/ch/queries/alerts.ts +++ b/packages/query-engine/src/ch/queries/alerts.ts @@ -52,7 +52,6 @@ export interface AlertTracesAggregateByServiceOutput extends AlertTracesAggregat readonly serviceName: string } -type AlertTracesParams = { orgId: string; startTime: string; endTime: string } function alertTracesSelectExprs($: ColumnAccessor, apdexThresholdMs: number) { const t = apdexThresholdMs @@ -62,11 +61,18 @@ function alertTracesSelectExprs($: ColumnAccessor, apdexT p50Duration: CH.quantile(0.5)($.Duration).div(1000000), p95Duration: CH.quantile(0.95)($.Duration).div(1000000), p99Duration: CH.quantile(0.99)($.Duration).div(1000000), - errorRate: CH.rawExpr(`if(count() > 0, countIf(StatusCode = 'Error') * 100.0 / count(), 0)`), + errorRate: CH.if_(CH.count().gt(0), CH.countIf($.StatusCode.eq("Error")).mul(100.0).div(CH.count()), CH.lit(0)), satisfiedCount: CH.countIf($.Duration.div(1000000).lt(t)), toleratingCount: CH.countIf($.Duration.div(1000000).gte(t).and($.Duration.div(1000000).lt(t * 4))), - apdexScore: CH.rawExpr( - `if(count() > 0, round((countIf(Duration / 1000000 < ${t}) + countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) * 0.5) / count(), 4), 0)`, + apdexScore: CH.if_( + CH.count().gt(0), + CH.round_( + CH.countIf($.Duration.div(1000000).lt(t)) + .add(CH.countIf($.Duration.div(1000000).gte(t).and($.Duration.div(1000000).lt(t * 4))).mul(0.5)) + .div(CH.count()), + 4, + ), + CH.lit(0), ), } } @@ -82,7 +88,7 @@ function alertTracesWhereConditions( CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), CH.when(opts.spanName, (v: string) => $.SpanName.eq(v)), CH.whenTrue(!!opts.rootOnly, () => - CH.rawCond("(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '')"), + $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), ), CH.whenTrue(!!opts.errorsOnly, () => $.StatusCode.eq("Error")), ] @@ -109,19 +115,18 @@ function alertTracesWhereConditions( export function alertTracesAggregateQuery( opts: AlertTracesOpts, -): CHQuery { +): CHQuery { const threshold = opts.apdexThresholdMs ?? 500 return from(Traces) .select(($) => alertTracesSelectExprs($, threshold)) .where(($) => alertTracesWhereConditions($, opts)) .format("JSON") - .withParams() } export function alertTracesAggregateByServiceQuery( opts: AlertTracesOpts, -): CHQuery { +): CHQuery { const threshold = opts.apdexThresholdMs ?? 500 return from(Traces) @@ -133,7 +138,6 @@ export function alertTracesAggregateByServiceQuery( .groupBy("serviceName") .orderBy(["count", "desc"]) .format("JSON") - .withParams() } // --------------------------------------------------------------------------- @@ -157,7 +161,6 @@ export interface AlertMetricsAggregateByServiceOutput extends AlertMetricsAggreg readonly serviceName: string } -type AlertMetricsParams = { orgId: string; metricName: string; startTime: string; endTime: string } const VALUE_TABLES = { sum: MetricsSum, @@ -171,7 +174,7 @@ const HISTOGRAM_TABLES = { function buildValueMetricsAggregate( opts: AlertMetricsOpts, -): CHQuery { +): CHQuery { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] return from(tbl as typeof MetricsSum) @@ -190,12 +193,11 @@ function buildValueMetricsAggregate( CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), ]) .format("JSON") - .withParams() } function buildHistogramMetricsAggregate( opts: AlertMetricsOpts, -): CHQuery { +): CHQuery { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] return from(tbl as typeof MetricsHistogram) @@ -214,12 +216,11 @@ function buildHistogramMetricsAggregate( CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), ]) .format("JSON") - .withParams() } function buildValueMetricsAggregateByService( opts: AlertMetricsOpts, -): CHQuery { +): CHQuery { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] return from(tbl as typeof MetricsSum) @@ -240,12 +241,11 @@ function buildValueMetricsAggregateByService( .groupBy("serviceName") .orderBy(["dataPointCount", "desc"]) .format("JSON") - .withParams() } function buildHistogramMetricsAggregateByService( opts: AlertMetricsOpts, -): CHQuery { +): CHQuery { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] return from(tbl as typeof MetricsHistogram) @@ -266,19 +266,18 @@ function buildHistogramMetricsAggregateByService( .groupBy("serviceName") .orderBy(["dataPointCount", "desc"]) .format("JSON") - .withParams() } export function alertMetricsAggregateQuery( opts: AlertMetricsOpts, -): CHQuery { +): CHQuery { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" return isHistogram ? buildHistogramMetricsAggregate(opts) : buildValueMetricsAggregate(opts) } export function alertMetricsAggregateByServiceQuery( opts: AlertMetricsOpts, -): CHQuery { +): CHQuery { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" return isHistogram ? buildHistogramMetricsAggregateByService(opts) : buildValueMetricsAggregateByService(opts) } @@ -302,7 +301,7 @@ export interface AlertLogsAggregateByServiceOutput extends AlertLogsAggregateOut export function alertLogsAggregateQuery( opts: AlertLogsOpts, -): CHQuery { +): CHQuery { return from(Logs) .select(() => ({ count: CH.count(), @@ -315,12 +314,11 @@ export function alertLogsAggregateQuery( CH.when(opts.severity, (v: string) => SeverityText.eq(v)), ]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } export function alertLogsAggregateByServiceQuery( opts: AlertLogsOpts, -): CHQuery { +): CHQuery { return from(Logs) .select(({ ServiceName }) => ({ serviceName: ServiceName, @@ -335,5 +333,4 @@ export function alertLogsAggregateByServiceQuery( .groupBy("serviceName") .orderBy(["count", "desc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } diff --git a/packages/query-engine/src/ch/queries/attribute-keys.ts b/packages/query-engine/src/ch/queries/attribute-keys.ts index b3662d6565..0d3c16c39b 100644 --- a/packages/query-engine/src/ch/queries/attribute-keys.ts +++ b/packages/query-engine/src/ch/queries/attribute-keys.ts @@ -17,8 +17,7 @@ export function attributeKeysQuery( opts: AttributeKeysQueryOpts, ): CHQuery< typeof AttributeKeysHourly.columns, - AttributeKeysOutput, - { orgId: string; startTime: string; endTime: string } + AttributeKeysOutput > { return from(AttributeKeysHourly) .select(($) => ({ @@ -35,7 +34,6 @@ export function attributeKeysQuery( .orderBy(["usageCount", "desc"]) .limit(opts.limit ?? 200) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -52,11 +50,9 @@ export interface AttributeValuesOutput { readonly usageCount: number } -type AttributeValuesParams = { orgId: string; startTime: string; endTime: string } - export function spanAttributeValuesQuery( opts: AttributeValuesOpts, -): CHQuery { +): CHQuery { return from(Traces) .select(($) => ({ attributeValue: $.SpanAttributes.get(opts.attributeKey), @@ -72,12 +68,11 @@ export function spanAttributeValuesQuery( .orderBy(["usageCount", "desc"]) .limit(opts.limit ?? 50) .format("JSON") - .withParams() } export function resourceAttributeValuesQuery( opts: AttributeValuesOpts, -): CHQuery { +): CHQuery { return from(Traces) .select(($) => ({ attributeValue: $.ResourceAttributes.get(opts.attributeKey), @@ -93,5 +88,4 @@ export function resourceAttributeValuesQuery( .orderBy(["usageCount", "desc"]) .limit(opts.limit ?? 50) .format("JSON") - .withParams() } diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 211c932732..b0904f378b 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -49,7 +49,7 @@ export interface ErrorsByTypeOutput { export function errorsByTypeQuery( opts: ErrorsByTypeOpts, -): CHQuery { +): CHQuery { return from(ErrorSpans) .select(($) => ({ errorType: CH.rawExpr(ERROR_FINGERPRINT_SQL), @@ -78,7 +78,6 @@ export function errorsByTypeQuery( .orderBy(["count", "desc"]) .limit(opts.limit ?? 50) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -97,7 +96,7 @@ export interface ErrorsTimeseriesOutput { export function errorsTimeseriesQuery( opts: ErrorsTimeseriesOpts, -): CHQuery { +): CHQuery { const esc = escapeClickHouseString return from(ErrorSpans) .select(($) => ({ @@ -116,7 +115,6 @@ export function errorsTimeseriesQuery( .groupBy("bucket") .orderBy(["bucket", "asc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string; bucketSeconds: number }>() } // --------------------------------------------------------------------------- @@ -144,11 +142,9 @@ export interface SpanHierarchyOutput { readonly relationship: string } -type SpanHierarchyParams = { orgId: string } - export function spanHierarchyQuery( opts: SpanHierarchyOpts, -): CHQuery { +): CHQuery { // HTTP span name rewriting: "http.server GET" + route → "GET /api/users" const httpRewriteExpr = CH.if_( CH.rawCond( @@ -191,7 +187,6 @@ export function spanHierarchyQuery( ]) .orderBy(["startTime", "asc"]) .format("JSON") - .withParams() } // --------------------------------------------------------------------------- @@ -221,11 +216,9 @@ export interface TracesDurationStatsOutput { readonly p95DurationMs: number } -type TracesDurationStatsParams = { orgId: string; startTime: string; endTime: string } - export function tracesDurationStatsQuery( opts: TracesDurationStatsOpts, -): CHQuery { +): CHQuery { const mm = opts.matchModes return from(TraceListMv) @@ -261,7 +254,6 @@ export function tracesDurationStatsQuery( ), ]) .format("JSON") - .withParams() } // --------------------------------------------------------------------------- @@ -296,11 +288,9 @@ export interface TracesFacetsOutput { readonly facetType: string } -type TracesFacetsParams = { orgId: string; startTime: string; endTime: string } - export function tracesFacetsQuery( opts: TracesFacetsOpts, -): CHUnionQuery { +): CHUnionQuery { const esc = escapeClickHouseString const baseWhere = ($: ColumnAccessor): Array => { @@ -385,7 +375,6 @@ export function tracesFacetsQuery( .groupBy("name") .orderBy(["count", "desc"]) .limit(limit) - .withParams() return unionAll( makeFacetQuery("ServiceName", "service"), @@ -399,8 +388,7 @@ export function tracesFacetsQuery( count: CH.count(), facetType: CH.lit("errorCount"), })) - .where(($) => [...baseWhere($), $.HasError.eq(1)]) - .withParams(), + .where(($) => [...baseWhere($), $.HasError.eq(1)]), ).format("JSON") } @@ -421,11 +409,9 @@ export interface ErrorsFacetsOutput { readonly facetType: string } -type ErrorsFacetsParams = { orgId: string; startTime: string; endTime: string } - export function errorsFacetsQuery( opts: ErrorsFacetsOpts, -): CHUnionQuery { +): CHUnionQuery { const baseWhere = ($: ColumnAccessor): Array => [ $.OrgId.eq(param.string("orgId")), $.Timestamp.gte(param.dateTime("startTime")), @@ -452,7 +438,6 @@ export function errorsFacetsQuery( .groupBy("name") .orderBy(["count", "desc"]) .limit(100) - .withParams() const envQuery = from(ErrorSpans) .select(($) => ({ @@ -464,7 +449,6 @@ export function errorsFacetsQuery( .groupBy("name") .orderBy(["count", "desc"]) .limit(100) - .withParams() const errorTypeQuery = from(ErrorSpans) .select(() => ({ @@ -476,7 +460,6 @@ export function errorsFacetsQuery( .groupBy("name") .orderBy(["count", "desc"]) .limit(50) - .withParams() return unionAll(serviceQuery, envQuery, errorTypeQuery) .format("JSON") diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index a94dc0ff69..34d741841a 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -37,16 +37,14 @@ export interface LogsTimeseriesOutput { export function logsTimeseriesQuery( opts: LogsTimeseriesOpts, -): CHQuery { +): CHQuery { const groupByService = opts.groupBy?.includes("service") const groupBySeverity = opts.groupBy?.includes("severity") - const groupNameExpr = buildLogsGroupNameExpr(groupByService, groupBySeverity) - return from(Logs) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), - groupName: groupNameExpr, + groupName: buildLogsGroupNameExpr($, groupByService, groupBySeverity), count: CH.count(), })) .where(($) => [ @@ -59,10 +57,10 @@ export function logsTimeseriesQuery( .groupBy("bucket", "groupName") .orderBy(["bucket", "asc"], ["groupName", "asc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string; bucketSeconds: number }>() } function buildLogsGroupNameExpr( + $: { ServiceName: CH.Expr; SeverityText: CH.Expr }, groupByService?: boolean, groupBySeverity?: boolean, ): CH.Expr { @@ -70,16 +68,19 @@ function buildLogsGroupNameExpr( return CH.lit("all") } - const parts: string[] = [] - if (groupByService) parts.push("toString(ServiceName)") - if (groupBySeverity) parts.push("toString(SeverityText)") + const parts: CH.Expr[] = [] + if (groupByService) parts.push(CH.toString_($.ServiceName)) + if (groupBySeverity) parts.push(CH.toString_($.SeverityText)) if (parts.length === 1) { - return CH.rawExpr(`coalesce(nullIf(${parts[0]}, ''), 'all')`) + return CH.coalesce(CH.nullIf(parts[0]!, ""), CH.lit("all")) } - return CH.rawExpr( - `coalesce(nullIf(arrayStringConcat(arrayFilter(x -> x != '', [${parts.join(", ")}]), ' \u00b7 '), ''), 'all')`, + // Multi-part: filter empty strings before joining with separator + const filtered = CH.arrayFilter("x -> x != ''", CH.arrayOf(...parts)) + return CH.coalesce( + CH.nullIf(CH.arrayStringConcat(filtered, " \u00b7 "), ""), + CH.lit("all"), ) } @@ -99,7 +100,7 @@ export interface LogsBreakdownOutput { export function logsBreakdownQuery( opts: LogsBreakdownOpts, -): CHQuery { +): CHQuery { return from(Logs) .select(($) => ({ name: opts.groupBy === "severity" ? $.SeverityText : $.ServiceName, @@ -116,7 +117,6 @@ export function logsBreakdownQuery( .orderBy(["count", "desc"]) .limit(opts.limit ?? 10) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -129,7 +129,7 @@ export interface LogsCountOutput { export function logsCountQuery( opts: LogsQueryOpts, -): CHQuery { +): CHQuery { return from(Logs) .select(() => ({ total: CH.count(), @@ -144,7 +144,6 @@ export function logsCountQuery( CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), ]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -170,11 +169,9 @@ export interface LogsListOutput { readonly resourceAttributes: string } -type LogsListParams = { orgId: string; startTime: string; endTime: string } - export function logsListQuery( opts: LogsListOpts, -): CHQuery { +): CHQuery { return from(Logs) .select(($) => ({ timestamp: $.Timestamp, @@ -202,7 +199,6 @@ export function logsListQuery( .orderBy(["timestamp", "desc"]) .limit(opts.limit ?? 50) .format("JSON") - .withParams() } // --------------------------------------------------------------------------- @@ -217,13 +213,13 @@ export interface ErrorRateByServiceOutput { } export function errorRateByServiceQuery( -): CHQuery { +): CHQuery { return from(Logs) .select(($) => ({ serviceName: $.ServiceName, totalLogs: CH.count(), errorLogs: CH.countIf(CH.inList($.SeverityText, ["ERROR", "FATAL"])), - errorRatePercent: CH.rawExpr("round(countIf(SeverityText IN ('ERROR', 'FATAL')) / count() * 100, 2)"), + errorRatePercent: CH.round_(CH.countIf(CH.inList($.SeverityText, ["ERROR", "FATAL"])).div(CH.count()).mul(100), 2), })) .where(($) => [ $.OrgId.eq(param.string("orgId")), @@ -233,7 +229,6 @@ export function errorRateByServiceQuery( .groupBy("serviceName") .orderBy(["errorRatePercent", "desc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -247,11 +242,9 @@ export interface LogsFacetsOutput { readonly facetType: string } -type LogsFacetsParams = { orgId: string; startTime: string; endTime: string } - export function logsFacetsQuery( opts: LogsQueryOpts, -): CHUnionQuery { +): CHUnionQuery { const baseWhere = ($: ColumnAccessor): Array => [ $.OrgId.eq(param.string("orgId")), $.Timestamp.gte(param.dateTime("startTime")), @@ -269,7 +262,6 @@ export function logsFacetsQuery( })) .where(baseWhere) .groupBy("severityText") - .withParams() const serviceQuery = from(Logs) .select(($) => ({ @@ -280,7 +272,6 @@ export function logsFacetsQuery( })) .where(baseWhere) .groupBy("serviceName") - .withParams() return unionAll(severityQuery, serviceQuery) .orderBy(["count", "desc"]) diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index f813ec180d..3d0ca0ebc3 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -58,13 +58,6 @@ export interface MetricsTimeseriesOutput { readonly dataPointCount: number } -type MetricsTimeseriesParams = { - orgId: string - metricName: string - startTime: string - endTime: string - bucketSeconds: number -} // --------------------------------------------------------------------------- // Timeseries query — handles all 4 metric types @@ -72,7 +65,7 @@ type MetricsTimeseriesParams = { export function metricsTimeseriesQuery( opts: MetricsTimeseriesOpts, -): CHQuery { +): CHQuery { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" if (isHistogram) { @@ -83,7 +76,7 @@ export function metricsTimeseriesQuery( function buildValueTimeseries( opts: MetricsTimeseriesOpts, -): CHQuery { +): CHQuery { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] const q = from(tbl as typeof MetricsSum) @@ -116,12 +109,11 @@ function buildValueTimeseries( ) .orderBy(["bucket", "asc"]) .format("JSON") - .withParams() } function buildHistogramTimeseries( opts: MetricsTimeseriesOpts, -): CHQuery { +): CHQuery { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] const q = from(tbl as typeof MetricsHistogram) @@ -154,7 +146,6 @@ function buildHistogramTimeseries( ) .orderBy(["bucket", "asc"]) .format("JSON") - .withParams() } // --------------------------------------------------------------------------- @@ -177,17 +168,9 @@ export interface MetricsRateTimeseriesOutput { readonly dataPointCount: number } -type MetricsRateTimeseriesParams = { - orgId: string - metricName: string - startTime: string - endTime: string - bucketSeconds: number -} - export function metricsTimeseriesRateSQL( opts: MetricsRateTimeseriesOpts, - params: MetricsRateTimeseriesParams, + params: { orgId: string; metricName: string; startTime: string; endTime: string; bucketSeconds: number }, ): CompiledQuery { const esc = escapeClickHouseString const bucketSeconds = Math.round(params.bucketSeconds) @@ -278,16 +261,9 @@ export interface MetricsBreakdownOutput { readonly count: number } -type MetricsBreakdownParams = { - orgId: string - metricName: string - startTime: string - endTime: string -} - export function metricsBreakdownQuery( opts: MetricsBreakdownOpts, -): CHQuery { +): CHQuery { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" const limit = opts.limit ?? 10 @@ -300,7 +276,7 @@ export function metricsBreakdownQuery( function buildValueBreakdown( opts: MetricsBreakdownOpts, limit: number, -): CHQuery { +): CHQuery { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] return from(tbl as typeof MetricsSum) @@ -320,13 +296,12 @@ function buildValueBreakdown( .orderBy(["count", "desc"]) .limit(limit) .format("JSON") - .withParams() } function buildHistogramBreakdown( opts: MetricsBreakdownOpts, limit: number, -): CHQuery { +): CHQuery { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] return from(tbl as typeof MetricsHistogram) @@ -346,7 +321,6 @@ function buildHistogramBreakdown( .orderBy(["count", "desc"]) .limit(limit) .format("JSON") - .withParams() } // --------------------------------------------------------------------------- @@ -373,11 +347,9 @@ export interface ListMetricsOutput { readonly isMonotonic: boolean | number } -type ListMetricsParams = { orgId: string; startTime: string; endTime: string } - export function listMetricsQuery( opts: ListMetricsOpts, -): CHUnionQuery { +): CHUnionQuery { function buildSubquery( tbl: typeof MetricsSum | typeof MetricsGauge | typeof MetricsHistogram | typeof MetricsExpHistogram, metricType: string, @@ -405,10 +377,9 @@ export function listMetricsQuery( CH.when(opts.search, (v: string) => $.MetricName.ilike(`%${v}%`)), ]) .groupBy("metricName", "serviceName") - .withParams() } - const queries: Array> = [] + const queries: Array> = [] const showSum = !opts.metricType || opts.metricType === "sum" const showGauge = !opts.metricType || opts.metricType === "gauge" const showHist = !opts.metricType || opts.metricType === "histogram" @@ -440,11 +411,9 @@ export interface MetricsSummaryOpts { serviceName?: string } -type MetricsSummaryParams = { orgId: string; startTime: string; endTime: string } - export function metricsSummaryQuery( opts?: MetricsSummaryOpts, -): CHUnionQuery { +): CHUnionQuery { function buildSubquery( tbl: typeof MetricsSum | typeof MetricsGauge | typeof MetricsHistogram | typeof MetricsExpHistogram, metricType: string, @@ -461,7 +430,6 @@ export function metricsSummaryQuery( $.TimeUnix.lte(param.dateTime("endTime")), CH.when(opts?.serviceName, (v: string) => $.ServiceName.eq(v)), ]) - .withParams() } return unionAll( diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index f7a799fa33..d1a2f779bb 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -36,7 +36,7 @@ export interface ServiceOverviewOutput { export function serviceOverviewQuery( opts: ServiceOverviewOpts, -): CHQuery { +): CHQuery { return from(ServiceOverviewSpans) .select(($) => ({ serviceName: $.ServiceName, @@ -67,7 +67,6 @@ export function serviceOverviewQuery( .orderBy(["throughput", "desc"]) .limit(100) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -86,7 +85,7 @@ export interface ServiceReleasesTimelineOutput { export function serviceReleasesTimelineQuery( opts: ServiceReleasesTimelineOpts, -): CHQuery { +): CHQuery { return from(ServiceOverviewSpans) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), @@ -104,7 +103,6 @@ export function serviceReleasesTimelineQuery( .orderBy(["bucket", "asc"]) .limit(1000) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string; bucketSeconds: number }>() } // --------------------------------------------------------------------------- @@ -126,7 +124,7 @@ export interface ServiceApdexTimeseriesOutput { export function serviceApdexTimeseriesQuery( opts: ServiceApdexTimeseriesOpts, -): CHQuery { +): CHQuery { const t = String(opts.apdexThresholdMs ?? 500) return from(Traces) @@ -135,12 +133,19 @@ export function serviceApdexTimeseriesQuery( totalCount: CH.count(), satisfiedCount: CH.countIf($.Duration.div(1000000).lt(Number(t))), toleratingCount: CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))), - apdexScore: CH.rawExpr( - `if(count() > 0, round((countIf(Duration / 1000000 < ${t}) + countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) * 0.5) / count(), 4), 0)`, + apdexScore: CH.if_( + CH.count().gt(0), + CH.round_( + CH.countIf($.Duration.div(1000000).lt(Number(t))) + .add(CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))).mul(0.5)) + .div(CH.count()), + 4, + ), + CH.lit(0), ), })) .where(($) => [ - CH.rawCond("(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '')"), + $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), $.OrgId.eq(param.string("orgId")), $.ServiceName.eq(opts.serviceName), $.Timestamp.gte(param.dateTime("startTime")), @@ -149,7 +154,6 @@ export function serviceApdexTimeseriesQuery( .groupBy("bucket") .orderBy(["bucket", "asc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string; bucketSeconds: number }>() } // --------------------------------------------------------------------------- @@ -179,7 +183,7 @@ export interface ServiceUsageOutput { export function serviceUsageQuery( opts: ServiceUsageOpts, -): CHQuery { +): CHQuery { return from(ServiceUsage) .select(($) => ({ serviceName: $.ServiceName, @@ -211,7 +215,6 @@ export function serviceUsageQuery( .groupBy("serviceName") .orderBy(["totalSizeBytes", "desc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -224,10 +227,8 @@ export interface ServicesFacetsOutput { readonly facetType: string } -type ServicesFacetsParams = { orgId: string; startTime: string; endTime: string } - export function servicesFacetsQuery( -): CHUnionQuery { +): CHUnionQuery { const baseWhere = ($: ColumnAccessor): Array => [ $.OrgId.eq(param.string("orgId")), $.Timestamp.gte(param.dateTime("startTime")), @@ -244,7 +245,6 @@ export function servicesFacetsQuery( .groupBy("name") .orderBy(["count", "desc"]) .limit(50) - .withParams() const commitQuery = from(ServiceOverviewSpans) .select(($) => ({ @@ -256,7 +256,6 @@ export function servicesFacetsQuery( .groupBy("name") .orderBy(["count", "desc"]) .limit(50) - .withParams() return unionAll(envQuery, commitQuery) .format("JSON") diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index f36bd2c82b..2e1ef47860 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -49,26 +49,35 @@ function metricSelectExprs( ? CH.quantile(0.99)($.Duration).div(1000000) : CH.lit(0), errorRate: needs.has("error_rate") - ? CH.rawExpr(`if(count() > 0, countIf(StatusCode = 'Error') * 100.0 / count(), 0)`) + ? CH.if_(CH.count().gt(0), CH.countIf($.StatusCode.eq("Error")).mul(100.0).div(CH.count()), CH.lit(0)) : CH.lit(0), satisfiedCount: needs.has("apdex") - ? CH.rawExpr(`countIf(Duration / 1000000 < ${t})`) + ? CH.countIf($.Duration.div(1000000).lt(Number(t))) : CH.lit(0), toleratingCount: needs.has("apdex") - ? CH.rawExpr(`countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4)`) + ? CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))) : CH.lit(0), apdexScore: needs.has("apdex") - ? CH.rawExpr(`if(count() > 0, round((countIf(Duration / 1000000 < ${t}) + countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) * 0.5) / count(), 4), 0)`) + ? CH.if_( + CH.count().gt(0), + CH.round_( + CH.countIf($.Duration.div(1000000).lt(Number(t))) + .add(CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))).mul(0.5)) + .div(CH.count()), + 4, + ), + CH.lit(0), + ) : CH.lit(0), sampledSpanCount: needsSampling - ? CH.rawExpr("countIf(TraceState LIKE '%th:%')") + ? CH.countIf($.TraceState.like("%th:%")) : CH.lit(0), unsampledSpanCount: needsSampling - ? CH.rawExpr("countIf(TraceState = '' OR TraceState NOT LIKE '%th:%')") + ? CH.countIf($.TraceState.eq("").or($.TraceState.notLike("%th:%"))) : CH.lit(0), dominantThreshold: needsSampling - ? CH.rawExpr("anyIf(extract(TraceState, 'th:([0-9a-f]+)'), TraceState LIKE '%th:%')") - : CH.rawExpr("''"), + ? CH.anyIf(CH.extract_($.TraceState, "th:([0-9a-f]+)"), $.TraceState.like("%th:%")) + : CH.lit(""), } } @@ -189,23 +198,23 @@ function buildWhereConditions( : $.SpanName.eq(v), ), CH.whenTrue(!!opts.rootOnly && !useTraceListMv, () => - CH.rawCond("(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '')"), + $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), ), ] // Duration filters (Duration column is nanoseconds in both MV and raw table) if (opts.minDurationMs != null) { - conditions.push(CH.rawCond(`Duration >= ${opts.minDurationMs} * 1000000`)) + conditions.push($.Duration.gte(opts.minDurationMs * 1000000)) } if (opts.maxDurationMs != null) { - conditions.push(CH.rawCond(`Duration <= ${opts.maxDurationMs} * 1000000`)) + conditions.push($.Duration.lte(opts.maxDurationMs * 1000000)) } if (opts.errorsOnly) { if (useTraceListMv) { - conditions.push(CH.rawCond("HasError = 1")) + conditions.push(CH.dynamicColumn("HasError").eq(1)) } else { - conditions.push(CH.rawCond("StatusCode = 'Error'")) + conditions.push($.StatusCode.eq("Error")) } } @@ -296,7 +305,7 @@ export interface TracesTimeseriesOutput { export function tracesTimeseriesQuery( opts: TracesTimeseriesOpts, -): CHQuery { +): CHQuery { const apdexThresholdMs = opts.apdexThresholdMs ?? 500 const useTraceListMv = canUseTraceListMv(opts) const tbl = useTraceListMv ? TraceListMv : Traces @@ -311,7 +320,6 @@ export function tracesTimeseriesQuery( .groupBy("bucket", "groupName") .orderBy(["bucket", "asc"], ["groupName", "asc"]) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string; bucketSeconds: number }>() } // --------------------------------------------------------------------------- @@ -343,7 +351,7 @@ export interface TracesBreakdownOutput { export function tracesBreakdownQuery( opts: TracesBreakdownOpts, -): CHQuery { +): CHQuery { const apdexThresholdMs = opts.apdexThresholdMs ?? 500 const limit = opts.limit ?? 10 const useTraceListMv = canUseTraceListMv({ @@ -367,7 +375,6 @@ export function tracesBreakdownQuery( .orderBy(["count", "desc"]) .limit(limit) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() } // --------------------------------------------------------------------------- @@ -417,7 +424,7 @@ function buildProjectedMapExpr( export function tracesListQuery( opts: TracesListOpts, -): CHQuery { +): CHQuery { const limit = opts.limit ?? 25 const offset = opts.offset ?? 0 const useTraceListMv = canUseTraceListMv({ ...opts, rootOnly: opts.rootOnly }) @@ -454,15 +461,15 @@ export function tracesListQuery( .select(($) => ({ traceId: $.TraceId, timestamp: $.Timestamp, - spanId: useTraceListMv ? CH.rawExpr("''") : $.SpanId, + spanId: useTraceListMv ? CH.lit("") : $.SpanId, serviceName: $.ServiceName, spanName: $.SpanName, - durationMs: CH.rawExpr("Duration / 1000000"), + durationMs: $.Duration.div(1000000), statusCode: $.StatusCode, spanKind: useTraceListMv ? CH.rawExpr("SpanKind") : $.SpanKind, hasError: useTraceListMv - ? CH.rawExpr("HasError") - : CH.rawExpr("if(StatusCode = 'Error', 1, 0)"), + ? CH.dynamicColumn("HasError") + : CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), spanAttributes: spanAttrExpr ?? $.SpanAttributes, resourceAttributes: resourceAttrExpr ?? $.ResourceAttributes, })) @@ -470,7 +477,6 @@ export function tracesListQuery( .orderBy(["timestamp", "desc"]) .limit(limit) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() if (offset > 0) { q = q.offset(offset) @@ -494,7 +500,7 @@ export interface TracesRootListOutput { readonly endTime: string readonly durationMicros: number readonly spanCount: number - readonly services: string[] + readonly services: readonly string[] readonly rootSpanName: string readonly rootSpanKind: string readonly rootSpanStatusCode: string @@ -506,7 +512,7 @@ export interface TracesRootListOutput { export function tracesRootListQuery( opts: TracesRootListOpts, -): CHQuery { +): CHQuery { const limit = opts.limit ?? 25 const offset = opts.offset ?? 0 const useTraceListMv = canUseTraceListMv({ ...opts, rootOnly: true }) @@ -517,9 +523,9 @@ export function tracesRootListQuery( traceId: $.TraceId, startTime: $.Timestamp, endTime: $.Timestamp, - durationMicros: CH.rawExpr("intDiv(Duration, 1000)"), - spanCount: CH.rawExpr("toUInt64(1)"), - services: CH.rawExpr("[ServiceName]"), + durationMicros: CH.intDiv($.Duration, 1000), + spanCount: CH.toUInt64(CH.lit(1)), + services: CH.arrayOf($.ServiceName), rootSpanName: $.SpanName, rootSpanKind: $.SpanKind, rootSpanStatusCode: $.StatusCode, @@ -533,14 +539,13 @@ export function tracesRootListQuery( ? CH.rawExpr("HttpStatusCode") : CH.rawExpr("SpanAttributes['http.status_code']"), hasError: useTraceListMv - ? CH.rawExpr("HasError") - : CH.rawExpr("if(StatusCode = 'Error', 1, 0)"), + ? CH.dynamicColumn("HasError") + : CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), })) .where(($) => buildWhereConditions($, { ...opts, rootOnly: true }, useTraceListMv)) .orderBy(["startTime", "desc"]) .limit(limit) .format("JSON") - .withParams<{ orgId: string; startTime: string; endTime: string }>() if (offset > 0) { q = q.offset(offset) diff --git a/packages/query-engine/src/ch/query.ts b/packages/query-engine/src/ch/query.ts index 6d5823dd50..862830a17d 100644 --- a/packages/query-engine/src/ch/query.ts +++ b/packages/query-engine/src/ch/query.ts @@ -18,7 +18,7 @@ // .format("JSON") // --------------------------------------------------------------------------- -import type { ColumnDefs, CHType } from "./types" +import type { ColumnDefs, CHType, InferTS } from "./types" import type { Table } from "./table" import type { Expr, Condition, ColumnRef } from "./expr" import { makeColumnRef } from "./expr" @@ -53,6 +53,9 @@ export interface CHQueryState { readonly limitValue?: number readonly offsetValue?: number readonly formatValue?: string + /** When set, the FROM clause uses a subquery instead of a table name. */ + readonly fromSubquerySql?: string + readonly fromSubqueryAlias?: string } // --------------------------------------------------------------------------- @@ -62,36 +65,50 @@ export interface CHQueryState { export interface CHQuery< Cols extends ColumnDefs = ColumnDefs, Output extends Record = {}, - Params extends Record = {}, > { /** @internal — runtime query state */ readonly _state: CHQueryState /** phantom */ - readonly _phantom?: { cols: Cols; output: Output; params: Params } + readonly _phantom?: { cols: Cols; output: Output } + /** Select specific columns by name. Output keys match column names. */ + select( + ...columns: K[] + ): CHQuery }> + + /** Select computed expressions via callback. */ select( fn: ($: ColumnAccessor) => S, - ): CHQuery, Params> + ): CHQuery> where( fn: ($: ColumnAccessor) => Array, - ): CHQuery + ): CHQuery - groupBy(...keys: Array): CHQuery + groupBy(...keys: Array): CHQuery - orderBy(...specs: Array>): CHQuery + orderBy(...specs: Array>): CHQuery - limit(n: number): CHQuery + limit(n: number): CHQuery - offset(n: number): CHQuery + offset(n: number): CHQuery - format(fmt: "JSON" | "JSONEachRow"): CHQuery + format(fmt: "JSON" | "JSONEachRow"): CHQuery - /** Declare the required compile-time params type. No-op at runtime — - * Params is phantom and only constrains what `compile()` accepts. */ - withParams

>(): CHQuery + /** + * @deprecated Params are now inferred at the `compile()` call site. + * This method is a no-op and can be safely removed. + */ + withParams<_P extends Record>(): CHQuery } +// --------------------------------------------------------------------------- +// Type utilities for extracting output types from queries +// --------------------------------------------------------------------------- + +/** Extract the Output type from a CHQuery. */ +export type InferQueryOutput = Q extends CHQuery ? O : never + // --------------------------------------------------------------------------- // ColumnAccessor factory (Proxy-based) // --------------------------------------------------------------------------- @@ -121,13 +138,25 @@ export function createColumnAccessor( function makeQuery< Cols extends ColumnDefs, Output extends Record, - Params extends Record, ->(state: CHQueryState): CHQuery { +>(state: CHQueryState): CHQuery { return { _state: state, - select(fn) { - return makeQuery({ ...state, selectFn: fn }) + select(...args: any[]): any { + // String overload: select("Col1", "Col2") → select($ => ({ Col1: $.Col1, Col2: $.Col2 })) + if (typeof args[0] === "string") { + const columns = args as string[] + return makeQuery({ + ...state, + selectFn: ($: any) => { + const result: Record = {} + for (const col of columns) result[col] = $[col] + return result + }, + }) + } + // Callback overload: select($ => ({ ... })) + return makeQuery({ ...state, selectFn: args[0] }) }, where(fn) { @@ -167,7 +196,7 @@ function makeQuery< export function from( table: Table, -): CHQuery { +): CHQuery { return makeQuery({ tableName: table.name, columns: table.columns, @@ -175,3 +204,28 @@ export function from( orderBySpecs: [], }) } + +/** + * Start a query from a subquery instead of a table. + * + * Usage: + * const inner = CH.compile( + * CH.from(ErrorSpans).select(...).where(...).limit(10), + * params, + * { skipFormat: true }, + * ) + * const outer = CH.fromSubquery(inner.sql, "e").select($ => ({ ... })) + */ +export function fromSubquery( + sql: string, + alias: string, +): CHQuery { + return makeQuery({ + tableName: alias, + columns: {}, + groupByKeys: [], + orderBySpecs: [], + fromSubquerySql: sql, + fromSubqueryAlias: alias, + }) +} diff --git a/packages/query-engine/src/ch/union.ts b/packages/query-engine/src/ch/union.ts index 6a939f1e50..0e175bb9c1 100644 --- a/packages/query-engine/src/ch/union.ts +++ b/packages/query-engine/src/ch/union.ts @@ -14,7 +14,7 @@ import type { CHQuery } from "./query" // --------------------------------------------------------------------------- export interface CHUnionState { - readonly queries: ReadonlyArray> + readonly queries: ReadonlyArray> readonly outerOrderBySpecs: Array<[string, "asc" | "desc"]> readonly outerLimitValue?: number readonly outerOffsetValue?: number @@ -27,33 +27,34 @@ export interface CHUnionState { export interface CHUnionQuery< Output extends Record = {}, - Params extends Record = {}, > { readonly _tag: "CHUnionQuery" /** @internal — runtime union state */ readonly _state: CHUnionState /** phantom */ - readonly _phantom?: { output: Output; params: Params } + readonly _phantom?: { output: Output } orderBy( ...specs: Array<[keyof Output & string, "asc" | "desc"]> - ): CHUnionQuery + ): CHUnionQuery - limit(n: number): CHUnionQuery + limit(n: number): CHUnionQuery - offset(n: number): CHUnionQuery + offset(n: number): CHUnionQuery - format(fmt: "JSON" | "JSONEachRow"): CHUnionQuery + format(fmt: "JSON" | "JSONEachRow"): CHUnionQuery } +/** Extract the Output type from a CHUnionQuery. */ +export type InferUnionOutput = Q extends CHUnionQuery ? O : never + // --------------------------------------------------------------------------- // Implementation // --------------------------------------------------------------------------- function makeUnionQuery< Output extends Record, - Params extends Record, ->(state: CHUnionState): CHUnionQuery { +>(state: CHUnionState): CHUnionQuery { return { _tag: "CHUnionQuery" as const, _state: state, @@ -85,10 +86,9 @@ function makeUnionQuery< export function unionAll< Output extends Record, - Params extends Record, >( - ...queries: Array> -): CHUnionQuery { + ...queries: Array> +): CHUnionQuery { return makeUnionQuery({ queries, outerOrderBySpecs: [], From 01144ab2f605e2277abbe92b5095cbb5bc74194f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 12:36:32 +0200 Subject: [PATCH 02/11] fix --- .../query-engine/src/ch/queries/alerts.ts | 22 +++++++++---------- .../src/ch/queries/attribute-keys.ts | 11 ++++------ .../query-engine/src/ch/queries/errors.ts | 10 ++++----- packages/query-engine/src/ch/queries/logs.ts | 12 +++++----- .../query-engine/src/ch/queries/metrics.ts | 12 +++++----- .../query-engine/src/ch/queries/services.ts | 10 ++++----- .../query-engine/src/ch/queries/traces.ts | 10 ++++----- 7 files changed, 42 insertions(+), 45 deletions(-) diff --git a/packages/query-engine/src/ch/queries/alerts.ts b/packages/query-engine/src/ch/queries/alerts.ts index 9f35469c8c..6f274e981c 100644 --- a/packages/query-engine/src/ch/queries/alerts.ts +++ b/packages/query-engine/src/ch/queries/alerts.ts @@ -9,7 +9,7 @@ import type { AttributeFilter, MetricType } from "../../query-engine" import * as CH from "../expr" import { param } from "../param" -import { from, type CHQuery, type ColumnAccessor } from "../query" +import { from, type ColumnAccessor } from "../query" import { Traces, Logs, @@ -115,7 +115,7 @@ function alertTracesWhereConditions( export function alertTracesAggregateQuery( opts: AlertTracesOpts, -): CHQuery { +) { const threshold = opts.apdexThresholdMs ?? 500 return from(Traces) @@ -126,7 +126,7 @@ export function alertTracesAggregateQuery( export function alertTracesAggregateByServiceQuery( opts: AlertTracesOpts, -): CHQuery { +) { const threshold = opts.apdexThresholdMs ?? 500 return from(Traces) @@ -174,7 +174,7 @@ const HISTOGRAM_TABLES = { function buildValueMetricsAggregate( opts: AlertMetricsOpts, -): CHQuery { +) { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] return from(tbl as typeof MetricsSum) @@ -197,7 +197,7 @@ function buildValueMetricsAggregate( function buildHistogramMetricsAggregate( opts: AlertMetricsOpts, -): CHQuery { +) { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] return from(tbl as typeof MetricsHistogram) @@ -220,7 +220,7 @@ function buildHistogramMetricsAggregate( function buildValueMetricsAggregateByService( opts: AlertMetricsOpts, -): CHQuery { +) { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] return from(tbl as typeof MetricsSum) @@ -245,7 +245,7 @@ function buildValueMetricsAggregateByService( function buildHistogramMetricsAggregateByService( opts: AlertMetricsOpts, -): CHQuery { +) { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] return from(tbl as typeof MetricsHistogram) @@ -270,14 +270,14 @@ function buildHistogramMetricsAggregateByService( export function alertMetricsAggregateQuery( opts: AlertMetricsOpts, -): CHQuery { +) { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" return isHistogram ? buildHistogramMetricsAggregate(opts) : buildValueMetricsAggregate(opts) } export function alertMetricsAggregateByServiceQuery( opts: AlertMetricsOpts, -): CHQuery { +) { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" return isHistogram ? buildHistogramMetricsAggregateByService(opts) : buildValueMetricsAggregateByService(opts) } @@ -301,7 +301,7 @@ export interface AlertLogsAggregateByServiceOutput extends AlertLogsAggregateOut export function alertLogsAggregateQuery( opts: AlertLogsOpts, -): CHQuery { +) { return from(Logs) .select(() => ({ count: CH.count(), @@ -318,7 +318,7 @@ export function alertLogsAggregateQuery( export function alertLogsAggregateByServiceQuery( opts: AlertLogsOpts, -): CHQuery { +) { return from(Logs) .select(({ ServiceName }) => ({ serviceName: ServiceName, diff --git a/packages/query-engine/src/ch/queries/attribute-keys.ts b/packages/query-engine/src/ch/queries/attribute-keys.ts index 0d3c16c39b..ad747e67b2 100644 --- a/packages/query-engine/src/ch/queries/attribute-keys.ts +++ b/packages/query-engine/src/ch/queries/attribute-keys.ts @@ -1,6 +1,6 @@ import * as CH from "../expr" import { param } from "../param" -import { from, type CHQuery } from "../query" +import { from } from "../query" import { AttributeKeysHourly, Traces } from "../tables" export interface AttributeKeysQueryOpts { @@ -15,10 +15,7 @@ export interface AttributeKeysOutput { export function attributeKeysQuery( opts: AttributeKeysQueryOpts, -): CHQuery< - typeof AttributeKeysHourly.columns, - AttributeKeysOutput -> { +) { return from(AttributeKeysHourly) .select(($) => ({ attributeKey: $.AttributeKey, @@ -52,7 +49,7 @@ export interface AttributeValuesOutput { export function spanAttributeValuesQuery( opts: AttributeValuesOpts, -): CHQuery { +) { return from(Traces) .select(($) => ({ attributeValue: $.SpanAttributes.get(opts.attributeKey), @@ -72,7 +69,7 @@ export function spanAttributeValuesQuery( export function resourceAttributeValuesQuery( opts: AttributeValuesOpts, -): CHQuery { +) { return from(Traces) .select(($) => ({ attributeValue: $.ResourceAttributes.get(opts.attributeKey), diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index b0904f378b..72420140b2 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -6,7 +6,7 @@ import * as CH from "../expr" import { param } from "../param" -import { from, type CHQuery, type ColumnAccessor } from "../query" +import { from, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" import { ErrorSpans, TraceListMv, Traces } from "../tables" import { escapeClickHouseString } from "../../sql/sql-fragment" @@ -49,7 +49,7 @@ export interface ErrorsByTypeOutput { export function errorsByTypeQuery( opts: ErrorsByTypeOpts, -): CHQuery { +) { return from(ErrorSpans) .select(($) => ({ errorType: CH.rawExpr(ERROR_FINGERPRINT_SQL), @@ -96,7 +96,7 @@ export interface ErrorsTimeseriesOutput { export function errorsTimeseriesQuery( opts: ErrorsTimeseriesOpts, -): CHQuery { +) { const esc = escapeClickHouseString return from(ErrorSpans) .select(($) => ({ @@ -144,7 +144,7 @@ export interface SpanHierarchyOutput { export function spanHierarchyQuery( opts: SpanHierarchyOpts, -): CHQuery { +) { // HTTP span name rewriting: "http.server GET" + route → "GET /api/users" const httpRewriteExpr = CH.if_( CH.rawCond( @@ -218,7 +218,7 @@ export interface TracesDurationStatsOutput { export function tracesDurationStatsQuery( opts: TracesDurationStatsOpts, -): CHQuery { +) { const mm = opts.matchModes return from(TraceListMv) diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 34d741841a..c90f576b8d 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -6,7 +6,7 @@ import * as CH from "../expr" import { param } from "../param" -import { from, type CHQuery, type ColumnAccessor } from "../query" +import { from, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" import { Logs } from "../tables" @@ -37,7 +37,7 @@ export interface LogsTimeseriesOutput { export function logsTimeseriesQuery( opts: LogsTimeseriesOpts, -): CHQuery { +) { const groupByService = opts.groupBy?.includes("service") const groupBySeverity = opts.groupBy?.includes("severity") @@ -100,7 +100,7 @@ export interface LogsBreakdownOutput { export function logsBreakdownQuery( opts: LogsBreakdownOpts, -): CHQuery { +) { return from(Logs) .select(($) => ({ name: opts.groupBy === "severity" ? $.SeverityText : $.ServiceName, @@ -129,7 +129,7 @@ export interface LogsCountOutput { export function logsCountQuery( opts: LogsQueryOpts, -): CHQuery { +) { return from(Logs) .select(() => ({ total: CH.count(), @@ -171,7 +171,7 @@ export interface LogsListOutput { export function logsListQuery( opts: LogsListOpts, -): CHQuery { +) { return from(Logs) .select(($) => ({ timestamp: $.Timestamp, @@ -213,7 +213,7 @@ export interface ErrorRateByServiceOutput { } export function errorRateByServiceQuery( -): CHQuery { +) { return from(Logs) .select(($) => ({ serviceName: $.ServiceName, diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index 3d0ca0ebc3..df988d90eb 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -65,7 +65,7 @@ export interface MetricsTimeseriesOutput { export function metricsTimeseriesQuery( opts: MetricsTimeseriesOpts, -): CHQuery { +) { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" if (isHistogram) { @@ -76,7 +76,7 @@ export function metricsTimeseriesQuery( function buildValueTimeseries( opts: MetricsTimeseriesOpts, -): CHQuery { +) { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] const q = from(tbl as typeof MetricsSum) @@ -113,7 +113,7 @@ function buildValueTimeseries( function buildHistogramTimeseries( opts: MetricsTimeseriesOpts, -): CHQuery { +) { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] const q = from(tbl as typeof MetricsHistogram) @@ -263,7 +263,7 @@ export interface MetricsBreakdownOutput { export function metricsBreakdownQuery( opts: MetricsBreakdownOpts, -): CHQuery { +) { const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" const limit = opts.limit ?? 10 @@ -276,7 +276,7 @@ export function metricsBreakdownQuery( function buildValueBreakdown( opts: MetricsBreakdownOpts, limit: number, -): CHQuery { +) { const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] return from(tbl as typeof MetricsSum) @@ -301,7 +301,7 @@ function buildValueBreakdown( function buildHistogramBreakdown( opts: MetricsBreakdownOpts, limit: number, -): CHQuery { +) { const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] return from(tbl as typeof MetricsHistogram) diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index d1a2f779bb..5902165039 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -6,7 +6,7 @@ import * as CH from "../expr" import { param } from "../param" -import { from, type CHQuery, type ColumnAccessor } from "../query" +import { from, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" import { ServiceOverviewSpans, ServiceUsage, Traces } from "../tables" @@ -36,7 +36,7 @@ export interface ServiceOverviewOutput { export function serviceOverviewQuery( opts: ServiceOverviewOpts, -): CHQuery { +) { return from(ServiceOverviewSpans) .select(($) => ({ serviceName: $.ServiceName, @@ -85,7 +85,7 @@ export interface ServiceReleasesTimelineOutput { export function serviceReleasesTimelineQuery( opts: ServiceReleasesTimelineOpts, -): CHQuery { +) { return from(ServiceOverviewSpans) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), @@ -124,7 +124,7 @@ export interface ServiceApdexTimeseriesOutput { export function serviceApdexTimeseriesQuery( opts: ServiceApdexTimeseriesOpts, -): CHQuery { +) { const t = String(opts.apdexThresholdMs ?? 500) return from(Traces) @@ -183,7 +183,7 @@ export interface ServiceUsageOutput { export function serviceUsageQuery( opts: ServiceUsageOpts, -): CHQuery { +) { return from(ServiceUsage) .select(($) => ({ serviceName: $.ServiceName, diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index 2e1ef47860..c1829948c5 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -7,7 +7,7 @@ import type { TracesMetric, AttributeFilter } from "../../query-engine" import * as CH from "../expr" import { param } from "../param" -import { from, type CHQuery, type ColumnAccessor } from "../query" +import { from, type ColumnAccessor } from "../query" import { Traces, TraceListMv } from "../tables" import { compile, str } from "../../sql/sql-fragment" import { @@ -305,7 +305,7 @@ export interface TracesTimeseriesOutput { export function tracesTimeseriesQuery( opts: TracesTimeseriesOpts, -): CHQuery { +) { const apdexThresholdMs = opts.apdexThresholdMs ?? 500 const useTraceListMv = canUseTraceListMv(opts) const tbl = useTraceListMv ? TraceListMv : Traces @@ -351,7 +351,7 @@ export interface TracesBreakdownOutput { export function tracesBreakdownQuery( opts: TracesBreakdownOpts, -): CHQuery { +) { const apdexThresholdMs = opts.apdexThresholdMs ?? 500 const limit = opts.limit ?? 10 const useTraceListMv = canUseTraceListMv({ @@ -424,7 +424,7 @@ function buildProjectedMapExpr( export function tracesListQuery( opts: TracesListOpts, -): CHQuery { +) { const limit = opts.limit ?? 25 const offset = opts.offset ?? 0 const useTraceListMv = canUseTraceListMv({ ...opts, rootOnly: opts.rootOnly }) @@ -512,7 +512,7 @@ export interface TracesRootListOutput { export function tracesRootListQuery( opts: TracesRootListOpts, -): CHQuery { +) { const limit = opts.limit ?? 25 const offset = opts.offset ?? 0 const useTraceListMv = canUseTraceListMv({ ...opts, rootOnly: true }) From 62a536db1aee59b022036bc48cf2dae77f282124 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 12:52:52 +0200 Subject: [PATCH 03/11] more stuff --- packages/query-engine/src/ch/expr.ts | 8 ++ packages/query-engine/src/ch/index.ts | 1 + .../query-engine/src/ch/queries/alerts.ts | 6 +- .../query-engine/src/ch/queries/errors.ts | 110 +++++++++++------- .../query-engine/src/ch/queries/metrics.ts | 4 +- .../query-engine/src/ch/queries/traces.ts | 96 +++++++-------- packages/query-engine/src/traces-shared.ts | 66 ++++++----- 7 files changed, 166 insertions(+), 125 deletions(-) diff --git a/packages/query-engine/src/ch/expr.ts b/packages/query-engine/src/ch/expr.ts index fa7bb7c06f..aa57205857 100644 --- a/packages/query-engine/src/ch/expr.ts +++ b/packages/query-engine/src/ch/expr.ts @@ -259,6 +259,14 @@ export function mapContains( return makeCond(raw(`mapContains(${compile(mapExpr.toFragment())}, ${compile(str(key))})`)) } +/** Access a key in a Map expression: `mapExpr['key']` */ +export function mapGet( + mapExpr: Expr>, + key: string, +): Expr { + return makeExpr(raw(`${compile(mapExpr.toFragment())}[${compile(str(key))}]`)) +} + export function arrayStringConcat( parts: Expr[] | Expr>, sep: string, diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 36f02efbdd..df88eb5f0f 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -57,6 +57,7 @@ export { inList, positionCaseInsensitive, mapContains, + mapGet, arrayStringConcat, arrayFilter, extract_ as extract, diff --git a/packages/query-engine/src/ch/queries/alerts.ts b/packages/query-engine/src/ch/queries/alerts.ts index 6f274e981c..e56fac3f1d 100644 --- a/packages/query-engine/src/ch/queries/alerts.ts +++ b/packages/query-engine/src/ch/queries/alerts.ts @@ -18,7 +18,7 @@ import { MetricsHistogram, MetricsExpHistogram, } from "../tables" -import { buildAttrFilterSQL, TRACE_LIST_MV_ATTR_MAP, TRACE_LIST_MV_RESOURCE_MAP } from "../../traces-shared" +import { buildAttrFilterCondition, TRACE_LIST_MV_ATTR_MAP, TRACE_LIST_MV_RESOURCE_MAP } from "../../traces-shared" // --------------------------------------------------------------------------- // Traces alert aggregate @@ -101,12 +101,12 @@ function alertTracesWhereConditions( } if (opts.attributeFilters) { for (const af of opts.attributeFilters) { - conditions.push(CH.rawCond(buildAttrFilterSQL(af, false, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP))) + conditions.push(buildAttrFilterCondition(af, false, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP)) } } if (opts.resourceAttributeFilters) { for (const rf of opts.resourceAttributeFilters) { - conditions.push(CH.rawCond(buildAttrFilterSQL(rf, false, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP))) + conditions.push(buildAttrFilterCondition(rf, false, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP)) } } diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 72420140b2..f531d85573 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -13,9 +13,33 @@ import { escapeClickHouseString } from "../../sql/sql-fragment" import type { CompiledQuery } from "../compile" // --------------------------------------------------------------------------- -// Shared: Error fingerprint SQL expression +// Shared: Error fingerprint expression (typed DSL) // --------------------------------------------------------------------------- +/** Extracts a short error "type" from StatusMessage for grouping. */ +export function errorFingerprint(statusMessage: CH.Expr): CH.Expr { + return CH.if_( + statusMessage.eq(""), + CH.lit("Unknown Error"), + CH.left_( + statusMessage, + CH.multiIf( + [ + [CH.position_(statusMessage, ": ").gt(3), CH.toInt64(CH.position_(statusMessage, ": ")).sub(1)], + [CH.position_(statusMessage, " (").gt(3), CH.toInt64(CH.position_(statusMessage, " (")).sub(1)], + [CH.position_(statusMessage, "\\n").gt(3), CH.toInt64(CH.position_(statusMessage, "\\n")).sub(1)], + [CH.position_(statusMessage, "{").gt(10), CH.toInt64(CH.position_(statusMessage, "{")).sub(1)], + ], + CH.least_(CH.toInt64(CH.length_(statusMessage)), CH.lit(150)), + ), + ), + ) +} + +/** + * @deprecated Use errorFingerprint($.StatusMessage) instead. + * Kept for raw SQL builders (errorsSummarySQL, errorDetailTracesSQL) that can't use the DSL. + */ export const ERROR_FINGERPRINT_SQL = `if(StatusMessage = '', 'Unknown Error', left(StatusMessage, multiIf( position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, @@ -52,7 +76,7 @@ export function errorsByTypeQuery( ) { return from(ErrorSpans) .select(($) => ({ - errorType: CH.rawExpr(ERROR_FINGERPRINT_SQL), + errorType: errorFingerprint($.StatusMessage), sampleMessage: CH.any_($.StatusMessage), count: CH.count(), affectedServicesCount: CH.uniq($.ServiceName), @@ -71,7 +95,7 @@ export function errorsByTypeQuery( ? CH.inList($.DeploymentEnv, opts.deploymentEnvs) : undefined, opts.errorTypes?.length - ? CH.inList(CH.rawExpr(ERROR_FINGERPRINT_SQL), opts.errorTypes) + ? CH.inList(errorFingerprint($.StatusMessage), opts.errorTypes) : undefined, ]) .groupBy("errorType") @@ -97,7 +121,6 @@ export interface ErrorsTimeseriesOutput { export function errorsTimeseriesQuery( opts: ErrorsTimeseriesOpts, ) { - const esc = escapeClickHouseString return from(ErrorSpans) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), @@ -105,7 +128,7 @@ export function errorsTimeseriesQuery( })) .where(($) => [ $.OrgId.eq(param.string("orgId")), - CH.rawCond(`${ERROR_FINGERPRINT_SQL} = '${esc(opts.errorType)}'`), + errorFingerprint($.StatusMessage).eq(opts.errorType), $.Timestamp.gte(param.dateTime("startTime")), $.Timestamp.lte(param.dateTime("endTime")), opts.services?.length @@ -145,42 +168,43 @@ export interface SpanHierarchyOutput { export function spanHierarchyQuery( opts: SpanHierarchyOpts, ) { - // HTTP span name rewriting: "http.server GET" + route → "GET /api/users" - const httpRewriteExpr = CH.if_( - CH.rawCond( - "(SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))" + - " AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')", - ), - CH.rawExpr( - "concat(" + - "if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), " + - "' ', " + - "if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])" + - ")", - ), - CH.rawExpr("SpanName"), - ) - - const relationshipExpr = opts.spanId - ? CH.if_(CH.rawExpr("SpanId").eq(opts.spanId), CH.lit("target"), CH.lit("related")) - : CH.lit("related") - return from(Traces) - .select(($) => ({ - traceId: $.TraceId, - spanId: $.SpanId, - parentSpanId: $.ParentSpanId, - spanName: httpRewriteExpr, - serviceName: $.ServiceName, - spanKind: $.SpanKind, - durationMs: $.Duration.div(1000000), - startTime: $.Timestamp, - statusCode: $.StatusCode, - statusMessage: $.StatusMessage, - spanAttributes: CH.toJSONString($.SpanAttributes), - resourceAttributes: CH.toJSONString($.ResourceAttributes), - relationship: relationshipExpr, - })) + .select(($) => { + // HTTP span name rewriting: "http.server GET" + route → "GET /api/users" + const route = $.SpanAttributes.get("http.route") + const urlPath = $.SpanAttributes.get("url.path") + const httpRewriteExpr = CH.if_( + $.SpanName.like("http.server %") + .or($.SpanName.in_("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS")) + .and(route.neq("").or(urlPath.neq(""))), + CH.concat( + CH.if_($.SpanName.like("http.server %"), CH.replaceOne($.SpanName, "http.server ", ""), $.SpanName), + CH.lit(" "), + CH.if_(route.neq(""), route, urlPath), + ), + $.SpanName, + ) + + const relationshipExpr = opts.spanId + ? CH.if_($.SpanId.eq(opts.spanId), CH.lit("target"), CH.lit("related")) + : CH.lit("related") + + return { + traceId: $.TraceId, + spanId: $.SpanId, + parentSpanId: $.ParentSpanId, + spanName: httpRewriteExpr, + serviceName: $.ServiceName, + spanKind: $.SpanKind, + durationMs: $.Duration.div(1000000), + startTime: $.Timestamp, + statusCode: $.StatusCode, + statusMessage: $.StatusMessage, + spanAttributes: CH.toJSONString($.SpanAttributes), + resourceAttributes: CH.toJSONString($.ResourceAttributes), + relationship: relationshipExpr, + } + }) .where(($) => [ $.TraceId.eq(opts.traceId), $.OrgId.eq(param.string("orgId")), @@ -424,7 +448,7 @@ export function errorsFacetsQuery( ? CH.inList($.DeploymentEnv, opts.deploymentEnvs) : undefined, opts.errorTypes?.length - ? CH.inList(CH.rawExpr(ERROR_FINGERPRINT_SQL), opts.errorTypes) + ? CH.inList(errorFingerprint($.StatusMessage), opts.errorTypes) : undefined, ] @@ -451,8 +475,8 @@ export function errorsFacetsQuery( .limit(100) const errorTypeQuery = from(ErrorSpans) - .select(() => ({ - name: CH.rawExpr(ERROR_FINGERPRINT_SQL), + .select(($) => ({ + name: errorFingerprint($.StatusMessage), count: CH.count(), facetType: CH.lit("error_type"), })) diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index df988d90eb..8fe63fbb19 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -366,8 +366,8 @@ export function listMetricsQuery( firstSeen: CH.min_($.TimeUnix), lastSeen: CH.max_($.TimeUnix), isMonotonic: hasIsMonotonic - ? CH.rawExpr("any(IsMonotonic)") - : CH.rawExpr("0"), + ? CH.any_(CH.dynamicColumn("IsMonotonic")) + : CH.lit(0), })) .where(($) => [ $.OrgId.eq(param.string("orgId")), diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index c1829948c5..43b6a85227 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -9,13 +9,12 @@ import * as CH from "../expr" import { param } from "../param" import { from, type ColumnAccessor } from "../query" import { Traces, TraceListMv } from "../tables" -import { compile, str } from "../../sql/sql-fragment" import { METRIC_NEEDS, TRACE_LIST_MV_ATTR_MAP, TRACE_LIST_MV_RESOURCE_MAP, canUseTraceListMv, - buildAttrFilterSQL, + buildAttrFilterCondition, } from "../../traces-shared" // --------------------------------------------------------------------------- @@ -88,6 +87,7 @@ function metricSelectExprs( // --------------------------------------------------------------------------- function buildGroupNameExpr( + $: ColumnAccessor, groupBy: readonly string[] | undefined, groupByAttributeKeys: readonly string[] | undefined, useTraceListMv: boolean, @@ -96,32 +96,39 @@ function buildGroupNameExpr( return CH.lit("all") } - const parts: string[] = [] + const parts: CH.Expr[] = [] for (const g of groupBy) { switch (g) { case "service": - parts.push("toString(ServiceName)") + parts.push(CH.toString_($.ServiceName)) break case "span_name": - parts.push("toString(SpanName)") + parts.push(CH.toString_($.SpanName)) break case "status_code": - parts.push("toString(StatusCode)") + parts.push(CH.toString_($.StatusCode)) break case "http_method": if (useTraceListMv) { - parts.push("toString(HttpMethod)") + parts.push(CH.toString_(CH.dynamicColumn("HttpMethod"))) } else { - parts.push("toString(SpanAttributes['http.method'])") + parts.push(CH.toString_($.SpanAttributes.get("http.method"))) } break case "attribute": if (groupByAttributeKeys?.length) { - const keys = groupByAttributeKeys.map((k) => { + const keys: CH.Expr[] = groupByAttributeKeys.map((k) => { const mvCol = useTraceListMv ? TRACE_LIST_MV_ATTR_MAP[k] : undefined - return mvCol ? `toString(${mvCol})` : `toString(SpanAttributes[${compile(str(k))}])` + return mvCol + ? CH.toString_(CH.dynamicColumn(mvCol)) + : CH.toString_($.SpanAttributes.get(k)) }) - parts.push(`arrayStringConcat([${keys.join(", ")}], ' \u00b7 ')`) + // When multiple attribute keys, join them into a single part + if (keys.length === 1) { + parts.push(keys[0]) + } else { + parts.push(CH.arrayStringConcat(keys, " \u00b7 ")) + } } break case "none": @@ -134,33 +141,37 @@ function buildGroupNameExpr( } if (parts.length === 1) { - return CH.rawExpr(`coalesce(nullIf(${parts[0]}, ''), 'all')`) + return CH.coalesce(CH.nullIf(parts[0], ""), CH.lit("all")) } - return CH.rawExpr( - `coalesce(nullIf(arrayStringConcat(arrayFilter(x -> x != '', [${parts.join(", ")}]), ' \u00b7 '), ''), 'all')`, + // Multi-part: filter empty strings before joining with separator + const filtered = CH.arrayFilter("x -> x != ''", CH.arrayOf(...parts)) + return CH.coalesce( + CH.nullIf(CH.arrayStringConcat(filtered, " \u00b7 "), ""), + CH.lit("all"), ) } function buildBreakdownGroupExpr( + $: ColumnAccessor, groupBy: string, groupByAttributeKey: string | undefined, ): CH.Expr { switch (groupBy) { case "service": - return CH.rawExpr("ServiceName") + return $.ServiceName case "span_name": - return CH.rawExpr("SpanName") + return $.SpanName case "status_code": - return CH.rawExpr("StatusCode") + return $.StatusCode case "http_method": - return CH.rawExpr("SpanAttributes['http.method']") + return $.SpanAttributes.get("http.method") case "attribute": return groupByAttributeKey - ? CH.rawExpr(`SpanAttributes[${compile(str(groupByAttributeKey))}]`) - : CH.rawExpr("ServiceName") + ? $.SpanAttributes.get(groupByAttributeKey) + : $.ServiceName default: - return CH.rawExpr("ServiceName") + return $.ServiceName } } @@ -168,15 +179,6 @@ function buildBreakdownGroupExpr( // WHERE clause builders // --------------------------------------------------------------------------- -function buildAttrFilterCondition( - af: AttributeFilter, - useMv: boolean, - mapName: "SpanAttributes" | "ResourceAttributes", - mvMap: Record, -): CH.Condition { - return CH.rawCond(buildAttrFilterSQL(af, useMv, mapName, mvMap)) -} - function buildWhereConditions( $: ColumnAccessor, opts: TracesQueryOpts, @@ -189,12 +191,12 @@ function buildWhereConditions( $.Timestamp.lte(param.dateTime("endTime")), CH.when(opts.serviceName, (v: string) => mm?.serviceName === "contains" - ? CH.rawCond(`positionCaseInsensitive(ServiceName, ${compile(str(v))}) > 0`) + ? CH.positionCaseInsensitive($.ServiceName, CH.lit(v)).gt(0) : $.ServiceName.eq(v), ), CH.when(opts.spanName, (v: string) => mm?.spanName === "contains" - ? CH.rawCond(`positionCaseInsensitive(SpanName, ${compile(str(v))}) > 0`) + ? CH.positionCaseInsensitive($.SpanName, CH.lit(v)).gt(0) : $.SpanName.eq(v), ), CH.whenTrue(!!opts.rootOnly && !useTraceListMv, () => @@ -220,17 +222,19 @@ function buildWhereConditions( if (opts.environments?.length) { if (mm?.deploymentEnv === "contains" && opts.environments.length === 1) { - const envCol = useTraceListMv ? "DeploymentEnv" : "ResourceAttributes['deployment.environment']" - conditions.push(CH.rawCond(`positionCaseInsensitive(${envCol}, ${compile(str(opts.environments[0]))}) > 0`)) + const envExpr = useTraceListMv + ? CH.dynamicColumn("DeploymentEnv") + : $.ResourceAttributes.get("deployment.environment") + conditions.push(CH.positionCaseInsensitive(envExpr, CH.lit(opts.environments[0])).gt(0)) } else if (useTraceListMv) { - conditions.push(CH.inList(CH.rawExpr("DeploymentEnv"), opts.environments)) + conditions.push(CH.inList(CH.dynamicColumn("DeploymentEnv"), opts.environments)) } else { - conditions.push(CH.inList(CH.rawExpr("ResourceAttributes['deployment.environment']"), opts.environments)) + conditions.push(CH.inList($.ResourceAttributes.get("deployment.environment"), opts.environments)) } } if (opts.commitShas?.length) { - conditions.push(CH.inList(CH.rawExpr("ResourceAttributes['deployment.commit_sha']"), opts.commitShas)) + conditions.push(CH.inList($.ResourceAttributes.get("deployment.commit_sha"), opts.commitShas)) } if (opts.attributeFilters) { @@ -313,7 +317,7 @@ export function tracesTimeseriesQuery( return from(tbl as typeof Traces) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), - groupName: buildGroupNameExpr(opts.groupBy, opts.groupByAttributeKeys, useTraceListMv), + groupName: buildGroupNameExpr($, opts.groupBy, opts.groupByAttributeKeys, useTraceListMv), ...metricSelectExprs($, opts.metric, apdexThresholdMs, opts.needsSampling, opts.allMetrics), })) .where(($) => buildWhereConditions($, opts, useTraceListMv)) @@ -366,7 +370,7 @@ export function tracesBreakdownQuery( const { sampledSpanCount, unsampledSpanCount, dominantThreshold, ...metrics } = metricSelectExprs($, opts.metric, apdexThresholdMs, false, opts.allMetrics) return { - name: buildBreakdownGroupExpr(opts.groupBy, opts.groupByAttributeKey), + name: buildBreakdownGroupExpr($, opts.groupBy, opts.groupByAttributeKey), ...metrics, } }) @@ -466,7 +470,7 @@ export function tracesListQuery( spanName: $.SpanName, durationMs: $.Duration.div(1000000), statusCode: $.StatusCode, - spanKind: useTraceListMv ? CH.rawExpr("SpanKind") : $.SpanKind, + spanKind: useTraceListMv ? CH.dynamicColumn("SpanKind") : $.SpanKind, hasError: useTraceListMv ? CH.dynamicColumn("HasError") : CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), @@ -530,14 +534,14 @@ export function tracesRootListQuery( rootSpanKind: $.SpanKind, rootSpanStatusCode: $.StatusCode, rootHttpMethod: useTraceListMv - ? CH.rawExpr("HttpMethod") - : CH.rawExpr("SpanAttributes['http.method']"), + ? CH.dynamicColumn("HttpMethod") + : $.SpanAttributes.get("http.method"), rootHttpRoute: useTraceListMv - ? CH.rawExpr("HttpRoute") - : CH.rawExpr("SpanAttributes['http.route']"), + ? CH.dynamicColumn("HttpRoute") + : $.SpanAttributes.get("http.route"), rootHttpStatusCode: useTraceListMv - ? CH.rawExpr("HttpStatusCode") - : CH.rawExpr("SpanAttributes['http.status_code']"), + ? CH.dynamicColumn("HttpStatusCode") + : $.SpanAttributes.get("http.status_code"), hasError: useTraceListMv ? CH.dynamicColumn("HasError") : CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), diff --git a/packages/query-engine/src/traces-shared.ts b/packages/query-engine/src/traces-shared.ts index 0c3dc67aa7..eff5de21bc 100644 --- a/packages/query-engine/src/traces-shared.ts +++ b/packages/query-engine/src/traces-shared.ts @@ -61,54 +61,58 @@ export function canUseTraceListMv(_params: { } // --------------------------------------------------------------------------- -// Attribute filter → SQL string +// Attribute filter → typed Condition // --------------------------------------------------------------------------- -const MODE_TO_OPERATOR: Record = { - gt: ">", - gte: ">=", - lt: "<", - lte: "<=", -} +import * as CH from "./ch/expr" -export function buildAttrFilterSQL( +export function buildAttrFilterCondition( af: AttributeFilter, useMv: boolean, mapName: "SpanAttributes" | "ResourceAttributes", mvMap: Record, -): string { +): CH.Condition { const mvColumn = useMv ? mvMap[af.key] : undefined - const escapedKey = `'${escapeForSQL(af.key)}'` - const escapedValue = `'${escapeForSQL(af.value ?? "")}'` + const colExpr: CH.Expr = mvColumn + ? CH.dynamicColumn(mvColumn) + : CH.mapGet(CH.dynamicColumn>(mapName), af.key) + const value = af.value ?? "" if (af.mode === "exists") { return mvColumn - ? `${mvColumn} != ''` - : `mapContains(${mapName}, ${escapedKey})` + ? CH.dynamicColumn(mvColumn).neq("") + : CH.mapContains(CH.dynamicColumn>(mapName), af.key) } if (af.mode === "contains") { - const col = mvColumn ?? `${mapName}[${escapedKey}]` - return `positionCaseInsensitive(${col}, ${escapedValue}) > 0` + return CH.positionCaseInsensitive(colExpr, CH.lit(value)).gt(0) } - const op = MODE_TO_OPERATOR[af.mode] - if (op) { - if (mvColumn) { - const cast = NUMERIC_MV_COLUMNS.has(mvColumn) ? `toUInt16OrZero(${mvColumn})` : mvColumn - return `${cast} ${op} ${escapedValue}` - } - const rawEscaped = af.value?.replace(/\\/g, "\\\\").replace(/'/g, "\\'") ?? "" - return `toFloat64OrZero(${mapName}[${escapedKey}]) ${op} ${rawEscaped}` + if (af.mode === "gt") { + const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) + ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) + : CH.toFloat64OrZero(colExpr) + return numExpr.gt(Number(value)) } - - // equals (default) - if (mvColumn) { - return `${mvColumn} = ${escapedValue}` + if (af.mode === "gte") { + const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) + ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) + : CH.toFloat64OrZero(colExpr) + return numExpr.gte(Number(value)) + } + if (af.mode === "lt") { + const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) + ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) + : CH.toFloat64OrZero(colExpr) + return numExpr.lt(Number(value)) + } + if (af.mode === "lte") { + const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) + ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) + : CH.toFloat64OrZero(colExpr) + return numExpr.lte(Number(value)) } - return `${mapName}[${escapedKey}] = ${escapedValue}` -} -function escapeForSQL(value: string): string { - return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'") + // equals (default) + return colExpr.eq(value) } From acdc056763435619a41091653c831feac2ab6922 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 13:03:02 +0200 Subject: [PATCH 04/11] fix subqueries --- apps/api/src/services/PipeQueryDispatcher.ts | 8 +- apps/api/src/services/QueryEngineService.ts | 6 +- packages/query-engine/src/ch/compile.ts | 19 +- packages/query-engine/src/ch/index.ts | 8 +- .../query-engine/src/ch/queries/errors.ts | 204 +++++++++--------- .../query-engine/src/ch/queries/metrics.ts | 142 ++++++------ packages/query-engine/src/ch/query.ts | 52 +++++ packages/query-engine/src/sql/sql-query.ts | 17 ++ 8 files changed, 264 insertions(+), 192 deletions(-) diff --git a/apps/api/src/services/PipeQueryDispatcher.ts b/apps/api/src/services/PipeQueryDispatcher.ts index 114aa7d21e..fdf16ed756 100644 --- a/apps/api/src/services/PipeQueryDispatcher.ts +++ b/apps/api/src/services/PipeQueryDispatcher.ts @@ -168,17 +168,17 @@ export function compilePipeQuery( }), { orgId, startTime, endTime })), ), Match.when("errors_summary", () => - eraseType(CH.errorsSummarySQL({ + eraseType(CH.compile(CH.errorsSummaryQuery({ rootOnly: bool("root_only"), services: str("services")?.split(",").filter(Boolean), deploymentEnvs: str("deployment_envs")?.split(",").filter(Boolean), errorTypes: str("error_types")?.split(",").filter(Boolean), - }, { orgId, startTime, endTime })), + }), { orgId, startTime, endTime })), ), Match.when("error_detail_traces", () => - eraseType(CH.errorDetailTracesSQL({ + eraseType(CH.compile(CH.errorDetailTracesQuery({ errorType: String(params.error_type), rootOnly: bool("root_only"), services: str("services")?.split(",").filter(Boolean), limit: int("limit", 10), - }, { orgId, startTime, endTime })), + }), { orgId, startTime, endTime })), ), // ----- Metrics ----- Match.when("list_metrics", () => diff --git a/apps/api/src/services/QueryEngineService.ts b/apps/api/src/services/QueryEngineService.ts index 3ae0b15dd5..73461ea698 100644 --- a/apps/api/src/services/QueryEngineService.ts +++ b/apps/api/src/services/QueryEngineService.ts @@ -614,13 +614,13 @@ export const makeQueryEngineExecute = (tinybird: QueryEngineTinybird) => const isRateOrIncrease = request.query.metric === "rate" || request.query.metric === "increase" if (isRateOrIncrease) { - const compiled = CH.metricsTimeseriesRateSQL( - { + const compiled = CH.compile( + CH.metricsTimeseriesRateQuery({ serviceName: request.query.filters.serviceName, groupByAttributeKey, attributeKey: attributeFilter?.key, attributeValue: attributeFilter?.value, - }, + }), { orgId: tenant.orgId, metricName: request.query.filters.metricName, diff --git a/packages/query-engine/src/ch/compile.ts b/packages/query-engine/src/ch/compile.ts index efbc9bd4f1..910e7fb82a 100644 --- a/packages/query-engine/src/ch/compile.ts +++ b/packages/query-engine/src/ch/compile.ts @@ -13,7 +13,7 @@ import type { CHQuery } from "./query" import type { CHUnionQuery } from "./union" import { createColumnAccessor } from "./query" import { aliased } from "./expr" -import { raw, ident, escapeClickHouseString } from "../sql/sql-fragment" +import { raw, ident, escapeClickHouseString, compile as compileSqlFragment } from "../sql/sql-fragment" import { compileQuery, type SqlQuery } from "../sql/sql-query" import { Schema } from "effect" @@ -75,9 +75,20 @@ export function compileCH< ? raw(`(${state.fromSubquerySql}) AS ${state.fromSubqueryAlias}`) : ident(state.tableName) + // JOINs + const joins = state.joins.length > 0 + ? state.joins.map((j) => ({ + type: j.type, + table: j.tableSql, + alias: j.alias, + on: j.on ? compileSqlFragment(j.on.toFragment()) : undefined, + })) + : undefined + const sqlQuery: SqlQuery = { select: selectFragments, from: fromFragment, + joins, where: whereFragments, groupBy: state.groupByKeys.map((k) => raw(k)), orderBy: state.orderBySpecs.map(([k, dir]) => raw(`${k} ${dir.toUpperCase()}`)), @@ -88,6 +99,12 @@ export function compileCH< let sql = compileQuery(sqlQuery) + // Prepend CTE definitions + if (state.ctes.length > 0) { + const cteDefs = state.ctes.map((c) => `${c.name} AS (\n${c.sql}\n)`).join(",\n") + sql = `WITH ${cteDefs}\n${sql}` + } + // Replace param placeholders with resolved values for (const [name, value] of Object.entries(params)) { const placeholder = `__PARAM_${name}__` diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index df88eb5f0f..1f300878a0 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -139,7 +139,7 @@ export { // Queries — Metrics export { metricsTimeseriesQuery, - metricsTimeseriesRateSQL, + metricsTimeseriesRateQuery, metricsBreakdownQuery, type MetricsTimeseriesOpts, type MetricsTimeseriesOutput, @@ -194,15 +194,15 @@ export { // Queries — Errors export { - ERROR_FINGERPRINT_SQL, + errorFingerprint, errorsByTypeQuery, errorsTimeseriesQuery, spanHierarchyQuery, tracesDurationStatsQuery, tracesFacetsQuery, errorsFacetsQuery, - errorsSummarySQL, - errorDetailTracesSQL, + errorsSummaryQuery, + errorDetailTracesQuery, type ErrorsByTypeOpts, type ErrorsByTypeOutput, type ErrorsTimeseriesOpts, diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index f531d85573..adadaa7431 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -6,11 +6,11 @@ import * as CH from "../expr" import { param } from "../param" -import { from, type ColumnAccessor } from "../query" +import { from, fromSubquery, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" -import { ErrorSpans, TraceListMv, Traces } from "../tables" +import { compileCH } from "../compile" +import { ErrorSpans, ServiceUsage, TraceListMv, Traces } from "../tables" import { escapeClickHouseString } from "../../sql/sql-fragment" -import type { CompiledQuery } from "../compile" // --------------------------------------------------------------------------- // Shared: Error fingerprint expression (typed DSL) @@ -36,19 +36,6 @@ export function errorFingerprint(statusMessage: CH.Expr): CH.Expr 3, toInt64(position(StatusMessage, ': ')) - 1, - position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, - position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1, - position(StatusMessage, '{') > 10, toInt64(position(StatusMessage, '{')) - 1, - least(toInt64(length(StatusMessage)), 150) - )) -)` // --------------------------------------------------------------------------- // Errors by type @@ -490,7 +477,7 @@ export function errorsFacetsQuery( } // --------------------------------------------------------------------------- -// Errors summary (raw SQL — CROSS JOIN between error_spans and service_usage) +// Errors summary (CROSS JOIN between error_spans and service_usage) // --------------------------------------------------------------------------- export interface ErrorsSummaryOpts { @@ -508,58 +495,64 @@ export interface ErrorsSummaryOutput { readonly affectedTracesCount: number } -export function errorsSummarySQL( +export function errorsSummaryQuery( opts: ErrorsSummaryOpts, - params: { orgId: string; startTime: string; endTime: string }, -): CompiledQuery { - const esc = escapeClickHouseString - const errorConditions: string[] = [ - `OrgId = '${esc(params.orgId)}'`, - `Timestamp >= '${esc(params.startTime)}'`, - `Timestamp <= '${esc(params.endTime)}'`, - ] - if (opts.rootOnly) errorConditions.push("ParentSpanId = ''") - if (opts.services?.length) { - errorConditions.push(`ServiceName IN (${opts.services.map((s) => `'${esc(s)}'`).join(", ")})`) - } - if (opts.deploymentEnvs?.length) { - errorConditions.push(`DeploymentEnv IN (${opts.deploymentEnvs.map((e) => `'${esc(e)}'`).join(", ")})`) - } - if (opts.errorTypes?.length) { - errorConditions.push(`${ERROR_FINGERPRINT_SQL} IN (${opts.errorTypes.map((t) => `'${esc(t)}'`).join(", ")})`) - } +) { + // Build the error subquery + const errorSubquery = compileCH( + from(ErrorSpans) + .select(($) => ({ + totalErrors: CH.count(), + affectedServicesCount: CH.uniq($.ServiceName), + affectedTracesCount: CH.uniq($.TraceId), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), + opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, + opts.deploymentEnvs?.length ? CH.inList($.DeploymentEnv, opts.deploymentEnvs) : undefined, + opts.errorTypes?.length ? CH.inList(errorFingerprint($.StatusMessage), opts.errorTypes) : undefined, + ]), + {}, // params resolved later by outer compile + { skipFormat: true }, + ) - const sql = `SELECT - e.totalErrors AS totalErrors, - s.totalSpans AS totalSpans, - if(s.totalSpans > 0, round(e.totalErrors / s.totalSpans * 100, 4), 0) AS errorRate, - e.affectedServicesCount AS affectedServicesCount, - e.affectedTracesCount AS affectedTracesCount -FROM ( - SELECT - count() AS totalErrors, - uniq(ServiceName) AS affectedServicesCount, - uniq(TraceId) AS affectedTracesCount - FROM error_spans - WHERE ${errorConditions.join("\n AND ")} -) AS e -CROSS JOIN ( - SELECT sum(TraceCount) AS totalSpans - FROM service_usage - WHERE OrgId = '${esc(params.orgId)}' - AND Hour >= '${esc(params.startTime)}' - AND Hour <= '${esc(params.endTime)}' -) AS s -FORMAT JSON` - - return { - sql, - castRows: (rows) => rows as unknown as ReadonlyArray, - } + // Build the service usage subquery + const usageSubquery = compileCH( + from(ServiceUsage) + .select(($) => ({ + totalSpans: CH.sum($.TraceCount), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Hour.gte(param.dateTime("startTime")), + $.Hour.lte(param.dateTime("endTime")), + ]), + {}, + { skipFormat: true }, + ) + + // Outer query: FROM error subquery, CROSS JOIN usage subquery + return fromSubquery(errorSubquery.sql, "e") + .join(`(${usageSubquery.sql})`, "s", undefined, "CROSS") + .select(() => ({ + totalErrors: CH.dynamicColumn("e.totalErrors"), + totalSpans: CH.dynamicColumn("s.totalSpans"), + errorRate: CH.if_( + CH.dynamicColumn("s.totalSpans").gt(0), + CH.round_(CH.dynamicColumn("e.totalErrors").div(CH.dynamicColumn("s.totalSpans")).mul(100), 4), + CH.lit(0), + ), + affectedServicesCount: CH.dynamicColumn("e.affectedServicesCount"), + affectedTracesCount: CH.dynamicColumn("e.affectedTracesCount"), + })) + .format("JSON") } // --------------------------------------------------------------------------- -// Error detail traces (raw SQL — subquery + INNER JOIN) +// Error detail traces (INNER JOIN with error subquery) // --------------------------------------------------------------------------- export interface ErrorDetailTracesOpts { @@ -574,53 +567,52 @@ export interface ErrorDetailTracesOutput { readonly startTime: string readonly durationMicros: number readonly spanCount: number - readonly services: string[] + readonly services: readonly string[] readonly rootSpanName: string readonly errorMessage: string } -export function errorDetailTracesSQL( +export function errorDetailTracesQuery( opts: ErrorDetailTracesOpts, - params: { orgId: string; startTime: string; endTime: string }, -): CompiledQuery { - const esc = escapeClickHouseString +) { const limit = opts.limit ?? 10 - const errorConditions: string[] = [ - `OrgId = '${esc(params.orgId)}'`, - `${ERROR_FINGERPRINT_SQL} = '${esc(opts.errorType)}'`, - `Timestamp >= '${esc(params.startTime)}'`, - `Timestamp <= '${esc(params.endTime)}'`, - ] - if (opts.rootOnly) errorConditions.push("ParentSpanId = ''") - if (opts.services?.length) { - errorConditions.push(`ServiceName IN (${opts.services.map((s) => `'${esc(s)}'`).join(", ")})`) - } - const sql = `SELECT - t.TraceId AS traceId, - min(t.Timestamp) AS startTime, - intDiv(max(t.Duration), 1000) AS durationMicros, - count() AS spanCount, - groupUniqArray(t.ServiceName) AS services, - anyIf(t.SpanName, t.ParentSpanId = '') AS rootSpanName, - any(t.StatusMessage) AS errorMessage -FROM traces AS t -INNER JOIN ( - SELECT DISTINCT TraceId - FROM error_spans - WHERE ${errorConditions.join("\n AND ")} - ORDER BY Timestamp DESC - LIMIT ${Math.round(limit)} -) AS e ON t.TraceId = e.TraceId -WHERE t.OrgId = '${esc(params.orgId)}' - AND t.Timestamp >= '${esc(params.startTime)}' - AND t.Timestamp <= '${esc(params.endTime)}' -GROUP BY t.TraceId -ORDER BY startTime DESC -FORMAT JSON` - - return { - sql, - castRows: (rows) => rows as unknown as ReadonlyArray, - } + // Subquery: find matching error TraceIds + const errorSubquery = compileCH( + from(ErrorSpans) + .select(($) => ({ TraceId: $.TraceId })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + errorFingerprint($.StatusMessage).eq(opts.errorType), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), + opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, + ]) + .orderBy(["TraceId", "desc"]) + .limit(limit), + {}, + { skipFormat: true }, + ) + + // Outer query: join traces with error subquery + return from(Traces) + .join(`(SELECT DISTINCT TraceId FROM (${errorSubquery.sql}))`, "e", CH.rawCond("traces.TraceId = e.TraceId"), "INNER") + .select(($) => ({ + traceId: $.TraceId, + startTime: CH.min_($.Timestamp), + durationMicros: CH.intDiv(CH.max_($.Duration), 1000), + spanCount: CH.count(), + services: CH.groupUniqArray($.ServiceName), + rootSpanName: CH.anyIf($.SpanName, $.ParentSpanId.eq("")), + errorMessage: CH.any_($.StatusMessage), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + ]) + .groupBy("traceId") + .orderBy(["startTime", "desc"]) + .format("JSON") } diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index 8fe63fbb19..d0d10a4124 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -7,8 +7,10 @@ import type { MetricType } from "../../query-engine" import * as CH from "../expr" +import * as T from "../types" import { param } from "../param" import { from, type CHQuery } from "../query" +import { table } from "../table" import { unionAll, type CHUnionQuery } from "../union" import { MetricsSum, @@ -16,8 +18,7 @@ import { MetricsHistogram, MetricsExpHistogram, } from "../tables" -import { escapeClickHouseString } from "../../sql/sql-fragment" -import type { CompiledQuery } from "../compile" +import { compileCH } from "../compile" // --------------------------------------------------------------------------- // Table lookup @@ -168,81 +169,74 @@ export interface MetricsRateTimeseriesOutput { readonly dataPointCount: number } -export function metricsTimeseriesRateSQL( +export function metricsTimeseriesRateQuery( opts: MetricsRateTimeseriesOpts, - params: { orgId: string; metricName: string; startTime: string; endTime: string; bucketSeconds: number }, -): CompiledQuery { - const esc = escapeClickHouseString - const bucketSeconds = Math.round(params.bucketSeconds) - - // CTE WHERE clauses - const cteWhereFragments = [ - `MetricName = '${esc(params.metricName)}'`, - `OrgId = '${esc(params.orgId)}'`, - `IsMonotonic = 1`, - `TimeUnix >= '${esc(params.startTime)}' - INTERVAL ${bucketSeconds} SECOND`, - `TimeUnix <= '${esc(params.endTime)}'`, - ] - if (opts.serviceName) { - cteWhereFragments.push(`ServiceName = '${esc(opts.serviceName)}'`) - } - if (opts.attributeKey) { - cteWhereFragments.push( - `Attributes['${esc(opts.attributeKey)}'] = '${esc(opts.attributeValue ?? '')}'`, - ) - } - - // Outer SELECT attribute column - const attributeSelect = opts.groupByAttributeKey - ? `Attributes['${esc(opts.groupByAttributeKey)}'] AS attributeValue` - : `'' AS attributeValue` +) { + // CTE: compute deltas using window functions + const cteSql = compileCH( + from(MetricsSum) + .select(($) => ({ + TimeUnix: $.TimeUnix, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + Value: $.Value, + delta: CH.rawExpr( + "Value - lagInFrame(Value, 1, Value) OVER (PARTITION BY ServiceName, MetricName, Attributes ORDER BY TimeUnix ASC)", + ), + time_delta: CH.rawExpr( + "toFloat64(toUnixTimestamp64Nano(TimeUnix) - toUnixTimestamp64Nano(lagInFrame(TimeUnix, 1, TimeUnix) OVER (PARTITION BY ServiceName, MetricName, Attributes ORDER BY TimeUnix ASC))) / 1000000000.0", + ), + })) + .where(($) => [ + $.MetricName.eq(param.string("metricName")), + $.OrgId.eq(param.string("orgId")), + CH.dynamicColumn("IsMonotonic").eq(1), + CH.rawCond(`TimeUnix >= __PARAM_startTime__ - INTERVAL __PARAM_bucketSeconds__ SECOND`), + $.TimeUnix.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), + CH.when(opts.attributeKey, (k: string) => + $.Attributes.get(k).eq(opts.attributeValue ?? ""), + ), + ]), + {}, + { skipFormat: true }, + ) - // Outer GROUP BY - const groupByParts = ["bucket", "ServiceName"] - if (opts.groupByAttributeKey) { - groupByParts.push(`Attributes['${esc(opts.groupByAttributeKey)}']`) - } + // Outer query: aggregate deltas into rate/increase per bucket + const cteTable = table("with_deltas", { + TimeUnix: T.dateTime64, + ServiceName: T.string, + Attributes: T.map(T.string, T.string), + Value: T.float64, + delta: T.float64, + time_delta: T.float64, + }) + + const q = from(cteTable) + .withCTE("with_deltas", cteSql.sql) + .select(($) => ({ + bucket: CH.toStartOfInterval($.TimeUnix, param.int("bucketSeconds")), + serviceName: $.ServiceName, + attributeValue: opts.groupByAttributeKey + ? $.Attributes.get(opts.groupByAttributeKey) + : CH.lit(""), + rateValue: CH.sumIf( + CH.dynamicColumn("delta").div(CH.dynamicColumn("time_delta")), + CH.dynamicColumn("delta").gte(0).and(CH.dynamicColumn("time_delta").gt(0)), + ), + increaseValue: CH.sumIf(CH.dynamicColumn("delta"), CH.dynamicColumn("delta").gte(0)), + dataPointCount: CH.count(), + })) + .where(($) => [ + $.TimeUnix.gte(param.dateTime("startTime")), + ]) - const sql = ` -WITH with_deltas AS ( - SELECT - TimeUnix, - ServiceName, - Attributes, - Value, - Value - lagInFrame(Value, 1, Value) OVER ( - PARTITION BY ServiceName, MetricName, Attributes - ORDER BY TimeUnix ASC - ) AS delta, - toFloat64( - toUnixTimestamp64Nano(TimeUnix) - toUnixTimestamp64Nano( - lagInFrame(TimeUnix, 1, TimeUnix) OVER ( - PARTITION BY ServiceName, MetricName, Attributes - ORDER BY TimeUnix ASC - ) - ) - ) / 1000000000.0 AS time_delta - FROM metrics_sum - WHERE ${cteWhereFragments.join("\n AND ")} -) -SELECT - toStartOfInterval(TimeUnix, INTERVAL ${bucketSeconds} SECOND) AS bucket, - ServiceName AS serviceName, - ${attributeSelect}, - sumIf(delta / time_delta, delta >= 0 AND time_delta > 0) AS rateValue, - sumIf(delta, delta >= 0) AS increaseValue, - count() AS dataPointCount -FROM with_deltas -WHERE TimeUnix >= '${esc(params.startTime)}' -GROUP BY ${groupByParts.join(", ")} -ORDER BY bucket ASC -FORMAT JSON -`.trim() - - return { - sql, - castRows: (rows) => rows as unknown as ReadonlyArray, - } + return (opts.groupByAttributeKey + ? q.groupBy("bucket", "serviceName", "attributeValue") + : q.groupBy("bucket", "serviceName") + ) + .orderBy(["bucket", "asc"]) + .format("JSON") } // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/query.ts b/packages/query-engine/src/ch/query.ts index 862830a17d..56c7fd3751 100644 --- a/packages/query-engine/src/ch/query.ts +++ b/packages/query-engine/src/ch/query.ts @@ -43,6 +43,15 @@ export type OrderBySpec = [keyof Output & string, "asc" | "desc"] // Query state (runtime storage) // --------------------------------------------------------------------------- +export interface JoinClause { + readonly type: "INNER" | "LEFT" | "CROSS" + /** Table name, or subquery SQL (without parens — compiler wraps subqueries). */ + readonly tableSql: string + readonly alias: string + /** ON condition. Omitted for CROSS JOIN. */ + readonly on?: Condition +} + export interface CHQueryState { readonly tableName: string readonly columns: ColumnDefs @@ -56,6 +65,9 @@ export interface CHQueryState { /** When set, the FROM clause uses a subquery instead of a table name. */ readonly fromSubquerySql?: string readonly fromSubqueryAlias?: string + readonly joins: JoinClause[] + /** CTE definitions prepended as WITH clauses. */ + readonly ctes: Array<{ name: string; sql: string }> } // --------------------------------------------------------------------------- @@ -95,6 +107,25 @@ export interface CHQuery< format(fmt: "JSON" | "JSONEachRow"): CHQuery + /** + * Add a JOIN clause. The joined table's columns are accessed via the alias + * in raw expressions (e.g., `CH.dynamicColumn("e.TraceId")`). + * + * For CROSS JOIN, pass `on` as `undefined`. + */ + join( + tableSql: string, + alias: string, + on: Condition | undefined, + type?: "INNER" | "LEFT" | "CROSS", + ): CHQuery + + /** + * Add a CTE (WITH clause). The CTE SQL is prepended to the compiled query. + * The CTE name can then be used as a table name via `from()` or in raw expressions. + */ + withCTE(name: string, sql: string): CHQuery + /** * @deprecated Params are now inferred at the `compile()` call site. * This method is a no-op and can be safely removed. @@ -183,6 +214,23 @@ function makeQuery< return makeQuery({ ...state, formatValue: fmt }) }, + join(tableSql, alias, on, type = "INNER") { + return makeQuery({ + ...state, + joins: [ + ...state.joins, + { type, tableSql, alias, on }, + ], + }) + }, + + withCTE(name, sql) { + return makeQuery({ + ...state, + ctes: [...state.ctes, { name, sql }], + }) + }, + withParams() { // Params is phantom — same runtime state, refined compile-time type return makeQuery(state) as any @@ -202,6 +250,8 @@ export function from( columns: table.columns, groupByKeys: [], orderBySpecs: [], + joins: [], + ctes: [], }) } @@ -225,6 +275,8 @@ export function fromSubquery( columns: {}, groupByKeys: [], orderBySpecs: [], + joins: [], + ctes: [], fromSubquerySql: sql, fromSubqueryAlias: alias, }) diff --git a/packages/query-engine/src/sql/sql-query.ts b/packages/query-engine/src/sql/sql-query.ts index d8908fc772..e124c43934 100644 --- a/packages/query-engine/src/sql/sql-query.ts +++ b/packages/query-engine/src/sql/sql-query.ts @@ -5,9 +5,17 @@ import { compile } from "./sql-fragment" // Query structure // --------------------------------------------------------------------------- +export interface SqlJoin { + readonly type: "INNER" | "LEFT" | "CROSS" + readonly table: string + readonly alias: string + readonly on?: string +} + export interface SqlQuery { readonly select: ReadonlyArray readonly from: SqlFragment + readonly joins?: ReadonlyArray readonly where: ReadonlyArray readonly groupBy: ReadonlyArray readonly orderBy: ReadonlyArray @@ -30,6 +38,15 @@ export function compileQuery(q: SqlQuery): string { // FROM parts.push(`FROM ${compile(q.from)}`) + // JOINs + if (q.joins?.length) { + for (const j of q.joins) { + const joinType = j.type === "CROSS" ? "CROSS JOIN" : `${j.type} JOIN` + const onClause = j.on ? ` ON ${j.on}` : "" + parts.push(`${joinType} ${j.table} AS ${j.alias}${onClause}`) + } + } + // WHERE const whereClauses = q.where.map(compile).filter(Boolean) if (whereClauses.length > 0) { From a2390c39cef97660b8b926b32e7ee966f1268f1c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 13:08:34 +0200 Subject: [PATCH 05/11] make more typesafe --- packages/query-engine/src/ch/expr.ts | 9 +++ packages/query-engine/src/ch/index.ts | 1 + .../query-engine/src/ch/queries/errors.ts | 66 ++++++++++++------- .../query-engine/src/ch/queries/metrics.ts | 2 +- .../query-engine/src/ch/queries/traces.ts | 26 ++++---- 5 files changed, 63 insertions(+), 41 deletions(-) diff --git a/packages/query-engine/src/ch/expr.ts b/packages/query-engine/src/ch/expr.ts index aa57205857..3e9f90a713 100644 --- a/packages/query-engine/src/ch/expr.ts +++ b/packages/query-engine/src/ch/expr.ts @@ -211,6 +211,15 @@ export function toStartOfInterval( return makeExpr(raw(`toStartOfInterval(${compile(col.toFragment())}, INTERVAL ${secStr} SECOND)`)) } +/** Subtract an interval: `expr - INTERVAL n SECOND` */ +export function intervalSub( + col: Expr, + seconds: number | Expr, +): Expr { + const secStr = typeof seconds === "number" ? String(Math.round(seconds)) : compile((seconds as Expr).toFragment()) + return makeExpr(raw(`${compile(col.toFragment())} - INTERVAL ${secStr} SECOND`)) +} + export function if_( cond: Condition, then_: Expr, diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 1f300878a0..459928f823 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -49,6 +49,7 @@ export { sumIf, groupUniqArray, toStartOfInterval, + intervalSub, toJSONString, if_, concat, diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index adadaa7431..ace9df191c 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -10,7 +10,6 @@ import { from, fromSubquery, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" import { compileCH } from "../compile" import { ErrorSpans, ServiceUsage, TraceListMv, Traces } from "../tables" -import { escapeClickHouseString } from "../../sql/sql-fragment" // --------------------------------------------------------------------------- // Shared: Error fingerprint expression (typed DSL) @@ -302,8 +301,6 @@ export interface TracesFacetsOutput { export function tracesFacetsQuery( opts: TracesFacetsOpts, ): CHUnionQuery { - const esc = escapeClickHouseString - const baseWhere = ($: ColumnAccessor): Array => { const conditions: Array = [ $.OrgId.eq(param.string("orgId")), @@ -338,30 +335,49 @@ export function tracesFacetsQuery( ) } - // Attribute filter EXISTS subqueries (kept as rawCond — involves correlated subquery) + // Attribute filter EXISTS subqueries (correlated — references outer TraceId) if (opts.attributeFilterKey) { - const matchExpr = opts.attributeFilterValueMatchMode === "contains" - ? `positionCaseInsensitive(t_attr.SpanAttributes['${esc(opts.attributeFilterKey)}'], '${esc(opts.attributeFilterValue ?? "")}') > 0` - : `t_attr.SpanAttributes['${esc(opts.attributeFilterKey)}'] = '${esc(opts.attributeFilterValue ?? "")}'` - conditions.push(CH.rawCond(`EXISTS ( - SELECT 1 FROM traces AS t_attr - WHERE t_attr.TraceId = TraceId AND t_attr.OrgId = __PARAM_orgId__ - AND t_attr.Timestamp >= __PARAM_startTime__ - AND t_attr.Timestamp <= __PARAM_endTime__ - AND ${matchExpr} - )`)) + const attrCol = CH.mapGet(CH.dynamicColumn>("t_attr.SpanAttributes"), opts.attributeFilterKey) + const matchCond = opts.attributeFilterValueMatchMode === "contains" + ? CH.positionCaseInsensitive(attrCol, CH.lit(opts.attributeFilterValue ?? "")).gt(0) + : attrCol.eq(opts.attributeFilterValue ?? "") + const innerSql = compileCH( + from(Traces) + .select(() => ({ _: CH.lit(1) })) + .where(() => [ + CH.dynamicColumn("t_attr.TraceId").eq(CH.outerRef("TraceId")), + CH.dynamicColumn("t_attr.OrgId").eq(param.string("orgId")), + CH.dynamicColumn("t_attr.Timestamp").gte(param.dateTime("startTime")), + CH.dynamicColumn("t_attr.Timestamp").lte(param.dateTime("endTime")), + matchCond, + ]), + {}, + { skipFormat: true }, + ) + // Replace the FROM clause to use aliased table + const existsSql = innerSql.sql.replace("FROM traces", "FROM traces AS t_attr") + conditions.push(CH.exists(existsSql)) } if (opts.resourceFilterKey) { - const matchExpr = opts.resourceFilterValueMatchMode === "contains" - ? `positionCaseInsensitive(t_res.ResourceAttributes['${esc(opts.resourceFilterKey)}'], '${esc(opts.resourceFilterValue ?? "")}') > 0` - : `t_res.ResourceAttributes['${esc(opts.resourceFilterKey)}'] = '${esc(opts.resourceFilterValue ?? "")}'` - conditions.push(CH.rawCond(`EXISTS ( - SELECT 1 FROM traces AS t_res - WHERE t_res.TraceId = TraceId AND t_res.OrgId = __PARAM_orgId__ - AND t_res.Timestamp >= __PARAM_startTime__ - AND t_res.Timestamp <= __PARAM_endTime__ - AND ${matchExpr} - )`)) + const resCol = CH.mapGet(CH.dynamicColumn>("t_res.ResourceAttributes"), opts.resourceFilterKey) + const matchCond = opts.resourceFilterValueMatchMode === "contains" + ? CH.positionCaseInsensitive(resCol, CH.lit(opts.resourceFilterValue ?? "")).gt(0) + : resCol.eq(opts.resourceFilterValue ?? "") + const innerSql = compileCH( + from(Traces) + .select(() => ({ _: CH.lit(1) })) + .where(() => [ + CH.dynamicColumn("t_res.TraceId").eq(CH.outerRef("TraceId")), + CH.dynamicColumn("t_res.OrgId").eq(param.string("orgId")), + CH.dynamicColumn("t_res.Timestamp").gte(param.dateTime("startTime")), + CH.dynamicColumn("t_res.Timestamp").lte(param.dateTime("endTime")), + matchCond, + ]), + {}, + { skipFormat: true }, + ) + const existsSql = innerSql.sql.replace("FROM traces", "FROM traces AS t_res") + conditions.push(CH.exists(existsSql)) } return conditions @@ -597,7 +613,7 @@ export function errorDetailTracesQuery( // Outer query: join traces with error subquery return from(Traces) - .join(`(SELECT DISTINCT TraceId FROM (${errorSubquery.sql}))`, "e", CH.rawCond("traces.TraceId = e.TraceId"), "INNER") + .join(`(SELECT DISTINCT TraceId FROM (${errorSubquery.sql}))`, "e", CH.dynamicColumn("traces.TraceId").eq(CH.dynamicColumn("e.TraceId")), "INNER") .select(($) => ({ traceId: $.TraceId, startTime: CH.min_($.Timestamp), diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index d0d10a4124..e44e3398e7 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -191,7 +191,7 @@ export function metricsTimeseriesRateQuery( $.MetricName.eq(param.string("metricName")), $.OrgId.eq(param.string("orgId")), CH.dynamicColumn("IsMonotonic").eq(1), - CH.rawCond(`TimeUnix >= __PARAM_startTime__ - INTERVAL __PARAM_bucketSeconds__ SECOND`), + $.TimeUnix.gte(CH.intervalSub(param.dateTime("startTime"), param.int("bucketSeconds"))), $.TimeUnix.lte(param.dateTime("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), CH.when(opts.attributeKey, (k: string) => diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index 43b6a85227..2845eb116c 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -414,16 +414,16 @@ function buildProjectedMapExpr( useMv: boolean, mapName: "SpanAttributes" | "ResourceAttributes", mvMap: Record, -): string { - if (requestedKeys.length === 0) return "map()" - const pairs: string[] = [] - for (const key of requestedKeys) { - const escaped = `'${key.replace(/'/g, "\\'")}'` +): CH.Expr> { + if (requestedKeys.length === 0) return CH.mapLiteral() + const pairs: Array<[string, CH.Expr]> = requestedKeys.map((key) => { const mvCol = useMv ? mvMap[key] : undefined - const value = mvCol ?? `${mapName}[${escaped}]` - pairs.push(`${escaped}, ${value}`) - } - return `map(${pairs.join(", ")})` + const valueExpr: CH.Expr = mvCol + ? CH.dynamicColumn(mvCol) + : CH.mapGet(CH.dynamicColumn>(mapName), key) + return [key, valueExpr] + }) + return CH.mapLiteral(...pairs) } export function tracesListQuery( @@ -452,14 +452,10 @@ export function tracesListQuery( // When using the raw table and no specific attr columns are requested, skip the full maps const spanAttrExpr = needsFullMaps && !useTraceListMv ? undefined // use $.SpanAttributes directly - : CH.rawExpr>( - buildProjectedMapExpr(requestedSpanAttrKeys, useTraceListMv, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP), - ) + : buildProjectedMapExpr(requestedSpanAttrKeys, useTraceListMv, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP) const resourceAttrExpr = needsFullMaps && !useTraceListMv ? undefined // use $.ResourceAttributes directly - : CH.rawExpr>( - buildProjectedMapExpr(requestedResourceAttrKeys, useTraceListMv, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP), - ) + : buildProjectedMapExpr(requestedResourceAttrKeys, useTraceListMv, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP) let q = from(tbl as typeof Traces) .select(($) => ({ From 8b12a8a2c416e766c9eb3474c0b8da203b2180e6 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 13:09:58 +0200 Subject: [PATCH 06/11] s --- packages/query-engine/src/ch/compile.ts | 4 +++- packages/query-engine/src/ch/queries/errors.ts | 11 ++++------- packages/query-engine/src/ch/query.ts | 3 +++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/query-engine/src/ch/compile.ts b/packages/query-engine/src/ch/compile.ts index 910e7fb82a..559295279b 100644 --- a/packages/query-engine/src/ch/compile.ts +++ b/packages/query-engine/src/ch/compile.ts @@ -73,7 +73,9 @@ export function compileCH< // Resolve param placeholders in the compiled SQL const fromFragment = state.fromSubquerySql ? raw(`(${state.fromSubquerySql}) AS ${state.fromSubqueryAlias}`) - : ident(state.tableName) + : state.tableAlias + ? raw(`${state.tableName} AS ${state.tableAlias}`) + : ident(state.tableName) // JOINs const joins = state.joins.length > 0 diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index ace9df191c..e48757707f 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -342,7 +342,7 @@ export function tracesFacetsQuery( ? CH.positionCaseInsensitive(attrCol, CH.lit(opts.attributeFilterValue ?? "")).gt(0) : attrCol.eq(opts.attributeFilterValue ?? "") const innerSql = compileCH( - from(Traces) + from(Traces, "t_attr") .select(() => ({ _: CH.lit(1) })) .where(() => [ CH.dynamicColumn("t_attr.TraceId").eq(CH.outerRef("TraceId")), @@ -354,9 +354,7 @@ export function tracesFacetsQuery( {}, { skipFormat: true }, ) - // Replace the FROM clause to use aliased table - const existsSql = innerSql.sql.replace("FROM traces", "FROM traces AS t_attr") - conditions.push(CH.exists(existsSql)) + conditions.push(CH.exists(innerSql.sql)) } if (opts.resourceFilterKey) { const resCol = CH.mapGet(CH.dynamicColumn>("t_res.ResourceAttributes"), opts.resourceFilterKey) @@ -364,7 +362,7 @@ export function tracesFacetsQuery( ? CH.positionCaseInsensitive(resCol, CH.lit(opts.resourceFilterValue ?? "")).gt(0) : resCol.eq(opts.resourceFilterValue ?? "") const innerSql = compileCH( - from(Traces) + from(Traces, "t_res") .select(() => ({ _: CH.lit(1) })) .where(() => [ CH.dynamicColumn("t_res.TraceId").eq(CH.outerRef("TraceId")), @@ -376,8 +374,7 @@ export function tracesFacetsQuery( {}, { skipFormat: true }, ) - const existsSql = innerSql.sql.replace("FROM traces", "FROM traces AS t_res") - conditions.push(CH.exists(existsSql)) + conditions.push(CH.exists(innerSql.sql)) } return conditions diff --git a/packages/query-engine/src/ch/query.ts b/packages/query-engine/src/ch/query.ts index 56c7fd3751..a6c7672b1f 100644 --- a/packages/query-engine/src/ch/query.ts +++ b/packages/query-engine/src/ch/query.ts @@ -54,6 +54,7 @@ export interface JoinClause { export interface CHQueryState { readonly tableName: string + readonly tableAlias?: string readonly columns: ColumnDefs readonly selectFn?: ($: any) => SelectRecord readonly whereFn?: ($: any) => Array @@ -244,9 +245,11 @@ function makeQuery< export function from( table: Table, + alias?: string, ): CHQuery { return makeQuery({ tableName: table.name, + tableAlias: alias, columns: table.columns, groupByKeys: [], orderBySpecs: [], From ba3f2cb1c21e071599f53aa35fb86db9b182a802 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 13:19:26 +0200 Subject: [PATCH 07/11] ad dtests --- packages/query-engine/src/ch/core-dsl.test.ts | 333 ++++++++++++++++++ .../src/ch/queries/alerts.test.ts | 210 +++++++++++ .../src/ch/queries/attribute-keys.test.ts | 79 +++++ .../src/ch/queries/errors.test.ts | 264 ++++++++++++++ .../query-engine/src/ch/queries/logs.test.ts | 244 +++++++++++++ .../src/ch/queries/metrics.test.ts | 237 +++++++++++++ .../src/ch/queries/services.test.ts | 159 +++++++++ packages/query-engine/src/ch/test-utils.ts | 79 +++++ 8 files changed, 1605 insertions(+) create mode 100644 packages/query-engine/src/ch/core-dsl.test.ts create mode 100644 packages/query-engine/src/ch/queries/alerts.test.ts create mode 100644 packages/query-engine/src/ch/queries/attribute-keys.test.ts create mode 100644 packages/query-engine/src/ch/queries/errors.test.ts create mode 100644 packages/query-engine/src/ch/queries/logs.test.ts create mode 100644 packages/query-engine/src/ch/queries/metrics.test.ts create mode 100644 packages/query-engine/src/ch/queries/services.test.ts create mode 100644 packages/query-engine/src/ch/test-utils.ts diff --git a/packages/query-engine/src/ch/core-dsl.test.ts b/packages/query-engine/src/ch/core-dsl.test.ts new file mode 100644 index 0000000000..9bc14d641d --- /dev/null +++ b/packages/query-engine/src/ch/core-dsl.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "bun:test" +import * as CH from "./index" +import { compileCH, QueryBuilderError } from "./compile" + +const TestTable = CH.table("test_table", { + Id: CH.string, + Name: CH.string, + Value: CH.uint64, + Attrs: CH.map(CH.string, CH.string), + Timestamp: CH.dateTime64, + Active: CH.uint8, +}) + +// --------------------------------------------------------------------------- +// Untested expression functions +// --------------------------------------------------------------------------- + +describe("expression functions", () => { + it("compiles coalesce", () => { + const q = CH.from(TestTable) + .select(($) => ({ result: CH.coalesce(CH.nullIf($.Name, ""), CH.lit("default")) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("coalesce(nullIf(Name, ''), 'default') AS result") + }) + + it("compiles nullIf", () => { + const q = CH.from(TestTable) + .select(($) => ({ result: CH.nullIf($.Name, "") })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("nullIf(Name, '') AS result") + }) + + it("compiles multiIf", () => { + const q = CH.from(TestTable) + .select(($) => ({ + result: CH.multiIf( + [ + [$.Value.gt(100), CH.lit("high")], + [$.Value.gt(50), CH.lit("medium")], + ], + CH.lit("low"), + ), + })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("multiIf(Value > 100, 'high', Value > 50, 'medium', 'low') AS result") + }) + + it("compiles mapContains", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [CH.mapContains($.Attrs, "http.method")]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("mapContains(Attrs, 'http.method')") + }) + + it("compiles mapGet", () => { + const q = CH.from(TestTable) + .select(($) => ({ method: CH.mapGet($.Attrs, "http.method") })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("Attrs['http.method'] AS method") + }) + + it("compiles mapLiteral", () => { + const q = CH.from(TestTable) + .select(($) => ({ m: CH.mapLiteral(["key1", $.Name], ["key2", CH.lit("val")]) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("map('key1', Name, 'key2', 'val') AS m") + }) + + it("compiles empty mapLiteral", () => { + const q = CH.from(TestTable) + .select(() => ({ m: CH.mapLiteral() })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("map() AS m") + }) + + it("compiles position_", () => { + const q = CH.from(TestTable) + .select(($) => ({ pos: CH.position($.Name, "foo") })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("position(Name, 'foo') AS pos") + }) + + it("compiles left_ and length_", () => { + const q = CH.from(TestTable) + .select(($) => ({ result: CH.left($.Name, CH.length($.Name)) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("left(Name, length(Name)) AS result") + }) + + it("compiles replaceOne", () => { + const q = CH.from(TestTable) + .select(($) => ({ result: CH.replaceOne($.Name, "old", "new") })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("replaceOne(Name, 'old', 'new') AS result") + }) + + it("compiles toFloat64OrZero", () => { + const q = CH.from(TestTable) + .select(($) => ({ num: CH.toFloat64OrZero($.Name) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("toFloat64OrZero(Name) AS num") + }) + + it("compiles toString_", () => { + const q = CH.from(TestTable) + .select(($) => ({ s: CH.toString($.Value) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("toString(Value) AS s") + }) + + it("compiles intervalSub", () => { + const q = CH.from(TestTable) + .select(($) => ({ ts: CH.intervalSub($.Timestamp, 3600) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("Timestamp - INTERVAL 3600 SECOND AS ts") + }) + + it("compiles outerRef", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(() => [CH.outerRef("t.TraceId").eq("abc")]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("t.TraceId = 'abc'") + }) + + it("compiles rawCond", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(() => [CH.rawCond("x = 1")]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("x = 1") + }) + + it("compiles notLike", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Name.notLike("%test%")]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("Name NOT LIKE '%test%'") + }) + + it("compiles notIn", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Name.notIn("a", "b")]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("Name NOT IN ('a', 'b')") + }) + + it("compiles least_ and greatest_", () => { + const q = CH.from(TestTable) + .select(($) => ({ + lo: CH.least($.Value, CH.lit(100)), + hi: CH.greatest($.Value, CH.lit(0)), + })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("least(Value, 100) AS lo") + expect(sql).toContain("greatest(Value, 0) AS hi") + }) + + it("compiles toUInt64 and toInt64", () => { + const q = CH.from(TestTable) + .select(($) => ({ + u: CH.toUInt64($.Value), + i: CH.toInt64($.Value), + })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("toUInt64(Value) AS u") + expect(sql).toContain("toInt64(Value) AS i") + }) + + it("compiles positionCaseInsensitive", () => { + const q = CH.from(TestTable) + .select(($) => ({ pos: CH.positionCaseInsensitive($.Name, CH.lit("foo")) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("positionCaseInsensitive(Name, 'foo') AS pos") + }) + + it("compiles extract_", () => { + const q = CH.from(TestTable) + .select(($) => ({ result: CH.extract($.Name, "th:([0-9]+)") })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("extract(Name, 'th:([0-9]+)') AS result") + }) + + it("compiles arrayFilter", () => { + const arr = CH.arrayOf(CH.lit("a"), CH.lit(""), CH.lit("b")) + const q = CH.from(TestTable) + .select(() => ({ result: CH.arrayFilter("x -> x != ''", arr) })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("arrayFilter(x -> x != '', ['a', '', 'b']) AS result") + }) + + it("compiles arrayStringConcat with Expr array", () => { + const q = CH.from(TestTable) + .select(($) => ({ result: CH.arrayStringConcat(CH.arrayOf($.Name, CH.lit("x")), " | ") })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("arrayStringConcat([Name, 'x'], ' | ') AS result") + }) +}) + +// --------------------------------------------------------------------------- +// Condition combinators +// --------------------------------------------------------------------------- + +describe("condition combinators", () => { + it("and() combines conditions", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Name.eq("alice").and($.Value.gt(10))]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("(Name = 'alice' AND Value > 10)") + }) + + it("or() combines conditions", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Name.eq("alice").or($.Name.eq("bob"))]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("(Name = 'alice' OR Name = 'bob')") + }) + + it("chains and/or", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [ + $.Name.eq("alice").or($.Name.eq("bob")).and($.Value.gt(0)), + ]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("((Name = 'alice' OR Name = 'bob') AND Value > 0)") + }) +}) + +// --------------------------------------------------------------------------- +// Compile edge cases +// --------------------------------------------------------------------------- + +describe("compile edge cases", () => { + it("throws QueryBuilderError when no select", () => { + const q = CH.from(TestTable).format("JSON") + expect(() => compileCH(q, {})).toThrow() + }) + + it("compiles CTE with withCTE", () => { + const q = CH.from(TestTable) + .withCTE("my_cte", "SELECT 1 AS x") + .select(($) => ({ id: $.Id })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("WITH my_cte AS") + expect(sql).toContain("SELECT 1 AS x") + }) + + it("compiles INNER JOIN", () => { + const q = CH.from(TestTable) + .join("other_table", "o", CH.dynamicColumn("test_table.Id").eq(CH.dynamicColumn("o.Id")), "INNER") + .select(($) => ({ id: $.Id })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("INNER JOIN other_table AS o ON test_table.Id = o.Id") + }) + + it("compiles CROSS JOIN (no ON clause)", () => { + const q = CH.from(TestTable) + .join("other_table", "o", undefined, "CROSS") + .select(($) => ({ id: $.Id })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("CROSS JOIN other_table AS o") + expect(sql).not.toContain(" ON ") + }) + + it("compiles table alias", () => { + const q = CH.from(TestTable, "t") + .select(($) => ({ id: $.Id })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("FROM test_table AS t") + }) + + it("compiles OFFSET", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .limit(10) + .offset(5) + const { sql } = compileCH(q, {}) + expect(sql).toContain("LIMIT 10") + expect(sql).toContain("OFFSET 5") + }) +}) + +// --------------------------------------------------------------------------- +// Param resolution +// --------------------------------------------------------------------------- + +describe("param resolution", () => { + it("resolves param.string", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Id.eq(CH.param.string("orgId"))]) + const { sql } = compileCH(q, { orgId: "org_123" }) + expect(sql).toContain("Id = 'org_123'") + }) + + it("resolves param.int", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Value.gt(CH.param.int("threshold"))]) + const { sql } = compileCH(q, { threshold: 42 }) + expect(sql).toContain("Value > 42") + }) + + it("resolves param.dateTime", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Timestamp.gte(CH.param.dateTime("startTime"))]) + const { sql } = compileCH(q, { startTime: "2024-01-01 00:00:00" }) + expect(sql).toContain("Timestamp >= '2024-01-01 00:00:00'") + }) + + it("resolves boolean param", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Active.eq(CH.param.int("isActive"))]) + const { sql } = compileCH(q, { isActive: true }) + expect(sql).toContain("Active = 1") + }) + + it("leaves unresolved param placeholders", () => { + const q = CH.from(TestTable) + .select(($) => ({ id: $.Id })) + .where(($) => [$.Id.eq(CH.param.string("orgId"))]) + const { sql } = compileCH(q, {}) + expect(sql).toContain("__PARAM_orgId__") + }) +}) diff --git a/packages/query-engine/src/ch/queries/alerts.test.ts b/packages/query-engine/src/ch/queries/alerts.test.ts new file mode 100644 index 0000000000..b4ece5504b --- /dev/null +++ b/packages/query-engine/src/ch/queries/alerts.test.ts @@ -0,0 +1,210 @@ +import { describe, expect, it } from "bun:test" +import { compileCH } from "../compile" +import { + alertTracesAggregateQuery, + alertTracesAggregateByServiceQuery, + alertMetricsAggregateQuery, + alertMetricsAggregateByServiceQuery, + alertLogsAggregateQuery, + alertLogsAggregateByServiceQuery, +} from "./alerts" + +const baseParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", +} + +const metricsParams = { + ...baseParams, + metricName: "http.request.duration", +} + +// --------------------------------------------------------------------------- +// alertTracesAggregateQuery +// --------------------------------------------------------------------------- + +describe("alertTracesAggregateQuery", () => { + it("compiles basic aggregate with all metrics", () => { + const q = alertTracesAggregateQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM traces") + expect(sql).toContain("count() AS count") + expect(sql).toContain("avg(Duration) / 1000000 AS avgDuration") + expect(sql).toContain("quantile(0.5)(Duration) / 1000000 AS p50Duration") + expect(sql).toContain("quantile(0.95)(Duration) / 1000000 AS p95Duration") + expect(sql).toContain("quantile(0.99)(Duration) / 1000000 AS p99Duration") + expect(sql).toContain("errorRate") + expect(sql).toContain("apdexScore") + expect(sql).toContain("FORMAT JSON") + expect(sql).not.toContain("GROUP BY") + }) + + it("applies serviceName filter", () => { + const q = alertTracesAggregateQuery({ serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) + + it("applies rootOnly filter", () => { + const q = alertTracesAggregateQuery({ rootOnly: true }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''") + }) + + it("applies errorsOnly filter", () => { + const q = alertTracesAggregateQuery({ errorsOnly: true }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("StatusCode = 'Error'") + }) + + it("applies environment filter", () => { + const q = alertTracesAggregateQuery({ environments: ["production", "staging"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production', 'staging')") + }) + + it("applies attribute filters", () => { + const q = alertTracesAggregateQuery({ + attributeFilters: [{ key: "http.status_code", value: "500", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("SpanAttributes['http.status_code'] = '500'") + }) + + it("uses custom apdex threshold", () => { + const q = alertTracesAggregateQuery({ apdexThresholdMs: 250 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Duration / 1000000 < 250") + }) +}) + +// --------------------------------------------------------------------------- +// alertTracesAggregateByServiceQuery +// --------------------------------------------------------------------------- + +describe("alertTracesAggregateByServiceQuery", () => { + it("compiles with GROUP BY serviceName", () => { + const q = alertTracesAggregateByServiceQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("ORDER BY count DESC") + }) + + it("applies environment filter", () => { + const q = alertTracesAggregateByServiceQuery({ environments: ["prod"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("deployment.environment") + expect(sql).toContain("GROUP BY serviceName") + }) +}) + +// --------------------------------------------------------------------------- +// alertMetricsAggregateQuery +// --------------------------------------------------------------------------- + +describe("alertMetricsAggregateQuery", () => { + it("compiles value metric (sum type)", () => { + const q = alertMetricsAggregateQuery({ metricType: "sum" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("FROM metrics_sum") + expect(sql).toContain("avg(Value) AS avgValue") + expect(sql).toContain("min(Value) AS minValue") + expect(sql).toContain("max(Value) AS maxValue") + expect(sql).toContain("sum(Value) AS sumValue") + expect(sql).toContain("count() AS dataPointCount") + expect(sql).toContain("FORMAT JSON") + }) + + it("compiles value metric (gauge type)", () => { + const q = alertMetricsAggregateQuery({ metricType: "gauge" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("FROM metrics_gauge") + }) + + it("compiles histogram metric", () => { + const q = alertMetricsAggregateQuery({ metricType: "histogram" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("FROM metrics_histogram") + expect(sql).toContain("sum(Count)") + expect(sql).toContain("sum(Sum)") + expect(sql).toContain("min(Min)") + expect(sql).toContain("max(Max)") + }) + + it("compiles exponential_histogram metric", () => { + const q = alertMetricsAggregateQuery({ metricType: "exponential_histogram" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("FROM metrics_exponential_histogram") + }) + + it("applies serviceName filter", () => { + const q = alertMetricsAggregateQuery({ metricType: "sum", serviceName: "api" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("ServiceName = 'api'") + }) +}) + +// --------------------------------------------------------------------------- +// alertMetricsAggregateByServiceQuery +// --------------------------------------------------------------------------- + +describe("alertMetricsAggregateByServiceQuery", () => { + it("compiles value type with GROUP BY", () => { + const q = alertMetricsAggregateByServiceQuery({ metricType: "sum" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("ORDER BY dataPointCount DESC") + }) + + it("compiles histogram type with GROUP BY", () => { + const q = alertMetricsAggregateByServiceQuery({ metricType: "histogram" }) + const { sql } = compileCH(q, metricsParams) + expect(sql).toContain("FROM metrics_histogram") + expect(sql).toContain("GROUP BY serviceName") + }) +}) + +// --------------------------------------------------------------------------- +// alertLogsAggregateQuery +// --------------------------------------------------------------------------- + +describe("alertLogsAggregateQuery", () => { + it("compiles basic log count aggregate", () => { + const q = alertLogsAggregateQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM logs") + expect(sql).toContain("count() AS count") + expect(sql).toContain("FORMAT JSON") + expect(sql).not.toContain("GROUP BY") + }) + + it("applies optional filters", () => { + const q = alertLogsAggregateQuery({ serviceName: "api", severity: "ERROR" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("SeverityText = 'ERROR'") + }) +}) + +// --------------------------------------------------------------------------- +// alertLogsAggregateByServiceQuery +// --------------------------------------------------------------------------- + +describe("alertLogsAggregateByServiceQuery", () => { + it("compiles with GROUP BY serviceName", () => { + const q = alertLogsAggregateByServiceQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("ORDER BY count DESC") + }) + + it("applies severity filter", () => { + const q = alertLogsAggregateByServiceQuery({ severity: "WARN" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("SeverityText = 'WARN'") + }) +}) diff --git a/packages/query-engine/src/ch/queries/attribute-keys.test.ts b/packages/query-engine/src/ch/queries/attribute-keys.test.ts new file mode 100644 index 0000000000..ff9bedd17f --- /dev/null +++ b/packages/query-engine/src/ch/queries/attribute-keys.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "bun:test" +import { compileCH } from "../compile" +import { + attributeKeysQuery, + spanAttributeValuesQuery, + resourceAttributeValuesQuery, +} from "./attribute-keys" + +const baseParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", + scope: "span", +} + +// --------------------------------------------------------------------------- +// attributeKeysQuery +// --------------------------------------------------------------------------- + +describe("attributeKeysQuery", () => { + it("compiles basic attribute keys query", () => { + const q = attributeKeysQuery({ scope: "span" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM attribute_keys_hourly") + expect(sql).toContain("AttributeKey AS attributeKey") + expect(sql).toContain("sum(UsageCount) AS usageCount") + expect(sql).toContain("GROUP BY attributeKey") + expect(sql).toContain("ORDER BY usageCount DESC") + expect(sql).toContain("LIMIT 200") + expect(sql).toContain("FORMAT JSON") + }) + + it("applies custom limit", () => { + const q = attributeKeysQuery({ scope: "resource", limit: 50 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("LIMIT 50") + }) +}) + +// --------------------------------------------------------------------------- +// spanAttributeValuesQuery +// --------------------------------------------------------------------------- + +describe("spanAttributeValuesQuery", () => { + it("compiles span attribute values", () => { + const q = spanAttributeValuesQuery({ attributeKey: "http.method" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM traces") + expect(sql).toContain("SpanAttributes['http.method'] AS attributeValue") + expect(sql).toContain("count() AS usageCount") + expect(sql).toContain("SpanAttributes['http.method'] != ''") + expect(sql).toContain("GROUP BY attributeValue") + expect(sql).toContain("ORDER BY usageCount DESC") + expect(sql).toContain("LIMIT 50") + expect(sql).toContain("FORMAT JSON") + }) + + it("applies custom limit", () => { + const q = spanAttributeValuesQuery({ attributeKey: "http.method", limit: 100 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("LIMIT 100") + }) +}) + +// --------------------------------------------------------------------------- +// resourceAttributeValuesQuery +// --------------------------------------------------------------------------- + +describe("resourceAttributeValuesQuery", () => { + it("compiles resource attribute values", () => { + const q = resourceAttributeValuesQuery({ attributeKey: "host.name" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM traces") + expect(sql).toContain("ResourceAttributes['host.name'] AS attributeValue") + expect(sql).toContain("ResourceAttributes['host.name'] != ''") + expect(sql).toContain("GROUP BY attributeValue") + expect(sql).toContain("ORDER BY usageCount DESC") + }) +}) diff --git a/packages/query-engine/src/ch/queries/errors.test.ts b/packages/query-engine/src/ch/queries/errors.test.ts new file mode 100644 index 0000000000..0a490d5815 --- /dev/null +++ b/packages/query-engine/src/ch/queries/errors.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from "bun:test" +import { compileCH, compileUnion } from "../compile" +import { + errorsByTypeQuery, + errorsTimeseriesQuery, + errorsSummaryQuery, + errorDetailTracesQuery, + errorsFacetsQuery, + tracesFacetsQuery, +} from "./errors" + +const baseParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", + bucketSeconds: 3600, +} + +// --------------------------------------------------------------------------- +// errorsByTypeQuery +// --------------------------------------------------------------------------- + +describe("errorsByTypeQuery", () => { + it("compiles basic errors by type", () => { + const q = errorsByTypeQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM error_spans") + expect(sql).toContain("AS errorType") + expect(sql).toContain("count() AS count") + expect(sql).toContain("uniq(ServiceName) AS affectedServicesCount") + expect(sql).toContain("min(Timestamp) AS firstSeen") + expect(sql).toContain("max(Timestamp) AS lastSeen") + expect(sql).toContain("GROUP BY errorType") + expect(sql).toContain("ORDER BY count DESC") + expect(sql).toContain("LIMIT 50") + expect(sql).toContain("FORMAT JSON") + // Fingerprint expression should contain position/left/multiIf + expect(sql).toContain("left(") + expect(sql).toContain("multiIf(") + expect(sql).toContain("position(") + }) + + it("applies rootOnly filter", () => { + const q = errorsByTypeQuery({ rootOnly: true }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ParentSpanId = ''") + }) + + it("applies services filter", () => { + const q = errorsByTypeQuery({ services: ["api", "web"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName IN ('api', 'web')") + }) + + it("applies deploymentEnvs filter", () => { + const q = errorsByTypeQuery({ deploymentEnvs: ["production"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("DeploymentEnv IN ('production')") + }) + + it("applies errorTypes filter", () => { + const q = errorsByTypeQuery({ errorTypes: ["TimeoutError"] }) + const { sql } = compileCH(q, baseParams) + // The fingerprint expression should appear in WHERE with IN + expect(sql).toContain("IN ('TimeoutError')") + }) + + it("applies custom limit", () => { + const q = errorsByTypeQuery({ limit: 25 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("LIMIT 25") + }) +}) + +// --------------------------------------------------------------------------- +// errorsTimeseriesQuery +// --------------------------------------------------------------------------- + +describe("errorsTimeseriesQuery", () => { + it("compiles error timeseries with bucket", () => { + const q = errorsTimeseriesQuery({ errorType: "NullPointerException" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM error_spans") + expect(sql).toContain("toStartOfInterval") + expect(sql).toContain("INTERVAL 3600 SECOND") + expect(sql).toContain("count() AS count") + expect(sql).toContain("GROUP BY bucket") + expect(sql).toContain("ORDER BY bucket ASC") + // Fingerprint match in WHERE + expect(sql).toContain("NullPointerException") + }) + + it("applies services filter", () => { + const q = errorsTimeseriesQuery({ errorType: "X", services: ["api"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName IN ('api')") + }) +}) + +// --------------------------------------------------------------------------- +// errorsSummaryQuery +// --------------------------------------------------------------------------- + +describe("errorsSummaryQuery", () => { + it("compiles CROSS JOIN between error_spans and service_usage", () => { + const q = errorsSummaryQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("CROSS JOIN") + expect(sql).toContain("FROM (SELECT") + expect(sql).toContain("e.totalErrors") + expect(sql).toContain("s.totalSpans") + expect(sql).toContain("AS errorRate") + expect(sql).toContain("round(") + expect(sql).toContain("e.affectedServicesCount") + expect(sql).toContain("e.affectedTracesCount") + expect(sql).toContain("FORMAT JSON") + }) + + it("applies rootOnly and services filters", () => { + const q = errorsSummaryQuery({ rootOnly: true, services: ["api"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ParentSpanId = ''") + expect(sql).toContain("ServiceName IN ('api')") + }) + + it("applies deploymentEnvs filter", () => { + const q = errorsSummaryQuery({ deploymentEnvs: ["production"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("DeploymentEnv IN ('production')") + }) +}) + +// --------------------------------------------------------------------------- +// errorDetailTracesQuery +// --------------------------------------------------------------------------- + +describe("errorDetailTracesQuery", () => { + it("compiles INNER JOIN with error subquery", () => { + const q = errorDetailTracesQuery({ errorType: "NullPointerException" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("INNER JOIN") + expect(sql).toContain("SELECT DISTINCT TraceId") + expect(sql).toContain("FROM traces") + expect(sql).toContain("GROUP BY traceId") + expect(sql).toContain("groupUniqArray(ServiceName)") + expect(sql).toContain("ORDER BY startTime DESC") + expect(sql).toContain("FORMAT JSON") + // Error subquery references error_spans + expect(sql).toContain("FROM error_spans") + // Fingerprint match + expect(sql).toContain("NullPointerException") + }) + + it("applies rootOnly filter", () => { + const q = errorDetailTracesQuery({ errorType: "X", rootOnly: true }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ParentSpanId = ''") + }) + + it("applies services filter", () => { + const q = errorDetailTracesQuery({ errorType: "X", services: ["api", "web"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName IN ('api', 'web')") + }) + + it("applies custom limit", () => { + const q = errorDetailTracesQuery({ errorType: "X", limit: 20 }) + const { sql } = compileCH(q, baseParams) + // The limit applies to the error subquery + expect(sql).toContain("LIMIT 20") + }) +}) + +// --------------------------------------------------------------------------- +// errorsFacetsQuery +// --------------------------------------------------------------------------- + +describe("errorsFacetsQuery", () => { + it("compiles UNION ALL with 3 facet dimensions", () => { + const q = errorsFacetsQuery({}) + const { sql } = compileUnion(q, baseParams) + const unionCount = (sql.match(/UNION ALL/g) || []).length + expect(unionCount).toBe(2) // 3 queries = 2 UNION ALL + expect(sql).toContain("'service' AS facetType") + expect(sql).toContain("'environment' AS facetType") + expect(sql).toContain("'error_type' AS facetType") + }) + + it("applies all optional filters", () => { + const q = errorsFacetsQuery({ + rootOnly: true, + services: ["api"], + deploymentEnvs: ["prod"], + errorTypes: ["TimeoutError"], + }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("ParentSpanId = ''") + expect(sql).toContain("ServiceName IN ('api')") + expect(sql).toContain("DeploymentEnv IN ('prod')") + expect(sql).toContain("IN ('TimeoutError')") + }) +}) + +// --------------------------------------------------------------------------- +// tracesFacetsQuery +// --------------------------------------------------------------------------- + +describe("tracesFacetsQuery", () => { + it("compiles UNION ALL with 6 facet dimensions", () => { + const q = tracesFacetsQuery({}) + const { sql } = compileUnion(q, baseParams) + const unionCount = (sql.match(/UNION ALL/g) || []).length + expect(unionCount).toBe(5) // 6 queries = 5 UNION ALL + expect(sql).toContain("'service' AS facetType") + expect(sql).toContain("'spanName' AS facetType") + expect(sql).toContain("'httpMethod' AS facetType") + expect(sql).toContain("'httpStatus' AS facetType") + expect(sql).toContain("'deploymentEnv' AS facetType") + expect(sql).toContain("'errorCount' AS facetType") + }) + + it("applies serviceName filter", () => { + const q = tracesFacetsQuery({ serviceName: "api" }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) + + it("applies hasError filter", () => { + const q = tracesFacetsQuery({ hasError: true }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("HasError = 1") + }) + + it("applies contains match mode for serviceName", () => { + const q = tracesFacetsQuery({ + serviceName: "api", + matchModes: { serviceName: "contains" }, + }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("positionCaseInsensitive(ServiceName, 'api') > 0") + }) + + it("applies attribute filter with correlated EXISTS", () => { + const q = tracesFacetsQuery({ + attributeFilterKey: "http.method", + attributeFilterValue: "GET", + }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("EXISTS") + expect(sql).toContain("t_attr.SpanAttributes") + expect(sql).toContain("http.method") + }) + + it("applies resource filter with correlated EXISTS", () => { + const q = tracesFacetsQuery({ + resourceFilterKey: "host.name", + resourceFilterValue: "server-1", + }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("EXISTS") + expect(sql).toContain("t_res.ResourceAttributes") + expect(sql).toContain("host.name") + }) +}) diff --git a/packages/query-engine/src/ch/queries/logs.test.ts b/packages/query-engine/src/ch/queries/logs.test.ts new file mode 100644 index 0000000000..088225abb3 --- /dev/null +++ b/packages/query-engine/src/ch/queries/logs.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it } from "bun:test" +import { compileCH, compileUnion } from "../compile" +import { + logsTimeseriesQuery, + logsBreakdownQuery, + logsCountQuery, + logsListQuery, + errorRateByServiceQuery, + logsFacetsQuery, +} from "./logs" + +const baseParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", + bucketSeconds: 3600, +} + +// --------------------------------------------------------------------------- +// logsTimeseriesQuery +// --------------------------------------------------------------------------- + +describe("logsTimeseriesQuery", () => { + it("compiles basic timeseries with no groupBy", () => { + const q = logsTimeseriesQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM logs") + expect(sql).toContain("toStartOfInterval") + expect(sql).toContain("INTERVAL 3600 SECOND") + expect(sql).toContain("'all' AS groupName") + expect(sql).toContain("count() AS count") + expect(sql).toContain("GROUP BY bucket, groupName") + expect(sql).toContain("ORDER BY bucket ASC, groupName ASC") + expect(sql).toContain("FORMAT JSON") + }) + + it("groups by service", () => { + const q = logsTimeseriesQuery({ groupBy: ["service"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("toString(ServiceName)") + expect(sql).not.toContain("'all' AS groupName") + }) + + it("groups by severity", () => { + const q = logsTimeseriesQuery({ groupBy: ["severity"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("toString(SeverityText)") + }) + + it("groups by service and severity", () => { + const q = logsTimeseriesQuery({ groupBy: ["service", "severity"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("arrayFilter") + expect(sql).toContain("arrayStringConcat") + }) + + it("applies serviceName filter", () => { + const q = logsTimeseriesQuery({ serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) + + it("applies severity filter", () => { + const q = logsTimeseriesQuery({ severity: "ERROR" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("SeverityText = 'ERROR'") + }) +}) + +// --------------------------------------------------------------------------- +// logsBreakdownQuery +// --------------------------------------------------------------------------- + +describe("logsBreakdownQuery", () => { + it("compiles breakdown by service", () => { + const q = logsBreakdownQuery({ groupBy: "service" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM logs") + expect(sql).toContain("ServiceName AS name") + expect(sql).toContain("count() AS count") + expect(sql).toContain("GROUP BY name") + expect(sql).toContain("ORDER BY count DESC") + expect(sql).toContain("LIMIT 10") + expect(sql).toContain("FORMAT JSON") + }) + + it("compiles breakdown by severity", () => { + const q = logsBreakdownQuery({ groupBy: "severity" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("SeverityText AS name") + }) + + it("applies optional filters", () => { + const q = logsBreakdownQuery({ groupBy: "service", serviceName: "api", severity: "ERROR" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("SeverityText = 'ERROR'") + }) + + it("applies custom limit", () => { + const q = logsBreakdownQuery({ groupBy: "service", limit: 25 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("LIMIT 25") + }) +}) + +// --------------------------------------------------------------------------- +// logsCountQuery +// --------------------------------------------------------------------------- + +describe("logsCountQuery", () => { + it("compiles basic count", () => { + const q = logsCountQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM logs") + expect(sql).toContain("count() AS total") + expect(sql).toContain("FORMAT JSON") + expect(sql).not.toContain("GROUP BY") + expect(sql).not.toContain("ORDER BY") + }) + + it("applies search filter", () => { + const q = logsCountQuery({ search: "exception" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Body ILIKE '%exception%'") + }) + + it("applies traceId filter", () => { + const q = logsCountQuery({ traceId: "abc123" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("TraceId = 'abc123'") + }) + + it("applies all filters simultaneously", () => { + const q = logsCountQuery({ serviceName: "api", severity: "ERROR", search: "timeout" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("Body ILIKE '%timeout%'") + }) +}) + +// --------------------------------------------------------------------------- +// logsListQuery +// --------------------------------------------------------------------------- + +describe("logsListQuery", () => { + it("compiles basic list with all columns", () => { + const q = logsListQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM logs") + expect(sql).toContain("Timestamp AS timestamp") + expect(sql).toContain("SeverityText AS severityText") + expect(sql).toContain("SeverityNumber AS severityNumber") + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("Body AS body") + expect(sql).toContain("TraceId AS traceId") + expect(sql).toContain("SpanId AS spanId") + expect(sql).toContain("toJSONString(LogAttributes) AS logAttributes") + expect(sql).toContain("toJSONString(ResourceAttributes) AS resourceAttributes") + expect(sql).toContain("ORDER BY timestamp DESC") + expect(sql).toContain("LIMIT 50") + expect(sql).toContain("FORMAT JSON") + }) + + it("applies cursor pagination", () => { + const q = logsListQuery({ cursor: "2024-01-01T12:00:00" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Timestamp < '2024-01-01T12:00:00'") + }) + + it("applies custom limit", () => { + const q = logsListQuery({ limit: 100 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("LIMIT 100") + }) + + it("applies all filters simultaneously", () => { + const q = logsListQuery({ + serviceName: "api", + severity: "ERROR", + traceId: "trace123", + spanId: "span456", + search: "timeout", + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("TraceId = 'trace123'") + expect(sql).toContain("SpanId = 'span456'") + expect(sql).toContain("Body ILIKE '%timeout%'") + }) + + it("applies minSeverity filter", () => { + const q = logsListQuery({ minSeverity: 9 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("SeverityNumber >= 9") + }) +}) + +// --------------------------------------------------------------------------- +// errorRateByServiceQuery +// --------------------------------------------------------------------------- + +describe("errorRateByServiceQuery", () => { + it("compiles error rate by service", () => { + const q = errorRateByServiceQuery() + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM logs") + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("count() AS totalLogs") + expect(sql).toContain("countIf(") + expect(sql).toContain("IN ('ERROR', 'FATAL')") + expect(sql).toContain("AS errorLogs") + expect(sql).toContain("round(") + expect(sql).toContain("AS errorRatePercent") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("ORDER BY errorRatePercent DESC") + expect(sql).toContain("FORMAT JSON") + }) +}) + +// --------------------------------------------------------------------------- +// logsFacetsQuery +// --------------------------------------------------------------------------- + +describe("logsFacetsQuery", () => { + it("compiles UNION ALL with severity and service facets", () => { + const q = logsFacetsQuery({}) + const { sql } = compileUnion(q, baseParams) + const unionCount = (sql.match(/UNION ALL/g) || []).length + expect(unionCount).toBe(1) + expect(sql).toContain("'severity' AS facetType") + expect(sql).toContain("'service' AS facetType") + expect(sql).toContain("ORDER BY count DESC") + }) + + it("applies optional filters", () => { + const q = logsFacetsQuery({ serviceName: "api", severity: "ERROR" }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("SeverityText = 'ERROR'") + }) +}) diff --git a/packages/query-engine/src/ch/queries/metrics.test.ts b/packages/query-engine/src/ch/queries/metrics.test.ts new file mode 100644 index 0000000000..cdbcfdeaae --- /dev/null +++ b/packages/query-engine/src/ch/queries/metrics.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from "bun:test" +import { compileCH } from "../compile" +import { compileUnion } from "../compile" +import { + metricsTimeseriesQuery, + metricsTimeseriesRateQuery, + metricsBreakdownQuery, + listMetricsQuery, + metricsSummaryQuery, +} from "./metrics" + +const baseParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", + bucketSeconds: 3600, + metricName: "cpu.utilization", +} + +// --------------------------------------------------------------------------- +// metricsTimeseriesQuery +// --------------------------------------------------------------------------- + +describe("metricsTimeseriesQuery", () => { + it("compiles value timeseries (sum)", () => { + const q = metricsTimeseriesQuery({ metricType: "sum" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM metrics_sum") + expect(sql).toContain("avg(Value) AS avgValue") + expect(sql).toContain("min(Value) AS minValue") + expect(sql).toContain("max(Value) AS maxValue") + expect(sql).toContain("sum(Value) AS sumValue") + expect(sql).toContain("count() AS dataPointCount") + expect(sql).toContain("INTERVAL 3600 SECOND") + expect(sql).toContain("GROUP BY bucket, serviceName") + expect(sql).toContain("ORDER BY bucket ASC") + expect(sql).toContain("FORMAT JSON") + }) + + it("compiles value timeseries (gauge)", () => { + const q = metricsTimeseriesQuery({ metricType: "gauge" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM metrics_gauge") + }) + + it("compiles histogram timeseries", () => { + const q = metricsTimeseriesQuery({ metricType: "histogram" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM metrics_histogram") + expect(sql).toContain("sum(Sum) / sum(Count)") + expect(sql).toContain("min(Min) AS minValue") + expect(sql).toContain("max(Max) AS maxValue") + expect(sql).toContain("sum(Sum) AS sumValue") + expect(sql).toContain("sum(Count) AS dataPointCount") + }) + + it("compiles exponential_histogram timeseries", () => { + const q = metricsTimeseriesQuery({ metricType: "exponential_histogram" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM metrics_exponential_histogram") + }) + + it("applies groupByAttributeKey", () => { + const q = metricsTimeseriesQuery({ metricType: "sum", groupByAttributeKey: "region" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Attributes['region']") + expect(sql).toContain("GROUP BY bucket, serviceName, attributeValue") + }) + + it("applies serviceName filter", () => { + const q = metricsTimeseriesQuery({ metricType: "sum", serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) + + it("applies attribute key/value filter", () => { + const q = metricsTimeseriesQuery({ + metricType: "sum", + attributeKey: "region", + attributeValue: "us-east-1", + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Attributes['region'] = 'us-east-1'") + }) + + it("shows empty string as attributeValue when no groupByAttributeKey", () => { + const q = metricsTimeseriesQuery({ metricType: "sum" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("'' AS attributeValue") + }) +}) + +// --------------------------------------------------------------------------- +// metricsTimeseriesRateQuery +// --------------------------------------------------------------------------- + +describe("metricsTimeseriesRateQuery", () => { + it("compiles CTE-based rate query", () => { + const q = metricsTimeseriesRateQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("WITH with_deltas AS") + expect(sql).toContain("lagInFrame") + expect(sql).toContain("PARTITION BY") + expect(sql).toContain("rateValue") + expect(sql).toContain("increaseValue") + expect(sql).toContain("sumIf(") + expect(sql).toContain("FROM with_deltas") + expect(sql).toContain("FORMAT JSON") + }) + + it("applies serviceName filter in CTE", () => { + const q = metricsTimeseriesRateQuery({ serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) + + it("applies attributeKey filter", () => { + const q = metricsTimeseriesRateQuery({ + attributeKey: "region", + attributeValue: "us-east-1", + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Attributes['region'] = 'us-east-1'") + }) + + it("applies groupByAttributeKey", () => { + const q = metricsTimeseriesRateQuery({ groupByAttributeKey: "host" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Attributes['host']") + expect(sql).toContain("GROUP BY bucket, serviceName, attributeValue") + }) +}) + +// --------------------------------------------------------------------------- +// metricsBreakdownQuery +// --------------------------------------------------------------------------- + +describe("metricsBreakdownQuery", () => { + it("compiles value breakdown", () => { + const q = metricsBreakdownQuery({ metricType: "sum" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM metrics_sum") + expect(sql).toContain("ServiceName AS name") + expect(sql).toContain("avg(Value) AS avgValue") + expect(sql).toContain("GROUP BY name") + expect(sql).toContain("ORDER BY count DESC") + expect(sql).toContain("LIMIT 10") + expect(sql).toContain("FORMAT JSON") + }) + + it("compiles histogram breakdown", () => { + const q = metricsBreakdownQuery({ metricType: "histogram" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM metrics_histogram") + expect(sql).toContain("sum(Sum)") + expect(sql).toContain("sum(Count)") + }) + + it("applies custom limit", () => { + const q = metricsBreakdownQuery({ metricType: "sum", limit: 25 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("LIMIT 25") + }) +}) + +// --------------------------------------------------------------------------- +// listMetricsQuery +// --------------------------------------------------------------------------- + +describe("listMetricsQuery", () => { + it("compiles UNION ALL of 4 metric tables", () => { + const q = listMetricsQuery({}) + const { sql } = compileUnion(q, baseParams) + const unionCount = (sql.match(/UNION ALL/g) || []).length + expect(unionCount).toBe(3) // 4 queries = 3 UNION ALL + expect(sql).toContain("FROM metrics_sum") + expect(sql).toContain("FROM metrics_gauge") + expect(sql).toContain("FROM metrics_histogram") + expect(sql).toContain("FROM metrics_exponential_histogram") + expect(sql).toContain("'sum' AS metricType") + expect(sql).toContain("'gauge' AS metricType") + expect(sql).toContain("ORDER BY lastSeen DESC") + expect(sql).toContain("LIMIT 100") + }) + + it("filters by metricType", () => { + const q = listMetricsQuery({ metricType: "sum" }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("FROM metrics_sum") + expect(sql).not.toContain("UNION ALL") + expect(sql).not.toContain("FROM metrics_gauge") + }) + + it("applies search filter", () => { + const q = listMetricsQuery({ search: "http" }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("ILIKE '%http%'") + }) + + it("applies serviceName filter", () => { + const q = listMetricsQuery({ serviceName: "api" }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) + + it("applies custom limit and offset", () => { + const q = listMetricsQuery({ limit: 50, offset: 10 }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("LIMIT 50") + expect(sql).toContain("OFFSET 10") + }) +}) + +// --------------------------------------------------------------------------- +// metricsSummaryQuery +// --------------------------------------------------------------------------- + +describe("metricsSummaryQuery", () => { + it("compiles summary with 4 metric type subqueries", () => { + const q = metricsSummaryQuery() + const { sql } = compileUnion(q, baseParams) + const unionCount = (sql.match(/UNION ALL/g) || []).length + expect(unionCount).toBe(3) + expect(sql).toContain("'sum' AS metricType") + expect(sql).toContain("'gauge' AS metricType") + expect(sql).toContain("'histogram' AS metricType") + expect(sql).toContain("'exponential_histogram' AS metricType") + expect(sql).toContain("uniq(MetricName)") + expect(sql).toContain("count()") + }) + + it("applies serviceName filter", () => { + const q = metricsSummaryQuery({ serviceName: "api" }) + const { sql } = compileUnion(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) +}) diff --git a/packages/query-engine/src/ch/queries/services.test.ts b/packages/query-engine/src/ch/queries/services.test.ts new file mode 100644 index 0000000000..86465998b7 --- /dev/null +++ b/packages/query-engine/src/ch/queries/services.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "bun:test" +import { compileCH, compileUnion } from "../compile" +import { + serviceOverviewQuery, + serviceReleasesTimelineQuery, + serviceApdexTimeseriesQuery, + serviceUsageQuery, + servicesFacetsQuery, +} from "./services" + +const baseParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", + bucketSeconds: 3600, +} + +// --------------------------------------------------------------------------- +// serviceOverviewQuery +// --------------------------------------------------------------------------- + +describe("serviceOverviewQuery", () => { + it("compiles basic overview with all columns", () => { + const q = serviceOverviewQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM service_overview_spans") + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("DeploymentEnv AS environment") + expect(sql).toContain("CommitSha AS commitSha") + expect(sql).toContain("count() AS throughput") + expect(sql).toContain("countIf(StatusCode = 'Error') AS errorCount") + expect(sql).toContain("quantile(0.5)(Duration) / 1000000 AS p50LatencyMs") + expect(sql).toContain("quantile(0.95)(Duration) / 1000000 AS p95LatencyMs") + expect(sql).toContain("quantile(0.99)(Duration) / 1000000 AS p99LatencyMs") + expect(sql).toContain("GROUP BY serviceName, environment, commitSha") + expect(sql).toContain("ORDER BY throughput DESC") + expect(sql).toContain("LIMIT 100") + expect(sql).toContain("FORMAT JSON") + }) + + it("includes sampling columns", () => { + const q = serviceOverviewQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("sampledSpanCount") + expect(sql).toContain("unsampledSpanCount") + expect(sql).toContain("dominantThreshold") + expect(sql).toContain("TraceState LIKE '%th:%'") + }) + + it("applies environment filter", () => { + const q = serviceOverviewQuery({ environments: ["production"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("DeploymentEnv IN ('production')") + }) + + it("applies commitSha filter", () => { + const q = serviceOverviewQuery({ commitShas: ["abc123", "def456"] }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("CommitSha IN ('abc123', 'def456')") + }) +}) + +// --------------------------------------------------------------------------- +// serviceReleasesTimelineQuery +// --------------------------------------------------------------------------- + +describe("serviceReleasesTimelineQuery", () => { + it("compiles releases timeline", () => { + const q = serviceReleasesTimelineQuery({ serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM service_overview_spans") + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("CommitSha != ''") + expect(sql).toContain("CommitSha AS commitSha") + expect(sql).toContain("count() AS count") + expect(sql).toContain("GROUP BY bucket, commitSha") + expect(sql).toContain("ORDER BY bucket ASC") + expect(sql).toContain("LIMIT 1000") + }) +}) + +// --------------------------------------------------------------------------- +// serviceApdexTimeseriesQuery +// --------------------------------------------------------------------------- + +describe("serviceApdexTimeseriesQuery", () => { + it("compiles apdex timeseries with default threshold", () => { + const q = serviceApdexTimeseriesQuery({ serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM traces") + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("count() AS totalCount") + expect(sql).toContain("Duration / 1000000 < 500") + expect(sql).toContain("AS satisfiedCount") + expect(sql).toContain("AS toleratingCount") + expect(sql).toContain("AS apdexScore") + expect(sql).toContain("GROUP BY bucket") + expect(sql).toContain("ORDER BY bucket ASC") + // Root-only filter + expect(sql).toContain("SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''") + }) + + it("compiles with custom threshold", () => { + const q = serviceApdexTimeseriesQuery({ serviceName: "api", apdexThresholdMs: 250 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("Duration / 1000000 < 250") + // Tolerating = 4x threshold + expect(sql).toContain("Duration / 1000000 < 1000") + }) +}) + +// --------------------------------------------------------------------------- +// serviceUsageQuery +// --------------------------------------------------------------------------- + +describe("serviceUsageQuery", () => { + it("compiles basic usage query", () => { + const q = serviceUsageQuery({}) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("FROM service_usage") + expect(sql).toContain("ServiceName AS serviceName") + expect(sql).toContain("sum(LogCount) AS totalLogCount") + expect(sql).toContain("sum(LogSizeBytes) AS totalLogSizeBytes") + expect(sql).toContain("sum(TraceCount) AS totalTraceCount") + expect(sql).toContain("sum(TraceSizeBytes) AS totalTraceSizeBytes") + expect(sql).toContain("sum(SumMetricCount) AS totalSumMetricCount") + expect(sql).toContain("sum(GaugeMetricCount) AS totalGaugeMetricCount") + expect(sql).toContain("sum(HistogramMetricCount) AS totalHistogramMetricCount") + expect(sql).toContain("sum(ExpHistogramMetricCount) AS totalExpHistogramMetricCount") + expect(sql).toContain("AS totalSizeBytes") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("ORDER BY totalSizeBytes DESC") + expect(sql).toContain("FORMAT JSON") + }) + + it("applies serviceName filter", () => { + const q = serviceUsageQuery({ serviceName: "api" }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("ServiceName = 'api'") + }) +}) + +// --------------------------------------------------------------------------- +// servicesFacetsQuery +// --------------------------------------------------------------------------- + +describe("servicesFacetsQuery", () => { + it("compiles UNION ALL with environment and commit_sha facets", () => { + const q = servicesFacetsQuery() + const { sql } = compileUnion(q, baseParams) + const unionCount = (sql.match(/UNION ALL/g) || []).length + expect(unionCount).toBe(1) + expect(sql).toContain("'environment' AS facetType") + expect(sql).toContain("'commit_sha' AS facetType") + expect(sql).toContain("DeploymentEnv != ''") + expect(sql).toContain("CommitSha != ''") + expect(sql).toContain("FROM service_overview_spans") + }) +}) diff --git a/packages/query-engine/src/ch/test-utils.ts b/packages/query-engine/src/ch/test-utils.ts new file mode 100644 index 0000000000..5e3b2c31f1 --- /dev/null +++ b/packages/query-engine/src/ch/test-utils.ts @@ -0,0 +1,79 @@ +// --------------------------------------------------------------------------- +// Test Utilities for CH Query Engine +// +// Inspired by Kysely's testSql() pattern: compile queries to SQL strings and +// provide both fragment-based and exact-match assertion helpers. +// --------------------------------------------------------------------------- + +import { expect } from "bun:test" +import { compileCH, compileUnion, type CompiledQuery } from "./compile" +import type { CHQuery } from "./query" +import type { CHUnionQuery } from "./union" + +/** + * Normalize SQL whitespace for stable exact-match comparison. + * Collapses all runs of whitespace (spaces, newlines, tabs) into single spaces. + */ +function normalizeSQL(sql: string): string { + return sql.replace(/\s+/g, " ").trim() +} + +/** + * Compile a CHQuery and return assertion helpers. + * + * Usage: + * const t = testSQL(myQuery, params) + * t.toContainSQL("FROM traces") // fragment check + * t.toMatchSQL("SELECT ... FROM traces") // exact normalized match + * t.sql // raw SQL for ad-hoc checks + */ +export function testSQL>( + query: CHQuery, + params: Record, + options?: { skipFormat?: boolean }, +) { + const compiled = compileCH(query, params, options) + const normalized = normalizeSQL(compiled.sql) + + return { + sql: compiled.sql, + normalizedSQL: normalized, + toContainSQL(fragment: string) { + expect(compiled.sql).toContain(fragment) + }, + toMatchSQL(expected: string) { + expect(normalized).toBe(normalizeSQL(expected)) + }, + toNotContainSQL(fragment: string) { + expect(compiled.sql).not.toContain(fragment) + }, + } +} + +/** + * Same as testSQL but for UNION ALL queries. + */ +export function testUnionSQL>( + query: CHUnionQuery, + params: Record, +) { + const compiled = compileUnion(query, params) + const normalized = normalizeSQL(compiled.sql) + + return { + sql: compiled.sql, + normalizedSQL: normalized, + toContainSQL(fragment: string) { + expect(compiled.sql).toContain(fragment) + }, + toMatchSQL(expected: string) { + expect(normalized).toBe(normalizeSQL(expected)) + }, + toNotContainSQL(fragment: string) { + expect(compiled.sql).not.toContain(fragment) + }, + unionCount() { + return (compiled.sql.match(/UNION ALL/g) || []).length + }, + } +} From 89594337e1978bc9179ce7804ba798e6376f6a43 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 13:36:29 +0200 Subject: [PATCH 08/11] fix --- packages/query-engine/src/ch/ch.test.ts | 176 ++++++- packages/query-engine/src/ch/compile.ts | 62 ++- packages/query-engine/src/ch/core-dsl.test.ts | 11 +- packages/query-engine/src/ch/index.ts | 7 +- .../query-engine/src/ch/queries/alerts.ts | 6 +- .../src/ch/queries/errors.test.ts | 4 +- .../query-engine/src/ch/queries/errors.ts | 112 ++--- .../query-engine/src/ch/queries/metrics.ts | 6 +- .../query-engine/src/ch/queries/traces.ts | 120 ++--- packages/query-engine/src/ch/query.ts | 320 +++++++++--- packages/query-engine/src/ch/types.ts | 10 + packages/query-engine/src/ch/union.ts | 4 +- packages/query-engine/src/index.ts | 1 - packages/query-engine/src/sql/clickhouse.ts | 88 ---- packages/query-engine/src/sql/index.ts | 11 - packages/query-engine/src/traces-shared.ts | 57 +-- .../src/traces-sql-builder.test.ts | 354 -------------- .../query-engine/src/traces-sql-builder.ts | 461 ------------------ 18 files changed, 584 insertions(+), 1226 deletions(-) delete mode 100644 packages/query-engine/src/sql/clickhouse.ts delete mode 100644 packages/query-engine/src/traces-sql-builder.test.ts delete mode 100644 packages/query-engine/src/traces-sql-builder.ts diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index e3fd7fb343..ec4fb29828 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -620,7 +620,7 @@ describe("subquery support", () => { Value: CH.uint64, }) - it("compiles fromSubquery", () => { + it("compiles inSubquery", () => { const innerSql = compileCH( CH.from(TestTable).select(($) => ({ id: $.Id })).where(($) => [$.Value.gt(100)]), {}, @@ -648,17 +648,15 @@ describe("subquery support", () => { expect(sql).toContain("EXISTS (SELECT 1 FROM other WHERE other.Id = test_table.Id)") }) - it("compiles fromSubquery as FROM source", () => { - const innerSql = compileCH( - CH.from(TestTable).select(($) => ({ id: $.Id, name: $.Name })).limit(10), - {}, - { skipFormat: true }, - ).sql + it("compiles fromQuery as typed FROM source", () => { + const inner = CH.from(TestTable) + .select(($) => ({ id: $.Id, name: $.Name })) + .limit(10) - const outer = CH.fromSubquery(innerSql, "sub") - .select(() => ({ - id: CH.dynamicColumn("id"), - name: CH.dynamicColumn("name"), + const outer = CH.fromQuery(inner, "sub") + .select(($) => ({ + id: $.id, + name: $.name, })) .format("JSON") @@ -666,6 +664,162 @@ describe("subquery support", () => { expect(sql).toContain("FROM (SELECT") expect(sql).toContain(") AS sub") expect(sql).toContain("LIMIT 10") + expect(sql).toContain("id AS id") + expect(sql).toContain("name AS name") + }) +}) + +// --------------------------------------------------------------------------- +// Type-safe joins +// --------------------------------------------------------------------------- + +describe("type-safe joins", () => { + const Users = CH.table("users", { + Id: CH.string, + Name: CH.string, + OrgId: CH.string, + }) + + const Orders = CH.table("orders", { + Id: CH.string, + UserId: CH.string, + Amount: CH.uint64, + Status: CH.string, + }) + + it("compiles innerJoin with Table", () => { + const q = CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + .select(($) => ({ + userName: $.Name, + orderAmount: $.o.Amount, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("INNER JOIN orders AS o ON users.Id = o.UserId") + expect(sql).toContain("users.Name AS userName") + expect(sql).toContain("o.Amount AS orderAmount") + }) + + it("compiles leftJoin with Table", () => { + const q = CH.from(Users) + .leftJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + .select(($) => ({ + userName: $.Name, + orderAmount: $.o.Amount, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("LEFT JOIN orders AS o ON users.Id = o.UserId") + }) + + it("compiles crossJoin with Table", () => { + const q = CH.from(Users) + .crossJoin(Orders, "o") + .select(($) => ({ + userName: $.Name, + orderStatus: $.o.Status, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("CROSS JOIN orders AS o") + expect(sql).not.toContain(" ON ") + }) + + it("compiles innerJoinQuery with subquery", () => { + const ordersSub = CH.from(Orders) + .select(($) => ({ userId: $.UserId, total: CH.sum($.Amount) })) + .groupBy("userId") + + const q = CH.from(Users) + .innerJoinQuery(ordersSub, "o", (u, o) => u.Id.eq(o.userId)) + .select(($) => ({ + userName: $.Name, + orderTotal: $.o.total, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("INNER JOIN (SELECT") + expect(sql).toContain(") AS o ON users.Id = o.userId") + expect(sql).toContain("o.total AS orderTotal") + }) + + it("compiles crossJoinQuery with subquery", () => { + const statsSub = CH.from(Orders) + .select(() => ({ totalOrders: CH.count() })) + + const q = CH.from(Users) + .crossJoinQuery(statsSub, "s") + .select(($) => ({ + userName: $.Name, + totalOrders: $.s.totalOrders, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("CROSS JOIN (SELECT") + expect(sql).toContain(") AS s") + expect(sql).toContain("s.totalOrders AS totalOrders") + }) + + it("compiles fromQuery + crossJoinQuery (two subqueries)", () => { + const sub1 = CH.from(Users) + .select(() => ({ userCount: CH.count() })) + + const sub2 = CH.from(Orders) + .select(() => ({ orderCount: CH.count() })) + + const q = CH.fromQuery(sub1, "u") + .crossJoinQuery(sub2, "o") + .select(($) => ({ + users: $.userCount, + orders: $.o.orderCount, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("FROM (SELECT") + expect(sql).toContain(") AS u") + expect(sql).toContain("CROSS JOIN (SELECT") + expect(sql).toContain(") AS o") + expect(sql).toContain("u.userCount AS users") + expect(sql).toContain("o.orderCount AS orders") + }) + + it("compiles multiple chained joins", () => { + const Tags = CH.table("tags", { Id: CH.string, UserId: CH.string, Label: CH.string }) + + const q = CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + .innerJoin(Tags, "t", (u, t) => u.Id.eq(t.UserId)) + .select(($) => ({ + userName: $.Name, + orderAmount: $.o.Amount, + tag: $.t.Label, + })) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("INNER JOIN orders AS o ON users.Id = o.UserId") + expect(sql).toContain("INNER JOIN tags AS t ON users.Id = t.UserId") + expect(sql).toContain("users.Name AS userName") + expect(sql).toContain("o.Amount AS orderAmount") + expect(sql).toContain("t.Label AS tag") + }) + + it("qualifies main table columns in where() with joins", () => { + const q = CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + .select(($) => ({ userName: $.Name })) + .where(($) => [$.OrgId.eq("org_1")]) + .format("JSON") + + const { sql } = compileCH(q, {}) + expect(sql).toContain("users.OrgId = 'org_1'") }) }) diff --git a/packages/query-engine/src/ch/compile.ts b/packages/query-engine/src/ch/compile.ts index 559295279b..a9699c667a 100644 --- a/packages/query-engine/src/ch/compile.ts +++ b/packages/query-engine/src/ch/compile.ts @@ -2,7 +2,7 @@ // Query Compilation // // Compiles a CHQuery + params into a SQL string by: -// 1. Creating a ColumnAccessor proxy for the table +// 1. Creating a ColumnAccessor proxy for the table (+ joined tables) // 2. Evaluating the selectFn to get aliased SqlFragments // 3. Evaluating the whereFn (with params resolved) to get Conditions // 4. Assembling into SqlQuery and calling the existing compileQuery() @@ -11,7 +11,7 @@ import type { ColumnDefs } from "./types" import type { CHQuery } from "./query" import type { CHUnionQuery } from "./union" -import { createColumnAccessor } from "./query" +import { createColumnAccessor, createJoinedColumnAccessor } from "./query" import { aliased } from "./expr" import { raw, ident, escapeClickHouseString, compile as compileSqlFragment } from "../sql/sql-fragment" import { compileQuery, type SqlQuery } from "../sql/sql-query" @@ -45,14 +45,25 @@ export interface CompiledQuery { export function compileCH< Cols extends ColumnDefs, Output extends Record, + Joins extends Record, Params extends Record, >( - query: CHQuery, + query: CHQuery, params: Params, options?: { skipFormat?: boolean }, ): CompiledQuery { const state = query._state - const $ = createColumnAccessor(state.columns) + + // Build column accessor — joined or simple depending on joins + const joinAliases = state.typedJoins.map((j) => j.alias) + const hasJoins = joinAliases.length > 0 + const mainAlias = hasJoins + ? (state.tableAlias ?? state.fromQueryAlias ?? state.tableName) + : undefined + + const $ = hasJoins + ? createJoinedColumnAccessor(state.columns, joinAliases, mainAlias) + : createColumnAccessor(state.columns) // SELECT const selectExprs = state.selectFn ? state.selectFn($) : {} @@ -70,21 +81,38 @@ export function compileCH< .filter((c): c is NonNullable => c != null) .map((c) => c.toFragment()) - // Resolve param placeholders in the compiled SQL - const fromFragment = state.fromSubquerySql - ? raw(`(${state.fromSubquerySql}) AS ${state.fromSubqueryAlias}`) - : state.tableAlias - ? raw(`${state.tableName} AS ${state.tableAlias}`) - : ident(state.tableName) + // FROM clause + let fromFragment + if (state.fromQuery) { + // Compile the inner query lazily + const innerCompiled = compileCH(state.fromQuery, params, { skipFormat: true }) + fromFragment = raw(`(${innerCompiled.sql}) AS ${state.fromQueryAlias}`) + } else if (state.tableAlias) { + fromFragment = raw(`${state.tableName} AS ${state.tableAlias}`) + } else { + fromFragment = ident(state.tableName) + } // JOINs - const joins = state.joins.length > 0 - ? state.joins.map((j) => ({ - type: j.type, - table: j.tableSql, - alias: j.alias, - on: j.on ? compileSqlFragment(j.on.toFragment()) : undefined, - })) + const joins = state.typedJoins.length > 0 + ? state.typedJoins.map((j) => { + let tableSql: string + if (j.innerQuery) { + const compiled = compileCH(j.innerQuery, params, { skipFormat: true }) + tableSql = `(${compiled.sql})` + } else if (j.tableName) { + tableSql = j.tableName + } else { + throw new QueryBuilderError({ code: "SelectRequired", message: "TypedJoin: missing table or query" }) + } + + return { + type: j.type, + table: tableSql, + alias: j.alias, + on: j.on ? compileSqlFragment(j.on.toFragment()) : undefined, + } + }) : undefined const sqlQuery: SqlQuery = { diff --git a/packages/query-engine/src/ch/core-dsl.test.ts b/packages/query-engine/src/ch/core-dsl.test.ts index 9bc14d641d..a6b529c4c7 100644 --- a/packages/query-engine/src/ch/core-dsl.test.ts +++ b/packages/query-engine/src/ch/core-dsl.test.ts @@ -252,17 +252,20 @@ describe("compile edge cases", () => { }) it("compiles INNER JOIN", () => { + const OtherTable = CH.table("other_table", { Id: CH.string, Score: CH.uint64 }) const q = CH.from(TestTable) - .join("other_table", "o", CH.dynamicColumn("test_table.Id").eq(CH.dynamicColumn("o.Id")), "INNER") - .select(($) => ({ id: $.Id })) + .innerJoin(OtherTable, "o", (main, o) => main.Id.eq(o.Id)) + .select(($) => ({ id: $.Id, score: $.o.Score })) const { sql } = compileCH(q, {}) expect(sql).toContain("INNER JOIN other_table AS o ON test_table.Id = o.Id") + expect(sql).toContain("o.Score AS score") }) it("compiles CROSS JOIN (no ON clause)", () => { + const OtherTable = CH.table("other_table", { Id: CH.string, Score: CH.uint64 }) const q = CH.from(TestTable) - .join("other_table", "o", undefined, "CROSS") - .select(($) => ({ id: $.Id })) + .crossJoin(OtherTable, "o") + .select(($) => ({ id: $.Id, score: $.o.Score })) const { sql } = compileCH(q, {}) expect(sql).toContain("CROSS JOIN other_table AS o") expect(sql).not.toContain(" ON ") diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 459928f823..c33bbca021 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -16,6 +16,8 @@ export { type CHNullable, type InferTS, type ColumnDefs, + type OutputToColumnDefs, + type NullableColumnDefs, string, uint8, uint64, @@ -69,7 +71,6 @@ export { toUInt16OrZero, rawExpr, rawCond, - dynamicColumn, when, whenTrue, arrayOf, @@ -95,10 +96,12 @@ export { param, type ParamMarker } from "./param" export { type CHQuery, type ColumnAccessor, + type JoinedColumnAccessor, + type JoinOnCallback, type InferOutput, type InferQueryOutput, from, - fromSubquery, + fromQuery, } from "./query" // Compilation diff --git a/packages/query-engine/src/ch/queries/alerts.ts b/packages/query-engine/src/ch/queries/alerts.ts index e56fac3f1d..1d5ac9f7f7 100644 --- a/packages/query-engine/src/ch/queries/alerts.ts +++ b/packages/query-engine/src/ch/queries/alerts.ts @@ -18,7 +18,7 @@ import { MetricsHistogram, MetricsExpHistogram, } from "../tables" -import { buildAttrFilterCondition, TRACE_LIST_MV_ATTR_MAP, TRACE_LIST_MV_RESOURCE_MAP } from "../../traces-shared" +import { buildAttrFilterCondition } from "../../traces-shared" // --------------------------------------------------------------------------- // Traces alert aggregate @@ -101,12 +101,12 @@ function alertTracesWhereConditions( } if (opts.attributeFilters) { for (const af of opts.attributeFilters) { - conditions.push(buildAttrFilterCondition(af, false, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP)) + conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) } } if (opts.resourceAttributeFilters) { for (const rf of opts.resourceAttributeFilters) { - conditions.push(buildAttrFilterCondition(rf, false, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP)) + conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) } } diff --git a/packages/query-engine/src/ch/queries/errors.test.ts b/packages/query-engine/src/ch/queries/errors.test.ts index 0a490d5815..34d577ba6e 100644 --- a/packages/query-engine/src/ch/queries/errors.test.ts +++ b/packages/query-engine/src/ch/queries/errors.test.ts @@ -139,10 +139,10 @@ describe("errorDetailTracesQuery", () => { const q = errorDetailTracesQuery({ errorType: "NullPointerException" }) const { sql } = compileCH(q, baseParams) expect(sql).toContain("INNER JOIN") - expect(sql).toContain("SELECT DISTINCT TraceId") + expect(sql).toContain("GROUP BY TraceId") expect(sql).toContain("FROM traces") expect(sql).toContain("GROUP BY traceId") - expect(sql).toContain("groupUniqArray(ServiceName)") + expect(sql).toContain("groupUniqArray(traces.ServiceName)") expect(sql).toContain("ORDER BY startTime DESC") expect(sql).toContain("FORMAT JSON") // Error subquery references error_spans diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index e48757707f..d71417f357 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -6,7 +6,7 @@ import * as CH from "../expr" import { param } from "../param" -import { from, fromSubquery, type ColumnAccessor } from "../query" +import { from, fromQuery, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" import { compileCH } from "../compile" import { ErrorSpans, ServiceUsage, TraceListMv, Traces } from "../tables" @@ -511,55 +511,44 @@ export interface ErrorsSummaryOutput { export function errorsSummaryQuery( opts: ErrorsSummaryOpts, ) { - // Build the error subquery - const errorSubquery = compileCH( - from(ErrorSpans) - .select(($) => ({ - totalErrors: CH.count(), - affectedServicesCount: CH.uniq($.ServiceName), - affectedTracesCount: CH.uniq($.TraceId), - })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTime("startTime")), - $.Timestamp.lte(param.dateTime("endTime")), - CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), - opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, - opts.deploymentEnvs?.length ? CH.inList($.DeploymentEnv, opts.deploymentEnvs) : undefined, - opts.errorTypes?.length ? CH.inList(errorFingerprint($.StatusMessage), opts.errorTypes) : undefined, - ]), - {}, // params resolved later by outer compile - { skipFormat: true }, - ) + const errorSub = from(ErrorSpans) + .select(($) => ({ + totalErrors: CH.count(), + affectedServicesCount: CH.uniq($.ServiceName), + affectedTracesCount: CH.uniq($.TraceId), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), + opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, + opts.deploymentEnvs?.length ? CH.inList($.DeploymentEnv, opts.deploymentEnvs) : undefined, + opts.errorTypes?.length ? CH.inList(errorFingerprint($.StatusMessage), opts.errorTypes) : undefined, + ]) - // Build the service usage subquery - const usageSubquery = compileCH( - from(ServiceUsage) - .select(($) => ({ - totalSpans: CH.sum($.TraceCount), - })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - $.Hour.gte(param.dateTime("startTime")), - $.Hour.lte(param.dateTime("endTime")), - ]), - {}, - { skipFormat: true }, - ) + const usageSub = from(ServiceUsage) + .select(($) => ({ + totalSpans: CH.sum($.TraceCount), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Hour.gte(param.dateTime("startTime")), + $.Hour.lte(param.dateTime("endTime")), + ]) - // Outer query: FROM error subquery, CROSS JOIN usage subquery - return fromSubquery(errorSubquery.sql, "e") - .join(`(${usageSubquery.sql})`, "s", undefined, "CROSS") - .select(() => ({ - totalErrors: CH.dynamicColumn("e.totalErrors"), - totalSpans: CH.dynamicColumn("s.totalSpans"), + return fromQuery(errorSub, "e") + .crossJoinQuery(usageSub, "s") + .select(($) => ({ + totalErrors: $.totalErrors, + totalSpans: $.s.totalSpans, errorRate: CH.if_( - CH.dynamicColumn("s.totalSpans").gt(0), - CH.round_(CH.dynamicColumn("e.totalErrors").div(CH.dynamicColumn("s.totalSpans")).mul(100), 4), + $.s.totalSpans.gt(0), + CH.round_($.totalErrors.div($.s.totalSpans).mul(100), 4), CH.lit(0), ), - affectedServicesCount: CH.dynamicColumn("e.affectedServicesCount"), - affectedTracesCount: CH.dynamicColumn("e.affectedTracesCount"), + affectedServicesCount: $.affectedServicesCount, + affectedTracesCount: $.affectedTracesCount, })) .format("JSON") } @@ -590,27 +579,24 @@ export function errorDetailTracesQuery( ) { const limit = opts.limit ?? 10 - // Subquery: find matching error TraceIds - const errorSubquery = compileCH( - from(ErrorSpans) - .select(($) => ({ TraceId: $.TraceId })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - errorFingerprint($.StatusMessage).eq(opts.errorType), - $.Timestamp.gte(param.dateTime("startTime")), - $.Timestamp.lte(param.dateTime("endTime")), - CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), - opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, - ]) - .orderBy(["TraceId", "desc"]) - .limit(limit), - {}, - { skipFormat: true }, - ) + // Subquery: find distinct matching error TraceIds + const errorSub = from(ErrorSpans) + .select(($) => ({ TraceId: $.TraceId })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + errorFingerprint($.StatusMessage).eq(opts.errorType), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), + opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, + ]) + .groupBy("TraceId") + .orderBy(["TraceId", "desc"]) + .limit(limit) // Outer query: join traces with error subquery return from(Traces) - .join(`(SELECT DISTINCT TraceId FROM (${errorSubquery.sql}))`, "e", CH.dynamicColumn("traces.TraceId").eq(CH.dynamicColumn("e.TraceId")), "INNER") + .innerJoinQuery(errorSub, "e", (main, e) => main.TraceId.eq(e.TraceId)) .select(($) => ({ traceId: $.TraceId, startTime: CH.min_($.Timestamp), diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index e44e3398e7..c67503a66e 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -221,10 +221,10 @@ export function metricsTimeseriesRateQuery( ? $.Attributes.get(opts.groupByAttributeKey) : CH.lit(""), rateValue: CH.sumIf( - CH.dynamicColumn("delta").div(CH.dynamicColumn("time_delta")), - CH.dynamicColumn("delta").gte(0).and(CH.dynamicColumn("time_delta").gt(0)), + $.delta.div($.time_delta), + $.delta.gte(0).and($.time_delta.gt(0)), ), - increaseValue: CH.sumIf(CH.dynamicColumn("delta"), CH.dynamicColumn("delta").gte(0)), + increaseValue: CH.sumIf($.delta, $.delta.gte(0)), dataPointCount: CH.count(), })) .where(($) => [ diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index 2845eb116c..c46afbb7c1 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -8,12 +8,9 @@ import type { TracesMetric, AttributeFilter } from "../../query-engine" import * as CH from "../expr" import { param } from "../param" import { from, type ColumnAccessor } from "../query" -import { Traces, TraceListMv } from "../tables" +import { Traces } from "../tables" import { METRIC_NEEDS, - TRACE_LIST_MV_ATTR_MAP, - TRACE_LIST_MV_RESOURCE_MAP, - canUseTraceListMv, buildAttrFilterCondition, } from "../../traces-shared" @@ -80,8 +77,6 @@ function metricSelectExprs( } } -// trace_list_mv constants + canUseTraceListMv imported from traces-shared.ts - // --------------------------------------------------------------------------- // GROUP BY expression builder // --------------------------------------------------------------------------- @@ -90,7 +85,6 @@ function buildGroupNameExpr( $: ColumnAccessor, groupBy: readonly string[] | undefined, groupByAttributeKeys: readonly string[] | undefined, - useTraceListMv: boolean, ): CH.Expr { if (!groupBy || groupBy.length === 0) { return CH.lit("all") @@ -109,20 +103,13 @@ function buildGroupNameExpr( parts.push(CH.toString_($.StatusCode)) break case "http_method": - if (useTraceListMv) { - parts.push(CH.toString_(CH.dynamicColumn("HttpMethod"))) - } else { - parts.push(CH.toString_($.SpanAttributes.get("http.method"))) - } + parts.push(CH.toString_($.SpanAttributes.get("http.method"))) break case "attribute": if (groupByAttributeKeys?.length) { - const keys: CH.Expr[] = groupByAttributeKeys.map((k) => { - const mvCol = useTraceListMv ? TRACE_LIST_MV_ATTR_MAP[k] : undefined - return mvCol - ? CH.toString_(CH.dynamicColumn(mvCol)) - : CH.toString_($.SpanAttributes.get(k)) - }) + const keys: CH.Expr[] = groupByAttributeKeys.map((k) => + CH.toString_($.SpanAttributes.get(k)), + ) // When multiple attribute keys, join them into a single part if (keys.length === 1) { parts.push(keys[0]) @@ -182,7 +169,6 @@ function buildBreakdownGroupExpr( function buildWhereConditions( $: ColumnAccessor, opts: TracesQueryOpts, - useTraceListMv: boolean, ): Array { const mm = opts.matchModes const conditions: Array = [ @@ -199,12 +185,12 @@ function buildWhereConditions( ? CH.positionCaseInsensitive($.SpanName, CH.lit(v)).gt(0) : $.SpanName.eq(v), ), - CH.whenTrue(!!opts.rootOnly && !useTraceListMv, () => + CH.whenTrue(!!opts.rootOnly, () => $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), ), ] - // Duration filters (Duration column is nanoseconds in both MV and raw table) + // Duration filters (Duration column is nanoseconds) if (opts.minDurationMs != null) { conditions.push($.Duration.gte(opts.minDurationMs * 1000000)) } @@ -213,21 +199,12 @@ function buildWhereConditions( } if (opts.errorsOnly) { - if (useTraceListMv) { - conditions.push(CH.dynamicColumn("HasError").eq(1)) - } else { - conditions.push($.StatusCode.eq("Error")) - } + conditions.push($.StatusCode.eq("Error")) } if (opts.environments?.length) { if (mm?.deploymentEnv === "contains" && opts.environments.length === 1) { - const envExpr = useTraceListMv - ? CH.dynamicColumn("DeploymentEnv") - : $.ResourceAttributes.get("deployment.environment") - conditions.push(CH.positionCaseInsensitive(envExpr, CH.lit(opts.environments[0])).gt(0)) - } else if (useTraceListMv) { - conditions.push(CH.inList(CH.dynamicColumn("DeploymentEnv"), opts.environments)) + conditions.push(CH.positionCaseInsensitive($.ResourceAttributes.get("deployment.environment"), CH.lit(opts.environments[0])).gt(0)) } else { conditions.push(CH.inList($.ResourceAttributes.get("deployment.environment"), opts.environments)) } @@ -239,13 +216,13 @@ function buildWhereConditions( if (opts.attributeFilters) { for (const af of opts.attributeFilters) { - conditions.push(buildAttrFilterCondition(af, useTraceListMv, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP)) + conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) } } if (opts.resourceAttributeFilters) { for (const rf of opts.resourceAttributeFilters) { - conditions.push(buildAttrFilterCondition(rf, useTraceListMv, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP)) + conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) } } @@ -311,16 +288,14 @@ export function tracesTimeseriesQuery( opts: TracesTimeseriesOpts, ) { const apdexThresholdMs = opts.apdexThresholdMs ?? 500 - const useTraceListMv = canUseTraceListMv(opts) - const tbl = useTraceListMv ? TraceListMv : Traces - return from(tbl as typeof Traces) + return from(Traces) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), - groupName: buildGroupNameExpr($, opts.groupBy, opts.groupByAttributeKeys, useTraceListMv), + groupName: buildGroupNameExpr($, opts.groupBy, opts.groupByAttributeKeys), ...metricSelectExprs($, opts.metric, apdexThresholdMs, opts.needsSampling, opts.allMetrics), })) - .where(($) => buildWhereConditions($, opts, useTraceListMv)) + .where(($) => buildWhereConditions($, opts)) .groupBy("bucket", "groupName") .orderBy(["bucket", "asc"], ["groupName", "asc"]) .format("JSON") @@ -358,14 +333,8 @@ export function tracesBreakdownQuery( ) { const apdexThresholdMs = opts.apdexThresholdMs ?? 500 const limit = opts.limit ?? 10 - const useTraceListMv = canUseTraceListMv({ - ...opts, - groupBy: [opts.groupBy], - groupByAttributeKeys: opts.groupByAttributeKey ? [opts.groupByAttributeKey] : undefined, - }) - const tbl = useTraceListMv ? TraceListMv : Traces - return from(tbl as typeof Traces) + return from(Traces) .select(($) => { const { sampledSpanCount, unsampledSpanCount, dominantThreshold, ...metrics } = metricSelectExprs($, opts.metric, apdexThresholdMs, false, opts.allMetrics) @@ -374,7 +343,7 @@ export function tracesBreakdownQuery( ...metrics, } }) - .where(($) => buildWhereConditions($, opts, useTraceListMv)) + .where(($) => buildWhereConditions($, opts)) .groupBy("name") .orderBy(["count", "desc"]) .limit(limit) @@ -406,21 +375,15 @@ export interface TracesListOutput { } /** - * Build a ClickHouse map() literal that extracts only the requested attribute - * keys, using MV columns when available and falling back to the raw Map. + * Build a ClickHouse map() literal that extracts only the requested attribute keys. */ function buildProjectedMapExpr( requestedKeys: string[], - useMv: boolean, mapName: "SpanAttributes" | "ResourceAttributes", - mvMap: Record, ): CH.Expr> { if (requestedKeys.length === 0) return CH.mapLiteral() const pairs: Array<[string, CH.Expr]> = requestedKeys.map((key) => { - const mvCol = useMv ? mvMap[key] : undefined - const valueExpr: CH.Expr = mvCol - ? CH.dynamicColumn(mvCol) - : CH.mapGet(CH.dynamicColumn>(mapName), key) + const valueExpr: CH.Expr = CH.mapGet(CH.dynamicColumn>(mapName), key) return [key, valueExpr] }) return CH.mapLiteral(...pairs) @@ -431,13 +394,11 @@ export function tracesListQuery( ) { const limit = opts.limit ?? 25 const offset = opts.offset ?? 0 - const useTraceListMv = canUseTraceListMv({ ...opts, rootOnly: opts.rootOnly }) - const tbl = useTraceListMv ? TraceListMv : Traces // Parse requested columns to determine which attribute keys are needed const requestedSpanAttrKeys: string[] = [] const requestedResourceAttrKeys: string[] = [] - let needsFullMaps = !opts.columns // backward-compatible when columns not specified + let needsFullMaps = !opts.columns if (opts.columns) { for (const col of opts.columns) { @@ -449,31 +410,28 @@ export function tracesListQuery( } } - // When using the raw table and no specific attr columns are requested, skip the full maps - const spanAttrExpr = needsFullMaps && !useTraceListMv + const spanAttrExpr = needsFullMaps ? undefined // use $.SpanAttributes directly - : buildProjectedMapExpr(requestedSpanAttrKeys, useTraceListMv, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP) - const resourceAttrExpr = needsFullMaps && !useTraceListMv + : buildProjectedMapExpr(requestedSpanAttrKeys, "SpanAttributes") + const resourceAttrExpr = needsFullMaps ? undefined // use $.ResourceAttributes directly - : buildProjectedMapExpr(requestedResourceAttrKeys, useTraceListMv, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP) + : buildProjectedMapExpr(requestedResourceAttrKeys, "ResourceAttributes") - let q = from(tbl as typeof Traces) + let q = from(Traces) .select(($) => ({ traceId: $.TraceId, timestamp: $.Timestamp, - spanId: useTraceListMv ? CH.lit("") : $.SpanId, + spanId: $.SpanId, serviceName: $.ServiceName, spanName: $.SpanName, durationMs: $.Duration.div(1000000), statusCode: $.StatusCode, - spanKind: useTraceListMv ? CH.dynamicColumn("SpanKind") : $.SpanKind, - hasError: useTraceListMv - ? CH.dynamicColumn("HasError") - : CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), + spanKind: $.SpanKind, + hasError: CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), spanAttributes: spanAttrExpr ?? $.SpanAttributes, resourceAttributes: resourceAttrExpr ?? $.ResourceAttributes, })) - .where(($) => buildWhereConditions($, opts, useTraceListMv)) + .where(($) => buildWhereConditions($, opts)) .orderBy(["timestamp", "desc"]) .limit(limit) .format("JSON") @@ -515,10 +473,8 @@ export function tracesRootListQuery( ) { const limit = opts.limit ?? 25 const offset = opts.offset ?? 0 - const useTraceListMv = canUseTraceListMv({ ...opts, rootOnly: true }) - const tbl = useTraceListMv ? TraceListMv : Traces - let q = from(tbl as typeof Traces) + let q = from(Traces) .select(($) => ({ traceId: $.TraceId, startTime: $.Timestamp, @@ -529,20 +485,12 @@ export function tracesRootListQuery( rootSpanName: $.SpanName, rootSpanKind: $.SpanKind, rootSpanStatusCode: $.StatusCode, - rootHttpMethod: useTraceListMv - ? CH.dynamicColumn("HttpMethod") - : $.SpanAttributes.get("http.method"), - rootHttpRoute: useTraceListMv - ? CH.dynamicColumn("HttpRoute") - : $.SpanAttributes.get("http.route"), - rootHttpStatusCode: useTraceListMv - ? CH.dynamicColumn("HttpStatusCode") - : $.SpanAttributes.get("http.status_code"), - hasError: useTraceListMv - ? CH.dynamicColumn("HasError") - : CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), + rootHttpMethod: $.SpanAttributes.get("http.method"), + rootHttpRoute: $.SpanAttributes.get("http.route"), + rootHttpStatusCode: $.SpanAttributes.get("http.status_code"), + hasError: CH.if_($.StatusCode.eq("Error"), CH.lit(1), CH.lit(0)), })) - .where(($) => buildWhereConditions($, { ...opts, rootOnly: true }, useTraceListMv)) + .where(($) => buildWhereConditions($, { ...opts, rootOnly: true })) .orderBy(["startTime", "desc"]) .limit(limit) .format("JSON") diff --git a/packages/query-engine/src/ch/query.ts b/packages/query-engine/src/ch/query.ts index a6c7672b1f..5be9add2d2 100644 --- a/packages/query-engine/src/ch/query.ts +++ b/packages/query-engine/src/ch/query.ts @@ -1,8 +1,8 @@ // --------------------------------------------------------------------------- // Query Builder // -// Fluent builder with progressive type accumulation, inspired by Effect's -// HttpApiEndpoint pattern. Each method call refines the type parameters. +// Fluent builder with progressive type accumulation, inspired by Kysely's +// type-safe joins. Each method call refines the type parameters. // // Usage: // const q = CH.from(Traces) @@ -16,9 +16,17 @@ // .groupBy("bucket") // .orderBy(["bucket", "asc"]) // .format("JSON") +// +// Type-safe joins: +// CH.from(Traces) +// .innerJoin(ErrorSpans, "e", (main, e) => main.TraceId.eq(e.TraceId)) +// .select($ => ({ +// traceId: $.TraceId, +// errorType: $.e.ErrorType, +// })) // --------------------------------------------------------------------------- -import type { ColumnDefs, CHType, InferTS } from "./types" +import type { ColumnDefs, CHType, InferTS, OutputToColumnDefs, NullableColumnDefs } from "./types" import type { Table } from "./table" import type { Expr, Condition, ColumnRef } from "./expr" import { makeColumnRef } from "./expr" @@ -31,6 +39,14 @@ export type ColumnAccessor = { readonly [K in keyof Cols & string]: ColumnRef } +/** Combined accessor: main table columns + nested alias accessors for joins. */ +export type JoinedColumnAccessor< + Cols extends ColumnDefs, + Joins extends Record, +> = ColumnAccessor & { + readonly [A in keyof Joins & string]: ColumnAccessor +} + type SelectRecord = Record> export type InferOutput = { @@ -39,14 +55,22 @@ export type InferOutput = { export type OrderBySpec = [keyof Output & string, "asc" | "desc"] +/** Callback for ON conditions — receives main and joined column accessors. */ +export type JoinOnCallback< + MainCols extends ColumnDefs, + JoinedCols extends ColumnDefs, +> = (main: ColumnAccessor, joined: ColumnAccessor) => Condition + // --------------------------------------------------------------------------- // Query state (runtime storage) // --------------------------------------------------------------------------- -export interface JoinClause { +export interface TypedJoinClause { readonly type: "INNER" | "LEFT" | "CROSS" - /** Table name, or subquery SQL (without parens — compiler wraps subqueries). */ - readonly tableSql: string + /** Table name for direct table joins. */ + readonly tableName?: string + /** Inner query for subquery joins (compiled lazily at compileCH time). */ + readonly innerQuery?: CHQuery readonly alias: string /** ON condition. Omitted for CROSS JOIN. */ readonly on?: Condition @@ -63,10 +87,11 @@ export interface CHQueryState { readonly limitValue?: number readonly offsetValue?: number readonly formatValue?: string - /** When set, the FROM clause uses a subquery instead of a table name. */ - readonly fromSubquerySql?: string - readonly fromSubqueryAlias?: string - readonly joins: JoinClause[] + /** Typed FROM subquery. Compiled lazily at compileCH time. */ + readonly fromQuery?: CHQuery + readonly fromQueryAlias?: string + /** Typed joins (compiled lazily at compileCH time). */ + readonly typedJoins: TypedJoinClause[] /** CTE definitions prepended as WITH clauses. */ readonly ctes: Array<{ name: string; sql: string }> } @@ -78,60 +103,84 @@ export interface CHQueryState { export interface CHQuery< Cols extends ColumnDefs = ColumnDefs, Output extends Record = {}, + Joins extends Record = {}, > { /** @internal — runtime query state */ readonly _state: CHQueryState /** phantom */ - readonly _phantom?: { cols: Cols; output: Output } + readonly _phantom?: { cols: Cols; output: Output; joins: Joins } /** Select specific columns by name. Output keys match column names. */ select( ...columns: K[] - ): CHQuery }> + ): CHQuery }, Joins> /** Select computed expressions via callback. */ select( - fn: ($: ColumnAccessor) => S, - ): CHQuery> + fn: ($: JoinedColumnAccessor) => S, + ): CHQuery, Joins> where( - fn: ($: ColumnAccessor) => Array, - ): CHQuery + fn: ($: JoinedColumnAccessor) => Array, + ): CHQuery - groupBy(...keys: Array): CHQuery + groupBy(...keys: Array): CHQuery - orderBy(...specs: Array>): CHQuery + orderBy(...specs: Array>): CHQuery - limit(n: number): CHQuery + limit(n: number): CHQuery - offset(n: number): CHQuery + offset(n: number): CHQuery - format(fmt: "JSON" | "JSONEachRow"): CHQuery + format(fmt: "JSON" | "JSONEachRow"): CHQuery - /** - * Add a JOIN clause. The joined table's columns are accessed via the alias - * in raw expressions (e.g., `CH.dynamicColumn("e.TraceId")`). - * - * For CROSS JOIN, pass `on` as `undefined`. - */ - join( - tableSql: string, - alias: string, - on: Condition | undefined, - type?: "INNER" | "LEFT" | "CROSS", - ): CHQuery + // --------------------------------------------------------------------------- + // Type-safe joins with Table + // --------------------------------------------------------------------------- + + innerJoin( + table: Table, + alias: Alias, + on: JoinOnCallback, + ): CHQuery + + leftJoin( + table: Table, + alias: Alias, + on: JoinOnCallback, + ): CHQuery }> + + crossJoin( + table: Table, + alias: Alias, + ): CHQuery + + // --------------------------------------------------------------------------- + // Type-safe joins with subquery (CHQuery) + // --------------------------------------------------------------------------- + + innerJoinQuery, JJoins extends Record, Alias extends string>( + query: CHQuery, + alias: Alias, + on: JoinOnCallback>, + ): CHQuery }> + + leftJoinQuery, JJoins extends Record, Alias extends string>( + query: CHQuery, + alias: Alias, + on: JoinOnCallback>, + ): CHQuery> }> + + crossJoinQuery, JJoins extends Record, Alias extends string>( + query: CHQuery, + alias: Alias, + ): CHQuery }> /** * Add a CTE (WITH clause). The CTE SQL is prepended to the compiled query. * The CTE name can then be used as a table name via `from()` or in raw expressions. */ - withCTE(name: string, sql: string): CHQuery - - /** - * @deprecated Params are now inferred at the `compile()` call site. - * This method is a no-op and can be safely removed. - */ - withParams<_P extends Record>(): CHQuery + withCTE(name: string, sql: string): CHQuery } // --------------------------------------------------------------------------- @@ -139,7 +188,7 @@ export interface CHQuery< // --------------------------------------------------------------------------- /** Extract the Output type from a CHQuery. */ -export type InferQueryOutput = Q extends CHQuery ? O : never +export type InferQueryOutput = Q extends CHQuery ? O : never // --------------------------------------------------------------------------- // ColumnAccessor factory (Proxy-based) @@ -163,6 +212,63 @@ export function createColumnAccessor( }) } +// --------------------------------------------------------------------------- +// Qualified ColumnAccessor for joined tables (generates alias.Column SQL) +// --------------------------------------------------------------------------- + +function createQualifiedColumnAccessor(alias: string): ColumnAccessor { + const cache = new Map>>() + + return new Proxy({} as ColumnAccessor, { + get(_target, prop) { + if (typeof prop !== "string") return undefined + let ref = cache.get(prop) + if (!ref) { + ref = makeColumnRef(`${alias}.${prop}`) + cache.set(prop, ref) + } + return ref + }, + }) +} + +// --------------------------------------------------------------------------- +// Joined ColumnAccessor — main columns + nested alias accessors +// --------------------------------------------------------------------------- + +export function createJoinedColumnAccessor< + Cols extends ColumnDefs, + Joins extends Record, +>( + _columns: Cols, + joinAliases: readonly string[], + mainAlias?: string, +): JoinedColumnAccessor { + const cache = new Map() + const aliasSet = new Set(joinAliases) + + return new Proxy({} as JoinedColumnAccessor, { + get(_target, prop) { + if (typeof prop !== "string") return undefined + let cached = cache.get(prop) + if (cached) return cached + + if (aliasSet.has(prop)) { + // Return a nested proxy for the joined table's columns + cached = createQualifiedColumnAccessor(prop) + cache.set(prop, cached) + return cached + } + + // Main table column — qualify with alias when joins are present + const qualifiedName = mainAlias ? `${mainAlias}.${prop}` : prop + cached = makeColumnRef(qualifiedName) + cache.set(prop, cached) + return cached + }, + }) +} + // --------------------------------------------------------------------------- // Query builder implementation // --------------------------------------------------------------------------- @@ -170,7 +276,8 @@ export function createColumnAccessor( function makeQuery< Cols extends ColumnDefs, Output extends Record, ->(state: CHQueryState): CHQuery { + Joins extends Record, +>(state: CHQueryState): CHQuery { return { _state: state, @@ -215,14 +322,92 @@ function makeQuery< return makeQuery({ ...state, formatValue: fmt }) }, - join(tableSql, alias, on, type = "INNER") { + // ----------------------------------------------------------------------- + // Type-safe joins with Table + // ----------------------------------------------------------------------- + + innerJoin(table, alias, onFn) { + const mainAlias = state.tableAlias ?? state.tableName + const mainAccessor = createQualifiedColumnAccessor(mainAlias) + const joinedAccessor = createQualifiedColumnAccessor(alias) + const condition = onFn(mainAccessor, joinedAccessor) + return makeQuery({ ...state, - joins: [ - ...state.joins, - { type, tableSql, alias, on }, + typedJoins: [ + ...state.typedJoins, + { type: "INNER", tableName: table.name, alias, on: condition }, ], - }) + }) as any + }, + + leftJoin(table, alias, onFn) { + const mainAlias = state.tableAlias ?? state.tableName + const mainAccessor = createQualifiedColumnAccessor(mainAlias) + const joinedAccessor = createQualifiedColumnAccessor(alias) + const condition = onFn(mainAccessor, joinedAccessor) + + return makeQuery({ + ...state, + typedJoins: [ + ...state.typedJoins, + { type: "LEFT", tableName: table.name, alias, on: condition }, + ], + }) as any + }, + + crossJoin(table, alias) { + return makeQuery({ + ...state, + typedJoins: [ + ...state.typedJoins, + { type: "CROSS", tableName: table.name, alias }, + ], + }) as any + }, + + // ----------------------------------------------------------------------- + // Type-safe joins with subquery (CHQuery) + // ----------------------------------------------------------------------- + + innerJoinQuery(query, alias, onFn) { + const mainAlias = state.tableAlias ?? state.fromQueryAlias ?? state.tableName + const mainAccessor = createQualifiedColumnAccessor(mainAlias) + const joinedAccessor = createQualifiedColumnAccessor(alias) + const condition = onFn(mainAccessor, joinedAccessor) + + return makeQuery({ + ...state, + typedJoins: [ + ...state.typedJoins, + { type: "INNER", innerQuery: query, alias, on: condition }, + ], + }) as any + }, + + leftJoinQuery(query, alias, onFn) { + const mainAlias = state.tableAlias ?? state.fromQueryAlias ?? state.tableName + const mainAccessor = createQualifiedColumnAccessor(mainAlias) + const joinedAccessor = createQualifiedColumnAccessor(alias) + const condition = onFn(mainAccessor, joinedAccessor) + + return makeQuery({ + ...state, + typedJoins: [ + ...state.typedJoins, + { type: "LEFT", innerQuery: query, alias, on: condition }, + ], + }) as any + }, + + crossJoinQuery(query, alias) { + return makeQuery({ + ...state, + typedJoins: [ + ...state.typedJoins, + { type: "CROSS", innerQuery: query, alias }, + ], + }) as any }, withCTE(name, sql) { @@ -231,56 +416,53 @@ function makeQuery< ctes: [...state.ctes, { name, sql }], }) }, - - withParams() { - // Params is phantom — same runtime state, refined compile-time type - return makeQuery(state) as any - }, } } // --------------------------------------------------------------------------- -// Entry point +// Entry points // --------------------------------------------------------------------------- export function from( table: Table, alias?: string, -): CHQuery { +): CHQuery { return makeQuery({ tableName: table.name, tableAlias: alias, columns: table.columns, groupByKeys: [], orderBySpecs: [], - joins: [], + typedJoins: [], ctes: [], }) } /** - * Start a query from a subquery instead of a table. + * Start a query from another query's output (type-safe subquery in FROM). * * Usage: - * const inner = CH.compile( - * CH.from(ErrorSpans).select(...).where(...).limit(10), - * params, - * { skipFormat: true }, - * ) - * const outer = CH.fromSubquery(inner.sql, "e").select($ => ({ ... })) + * const inner = CH.from(Traces).select($ => ({ traceId: $.TraceId })) + * const outer = CH.fromQuery(inner, "sub") + * .select($ => ({ id: $.sub.traceId })) // fully typed! */ -export function fromSubquery( - sql: string, - alias: string, -): CHQuery { +export function fromQuery< + InnerCols extends ColumnDefs, + InnerOutput extends Record, + InnerJoins extends Record, + Alias extends string, +>( + query: CHQuery, + alias: Alias, +): CHQuery, {}, {}> { return makeQuery({ tableName: alias, columns: {}, groupByKeys: [], orderBySpecs: [], - joins: [], + typedJoins: [], ctes: [], - fromSubquerySql: sql, - fromSubqueryAlias: alias, + fromQuery: query, + fromQueryAlias: alias, }) } diff --git a/packages/query-engine/src/ch/types.ts b/packages/query-engine/src/ch/types.ts index 279e071435..52edfec2d8 100644 --- a/packages/query-engine/src/ch/types.ts +++ b/packages/query-engine/src/ch/types.ts @@ -53,6 +53,16 @@ export type InferTS = T extends CHType ? TS : never export type ColumnDefs = Record> +/** Convert a query's Output record to synthetic ColumnDefs for subquery-as-table usage. */ +export type OutputToColumnDefs> = { + readonly [K in keyof O & string]: CHType<"Inferred", O[K]> +} + +/** Wrap each column type with `| null` for LEFT JOIN results. */ +export type NullableColumnDefs = { + readonly [K in keyof Cols & string]: CHType<"Nullable", InferTS | null> +} + // --------------------------------------------------------------------------- // Constructors // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/union.ts b/packages/query-engine/src/ch/union.ts index 0e175bb9c1..3fc6e80c13 100644 --- a/packages/query-engine/src/ch/union.ts +++ b/packages/query-engine/src/ch/union.ts @@ -14,7 +14,7 @@ import type { CHQuery } from "./query" // --------------------------------------------------------------------------- export interface CHUnionState { - readonly queries: ReadonlyArray> + readonly queries: ReadonlyArray> readonly outerOrderBySpecs: Array<[string, "asc" | "desc"]> readonly outerLimitValue?: number readonly outerOffsetValue?: number @@ -87,7 +87,7 @@ function makeUnionQuery< export function unionAll< Output extends Record, >( - ...queries: Array> + ...queries: Array> ): CHUnionQuery { return makeUnionQuery({ queries, diff --git a/packages/query-engine/src/index.ts b/packages/query-engine/src/index.ts index fa379591a8..1552ff8999 100644 --- a/packages/query-engine/src/index.ts +++ b/packages/query-engine/src/index.ts @@ -1,7 +1,6 @@ export * from "./query-engine" export * from "./where-clause" export * from "./traces-shared" -export * from "./traces-sql-builder" export * from "./sql" export * as CH from "./ch" export * as Observability from "./observability" diff --git a/packages/query-engine/src/sql/clickhouse.ts b/packages/query-engine/src/sql/clickhouse.ts deleted file mode 100644 index a15082225e..0000000000 --- a/packages/query-engine/src/sql/clickhouse.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { AttributeFilter } from "../query-engine" -import type { SqlFragment } from "./sql-fragment" -import { compile, raw, str } from "./sql-fragment" - -// --------------------------------------------------------------------------- -// Comparison operators -// --------------------------------------------------------------------------- - -export const eq = (col: string, value: SqlFragment): SqlFragment => - raw(`${col} = ${compile(value)}`) - -export const neq = (col: string, value: SqlFragment): SqlFragment => - raw(`${col} != ${compile(value)}`) - -export const gt = (col: string, value: SqlFragment): SqlFragment => - raw(`${col} > ${compile(value)}`) - -export const gte = (col: string, value: SqlFragment): SqlFragment => - raw(`${col} >= ${compile(value)}`) - -export const lt = (col: string, value: SqlFragment): SqlFragment => - raw(`${col} < ${compile(value)}`) - -export const lte = (col: string, value: SqlFragment): SqlFragment => - raw(`${col} <= ${compile(value)}`) - -export const inList = (col: string, values: ReadonlyArray): SqlFragment => - raw(`${col} IN (${values.map(compile).join(", ")})`) - -// --------------------------------------------------------------------------- -// ClickHouse functions -// --------------------------------------------------------------------------- - -export const toStartOfInterval = (col: string, seconds: number): SqlFragment => - raw(`toStartOfInterval(${col}, INTERVAL ${Math.round(seconds)} SECOND)`) - -// --------------------------------------------------------------------------- -// Attribute filter builder -// --------------------------------------------------------------------------- - -const MODE_TO_OPERATOR: Record = { - gt: ">", - gte: ">=", - lt: "<", - lte: "<=", -} - -/** Numeric columns in trace_list_mv that need casting from string for comparisons */ -const NUMERIC_MV_COLUMNS = new Set(["HttpStatusCode"]) - -export function attrFilter( - af: AttributeFilter, - useMv: boolean, - mapName: "SpanAttributes" | "ResourceAttributes", - mvMap: Record, -): SqlFragment { - const mvColumn = useMv ? mvMap[af.key] : undefined - const escapedKey = compile(str(af.key)) - const escapedValue = compile(str(af.value ?? "")) - - if (af.mode === "exists") { - return mvColumn - ? raw(`${mvColumn} != ''`) - : raw(`mapContains(${mapName}, ${escapedKey})`) - } - - if (af.mode === "contains") { - const col = mvColumn ?? `${mapName}[${escapedKey}]` - return raw(`positionCaseInsensitive(${col}, ${escapedValue}) > 0`) - } - - const op = MODE_TO_OPERATOR[af.mode] - if (op) { - if (mvColumn) { - const cast = NUMERIC_MV_COLUMNS.has(mvColumn) ? `toUInt16OrZero(${mvColumn})` : mvColumn - return raw(`${cast} ${op} ${escapedValue}`) - } - // For numeric comparisons on map values, use escapeClickHouseString (no quotes) - const rawEscapedValue = af.value?.replace(/\\/g, "\\\\").replace(/'/g, "\\'") ?? "" - return raw(`toFloat64OrZero(${mapName}[${escapedKey}]) ${op} ${rawEscapedValue}`) - } - - // equals (default) - if (mvColumn) { - return raw(`${mvColumn} = ${escapedValue}`) - } - return raw(`${mapName}[${escapedKey}] = ${escapedValue}`) -} diff --git a/packages/query-engine/src/sql/index.ts b/packages/query-engine/src/sql/index.ts index 11eb4e9917..a9c34994c3 100644 --- a/packages/query-engine/src/sql/index.ts +++ b/packages/query-engine/src/sql/index.ts @@ -13,14 +13,3 @@ export { export { type SqlQuery, compileQuery } from "./sql-query" -export { - eq, - neq, - gt, - gte, - lt, - lte, - inList, - toStartOfInterval, - attrFilter, -} from "./clickhouse" diff --git a/packages/query-engine/src/traces-shared.ts b/packages/query-engine/src/traces-shared.ts index eff5de21bc..b5b8f13900 100644 --- a/packages/query-engine/src/traces-shared.ts +++ b/packages/query-engine/src/traces-shared.ts @@ -1,6 +1,5 @@ // --------------------------------------------------------------------------- -// Shared constants and helpers used by both the legacy traces-sql-builder -// and the new CH DSL queries. +// Shared constants and helpers used by the CH DSL queries. // --------------------------------------------------------------------------- import type { TracesMetric, AttributeFilter } from "./query-engine" @@ -22,7 +21,7 @@ export const METRIC_NEEDS: Record = { } // --------------------------------------------------------------------------- -// trace_list_mv column mappings +// trace_list_mv column mappings (used by performance-hints UI) // --------------------------------------------------------------------------- export const TRACE_LIST_MV_ATTR_MAP: Record = { @@ -39,27 +38,6 @@ export const TRACE_LIST_MV_RESOURCE_MAP: Record = { "deployment.environment": "DeploymentEnv", } -export const NUMERIC_MV_COLUMNS = new Set(["HttpStatusCode"]) - -// --------------------------------------------------------------------------- -// trace_list_mv eligibility check -// --------------------------------------------------------------------------- - -export function canUseTraceListMv(_params: { - rootOnly?: boolean - attributeFilters?: readonly AttributeFilter[] - resourceAttributeFilters?: readonly AttributeFilter[] - commitShas?: readonly string[] - groupBy?: readonly string[] | string - groupByAttributeKeys?: readonly string[] - groupByAttributeKey?: string -}): boolean { - // The MV only contains ParentSpanId='' rows, but rootOnly means - // "entry point spans" (Server/Consumer OR ParentSpanId=''), which is broader. - // Disable MV until it is updated to include Server/Consumer spans. - return false -} - // --------------------------------------------------------------------------- // Attribute filter → typed Condition // --------------------------------------------------------------------------- @@ -68,20 +46,13 @@ import * as CH from "./ch/expr" export function buildAttrFilterCondition( af: AttributeFilter, - useMv: boolean, mapName: "SpanAttributes" | "ResourceAttributes", - mvMap: Record, ): CH.Condition { - const mvColumn = useMv ? mvMap[af.key] : undefined - const colExpr: CH.Expr = mvColumn - ? CH.dynamicColumn(mvColumn) - : CH.mapGet(CH.dynamicColumn>(mapName), af.key) + const colExpr: CH.Expr = CH.mapGet(CH.dynamicColumn>(mapName), af.key) const value = af.value ?? "" if (af.mode === "exists") { - return mvColumn - ? CH.dynamicColumn(mvColumn).neq("") - : CH.mapContains(CH.dynamicColumn>(mapName), af.key) + return CH.mapContains(CH.dynamicColumn>(mapName), af.key) } if (af.mode === "contains") { @@ -89,28 +60,16 @@ export function buildAttrFilterCondition( } if (af.mode === "gt") { - const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) - ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) - : CH.toFloat64OrZero(colExpr) - return numExpr.gt(Number(value)) + return CH.toFloat64OrZero(colExpr).gt(Number(value)) } if (af.mode === "gte") { - const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) - ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) - : CH.toFloat64OrZero(colExpr) - return numExpr.gte(Number(value)) + return CH.toFloat64OrZero(colExpr).gte(Number(value)) } if (af.mode === "lt") { - const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) - ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) - : CH.toFloat64OrZero(colExpr) - return numExpr.lt(Number(value)) + return CH.toFloat64OrZero(colExpr).lt(Number(value)) } if (af.mode === "lte") { - const numExpr = mvColumn && NUMERIC_MV_COLUMNS.has(mvColumn) - ? CH.toUInt16OrZero(CH.dynamicColumn(mvColumn)) - : CH.toFloat64OrZero(colExpr) - return numExpr.lte(Number(value)) + return CH.toFloat64OrZero(colExpr).lte(Number(value)) } // equals (default) diff --git a/packages/query-engine/src/traces-sql-builder.test.ts b/packages/query-engine/src/traces-sql-builder.test.ts deleted file mode 100644 index 351fe7b15a..0000000000 --- a/packages/query-engine/src/traces-sql-builder.test.ts +++ /dev/null @@ -1,354 +0,0 @@ -import { describe, expect, it } from "bun:test" -import { buildTracesTimeseriesSQL, buildTracesBreakdownSQL, escapeClickHouseString } from "./traces-sql-builder" - -describe("escapeClickHouseString", () => { - it("escapes single quotes", () => { - expect(escapeClickHouseString("it's")).toBe("it\\'s") - }) - - it("escapes backslashes", () => { - expect(escapeClickHouseString("a\\b")).toBe("a\\\\b") - }) - - it("prevents SQL injection", () => { - const result = escapeClickHouseString("'; DROP TABLE traces; --") - expect(result).toBe("\\'; DROP TABLE traces; --") - }) -}) - -describe("buildTracesTimeseriesSQL", () => { - const baseParams = { - orgId: "org_123", - startTime: "2024-01-01 00:00:00", - endTime: "2024-01-02 00:00:00", - bucketSeconds: 3600, - metric: "count" as const, - needsSampling: false, - } - - it("builds basic count timeseries", () => { - const sql = buildTracesTimeseriesSQL(baseParams) - expect(sql).toContain("SELECT") - expect(sql).toContain("FROM traces") - expect(sql).toContain("OrgId = 'org_123'") - expect(sql).toContain("count() AS count") - expect(sql).toContain("INTERVAL 3600 SECOND") - expect(sql).toContain("GROUP BY bucket, groupName") - expect(sql).toContain("ORDER BY bucket ASC, groupName ASC") - expect(sql).toContain("FORMAT JSON") - // Default group name - expect(sql).toContain("'all' AS groupName") - // count metric should not include quantiles - expect(sql).toContain("0 AS p50Duration") - }) - - it("builds apdex timeseries with threshold", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - metric: "apdex", - apdexThresholdMs: 250, - }) - expect(sql).toContain("countIf(Duration / 1000000 < 250) AS satisfiedCount") - expect(sql).toContain("toleratingCount") - expect(sql).toContain("apdexScore") - }) - - it("builds p95 duration timeseries", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - metric: "p95_duration", - }) - expect(sql).toContain("quantile(0.5)(Duration) / 1000000 AS p50Duration") - expect(sql).toContain("quantile(0.95)(Duration) / 1000000 AS p95Duration") - expect(sql).toContain("quantile(0.99)(Duration) / 1000000 AS p99Duration") - }) - - it("builds error_rate timeseries", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - metric: "error_rate", - }) - expect(sql).toContain("countIf(StatusCode = 'Error') * 100.0 / count(), 0) AS errorRate") - }) - - it("includes sampling columns when needsSampling is true", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - needsSampling: true, - }) - expect(sql).toContain("sampledSpanCount") - expect(sql).toContain("unsampledSpanCount") - expect(sql).toContain("dominantThreshold") - expect(sql).toContain("TraceState LIKE '%th:%'") - }) - - it("excludes sampling columns when needsSampling is false", () => { - const sql = buildTracesTimeseriesSQL(baseParams) - expect(sql).toContain("0 AS sampledSpanCount") - expect(sql).toContain("0 AS unsampledSpanCount") - expect(sql).toContain("'' AS dominantThreshold") - }) - - it("groups by service", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - groupBy: ["service"], - }) - expect(sql).toContain("toString(ServiceName)") - expect(sql).toContain("AS groupName") - }) - - it("groups by span_name", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - groupBy: ["span_name"], - }) - expect(sql).toContain("toString(SpanName)") - }) - - it("groups by multiple dimensions", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - groupBy: ["service", "status_code"], - }) - expect(sql).toContain("arrayStringConcat") - expect(sql).toContain("arrayFilter") - }) - - it("groups by attribute with keys", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - groupBy: ["attribute"], - groupByAttributeKeys: ["http.route"], - }) - expect(sql).toContain("SpanAttributes['http.route']") - }) - - it("uses traces table when rootOnly (MV disabled)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - rootOnly: true, - }) - expect(sql).toContain("FROM traces") - expect(sql).toContain("SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''") - }) - - it("uses traces table when not rootOnly", () => { - const sql = buildTracesTimeseriesSQL(baseParams) - expect(sql).toContain("FROM traces") - }) - - it("falls back to traces table when commitShas present", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - rootOnly: true, - commitShas: ["abc123"], - }) - expect(sql).toContain("FROM traces") - expect(sql).toContain("ParentSpanId = ''") - expect(sql).toContain("deployment.commit_sha") - }) - - it("filters by serviceName", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - serviceName: "api-service", - }) - expect(sql).toContain("ServiceName = 'api-service'") - }) - - it("filters by spanName", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - spanName: "GET /users", - }) - expect(sql).toContain("SpanName = 'GET /users'") - }) - - it("filters errorsOnly", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - errorsOnly: true, - }) - expect(sql).toContain("StatusCode = 'Error'") - }) - - it("filters errorsOnly with rootOnly (MV disabled)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - rootOnly: true, - errorsOnly: true, - }) - expect(sql).toContain("StatusCode = 'Error'") - }) - - it("filters by environments", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - environments: ["production", "staging"], - }) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production', 'staging')") - }) - - it("filters by environments with rootOnly (MV disabled)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - rootOnly: true, - environments: ["production"], - }) - expect(sql).toContain("deployment.environment") - }) - - it("filters by attribute filters (equals)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - attributeFilters: [{ key: "http.status_code", value: "200", mode: "equals" }], - }) - expect(sql).toContain("SpanAttributes['http.status_code'] = '200'") - }) - - it("filters by attribute filters (exists)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - attributeFilters: [{ key: "http.route", mode: "exists" }], - }) - expect(sql).toContain("mapContains(SpanAttributes, 'http.route')") - }) - - it("filters by attribute filters (contains)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - attributeFilters: [{ key: "http.route", value: "/api", mode: "contains" }], - }) - expect(sql).toContain("positionCaseInsensitive(SpanAttributes['http.route'], '/api') > 0") - }) - - it("filters by attribute filters (gt)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - attributeFilters: [{ key: "http.status_code", value: "400", mode: "gt" }], - }) - expect(sql).toContain("toFloat64OrZero(SpanAttributes['http.status_code']) > 400") - }) - - it("filters by resource attribute filters", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - resourceAttributeFilters: [{ key: "host.name", value: "server-1", mode: "equals" }], - }) - expect(sql).toContain("ResourceAttributes['host.name'] = 'server-1'") - }) - - it("filters attribute with rootOnly on raw table (MV disabled)", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - rootOnly: true, - attributeFilters: [{ key: "http.method", value: "GET", mode: "equals" }], - }) - expect(sql).toContain("FROM traces") - expect(sql).toContain("SpanAttributes['http.method'] = 'GET'") - }) - - it("escapes special characters in filter values", () => { - const sql = buildTracesTimeseriesSQL({ - ...baseParams, - serviceName: "it's-a-service", - }) - expect(sql).toContain("ServiceName = 'it\\'s-a-service'") - }) -}) - -describe("buildTracesBreakdownSQL", () => { - const baseParams = { - orgId: "org_123", - startTime: "2024-01-01 00:00:00", - endTime: "2024-01-02 00:00:00", - metric: "count" as const, - groupBy: "service" as const, - } - - it("builds basic breakdown by service", () => { - const sql = buildTracesBreakdownSQL(baseParams) - expect(sql).toContain("SELECT") - expect(sql).toContain("ServiceName AS name") - expect(sql).toContain("count() AS count") - expect(sql).toContain("GROUP BY name") - expect(sql).toContain("ORDER BY count DESC") - expect(sql).toContain("LIMIT 10") - expect(sql).toContain("FORMAT JSON") - }) - - it("groups by span_name", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - groupBy: "span_name", - }) - expect(sql).toContain("SpanName AS name") - }) - - it("groups by status_code", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - groupBy: "status_code", - }) - expect(sql).toContain("StatusCode AS name") - }) - - it("groups by http_method", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - groupBy: "http_method", - }) - expect(sql).toContain("SpanAttributes['http.method'] AS name") - }) - - it("groups by custom attribute", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - groupBy: "attribute", - groupByAttributeKey: "rpc.service", - }) - expect(sql).toContain("SpanAttributes['rpc.service'] AS name") - }) - - it("applies custom limit", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - limit: 25, - }) - expect(sql).toContain("LIMIT 25") - }) - - it("uses default limit of 10", () => { - const sql = buildTracesBreakdownSQL(baseParams) - expect(sql).toContain("LIMIT 10") - }) - - it("includes apdex columns for apdex metric", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - metric: "apdex", - apdexThresholdMs: 300, - }) - expect(sql).toContain("countIf(Duration / 1000000 < 300) AS satisfiedCount") - expect(sql).toContain("apdexScore") - }) - - it("includes quantile columns for p99 metric", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - metric: "p99_duration", - }) - expect(sql).toContain("quantile(0.99)(Duration) / 1000000 AS p99Duration") - }) - - it("applies WHERE filters", () => { - const sql = buildTracesBreakdownSQL({ - ...baseParams, - serviceName: "api", - errorsOnly: true, - }) - expect(sql).toContain("ServiceName = 'api'") - expect(sql).toContain("StatusCode = 'Error'") - }) -}) diff --git a/packages/query-engine/src/traces-sql-builder.ts b/packages/query-engine/src/traces-sql-builder.ts deleted file mode 100644 index e1b5454da5..0000000000 --- a/packages/query-engine/src/traces-sql-builder.ts +++ /dev/null @@ -1,461 +0,0 @@ -import type { TracesMetric, AttributeFilter } from "./query-engine" -import { - type SqlFragment, - escapeClickHouseString, - raw, - str, - ident, - as_, - when, - compile, -} from "./sql/sql-fragment" -import { compileQuery } from "./sql/sql-query" -import { attrFilter, eq, gte, lte, inList, toStartOfInterval } from "./sql/clickhouse" -import { - METRIC_NEEDS, - TRACE_LIST_MV_ATTR_MAP, - TRACE_LIST_MV_RESOURCE_MAP, - canUseTraceListMv, -} from "./traces-shared" - -export { escapeClickHouseString } - -// --------------------------------------------------------------------------- -// Row types returned by SQL queries -// --------------------------------------------------------------------------- - -export interface TracesTimeseriesRow { - readonly bucket: string | Date - readonly groupName: string - readonly count: number - readonly avgDuration: number - readonly p50Duration: number - readonly p95Duration: number - readonly p99Duration: number - readonly errorRate: number - readonly satisfiedCount: number - readonly toleratingCount: number - readonly apdexScore: number - readonly sampledSpanCount: number - readonly unsampledSpanCount: number - readonly dominantThreshold: string -} - -export interface TracesBreakdownRow { - readonly name: string - readonly count: number - readonly avgDuration: number - readonly p50Duration: number - readonly p95Duration: number - readonly p99Duration: number - readonly errorRate: number - readonly satisfiedCount: number - readonly toleratingCount: number - readonly apdexScore: number -} - -// --------------------------------------------------------------------------- -// Metric → SELECT columns mapping (imported from traces-shared.ts) -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// Metric SELECT fragments -// --------------------------------------------------------------------------- - -function metricSelectFragments( - metric: TracesMetric, - needsSampling: boolean, - apdexThresholdMs: number, -): SqlFragment[] { - const needs = new Set(METRIC_NEEDS[metric]) - const cols: SqlFragment[] = [raw("count() AS count")] - - if (needs.has("avg_duration")) { - cols.push(raw("avg(Duration) / 1000000 AS avgDuration")) - } else { - cols.push(raw("0 AS avgDuration")) - } - - if (needs.has("quantiles")) { - cols.push( - raw("quantile(0.5)(Duration) / 1000000 AS p50Duration"), - raw("quantile(0.95)(Duration) / 1000000 AS p95Duration"), - raw("quantile(0.99)(Duration) / 1000000 AS p99Duration"), - ) - } else { - cols.push(raw("0 AS p50Duration"), raw("0 AS p95Duration"), raw("0 AS p99Duration")) - } - - if (needs.has("error_rate")) { - cols.push(raw("if(count() > 0, countIf(StatusCode = 'Error') * 100.0 / count(), 0) AS errorRate")) - } else { - cols.push(raw("0 AS errorRate")) - } - - const t = String(apdexThresholdMs) - if (needs.has("apdex")) { - cols.push( - raw(`countIf(Duration / 1000000 < ${t}) AS satisfiedCount`), - raw(`countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) AS toleratingCount`), - raw(`if(count() > 0, round((countIf(Duration / 1000000 < ${t}) + countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) * 0.5) / count(), 4), 0) AS apdexScore`), - ) - } else { - cols.push(raw("0 AS satisfiedCount"), raw("0 AS toleratingCount"), raw("0 AS apdexScore")) - } - - if (needsSampling) { - cols.push( - raw("countIf(TraceState LIKE '%th:%') AS sampledSpanCount"), - raw("countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS unsampledSpanCount"), - raw("anyIf(extract(TraceState, 'th:([0-9a-f]+)'), TraceState LIKE '%th:%') AS dominantThreshold"), - ) - } else { - cols.push(raw("0 AS sampledSpanCount"), raw("0 AS unsampledSpanCount"), raw("'' AS dominantThreshold")) - } - - return cols -} - -function breakdownMetricSelectFragments( - metric: TracesMetric, - apdexThresholdMs: number, -): SqlFragment[] { - const needs = new Set(METRIC_NEEDS[metric]) - const cols: SqlFragment[] = [raw("count() AS count")] - - if (needs.has("avg_duration")) { - cols.push(raw("avg(Duration) / 1000000 AS avgDuration")) - } else { - cols.push(raw("0 AS avgDuration")) - } - - if (needs.has("quantiles")) { - cols.push( - raw("quantile(0.5)(Duration) / 1000000 AS p50Duration"), - raw("quantile(0.95)(Duration) / 1000000 AS p95Duration"), - raw("quantile(0.99)(Duration) / 1000000 AS p99Duration"), - ) - } else { - cols.push(raw("0 AS p50Duration"), raw("0 AS p95Duration"), raw("0 AS p99Duration")) - } - - if (needs.has("error_rate")) { - cols.push(raw("if(count() > 0, countIf(StatusCode = 'Error') * 100.0 / count(), 0) AS errorRate")) - } else { - cols.push(raw("0 AS errorRate")) - } - - const t = String(apdexThresholdMs) - if (needs.has("apdex")) { - cols.push( - raw(`countIf(Duration / 1000000 < ${t}) AS satisfiedCount`), - raw(`countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) AS toleratingCount`), - raw(`if(count() > 0, round((countIf(Duration / 1000000 < ${t}) + countIf(Duration / 1000000 >= ${t} AND Duration / 1000000 < ${t} * 4) * 0.5) / count(), 4), 0) AS apdexScore`), - ) - } else { - cols.push(raw("0 AS satisfiedCount"), raw("0 AS toleratingCount"), raw("0 AS apdexScore")) - } - - return cols -} - -// --------------------------------------------------------------------------- -// GROUP BY expression builder -// --------------------------------------------------------------------------- - -function groupNameFragment( - groupBy: readonly string[] | undefined, - groupByAttributeKeys: readonly string[] | undefined, - useTraceListMv: boolean, -): SqlFragment { - if (!groupBy || groupBy.length === 0) { - return raw("'all' AS groupName") - } - - const parts: string[] = [] - for (const g of groupBy) { - switch (g) { - case "service": - parts.push("toString(ServiceName)") - break - case "span_name": - parts.push("toString(SpanName)") - break - case "status_code": - parts.push("toString(StatusCode)") - break - case "http_method": - if (useTraceListMv) { - parts.push("toString(HttpMethod)") - } else { - parts.push("toString(SpanAttributes['http.method'])") - } - break - case "attribute": - if (groupByAttributeKeys?.length) { - const keys = groupByAttributeKeys.map((k) => { - const mvCol = useTraceListMv ? TRACE_LIST_MV_ATTR_MAP[k] : undefined - return mvCol ? `toString(${mvCol})` : `toString(SpanAttributes[${compile(str(k))}])` - }) - parts.push(`arrayStringConcat([${keys.join(", ")}], ' · ')`) - } - break - case "none": - break - } - } - - if (parts.length === 0) { - return raw("'all' AS groupName") - } - - if (parts.length === 1) { - return raw(`coalesce(nullIf(${parts[0]}, ''), 'all') AS groupName`) - } - - return raw(`coalesce(nullIf(arrayStringConcat(arrayFilter(x -> x != '', [${parts.join(", ")}]), ' · '), ''), 'all') AS groupName`) -} - -function breakdownGroupFragment(groupBy: string, groupByAttributeKey?: string): SqlFragment { - switch (groupBy) { - case "service": - return raw("ServiceName AS name") - case "span_name": - return raw("SpanName AS name") - case "status_code": - return raw("StatusCode AS name") - case "http_method": - return raw("SpanAttributes['http.method'] AS name") - case "attribute": - return groupByAttributeKey - ? raw(`SpanAttributes[${compile(str(groupByAttributeKey))}] AS name`) - : raw("ServiceName AS name") - default: - return raw("ServiceName AS name") - } -} - -// trace_list_mv column mapping + canUseTraceListMv imported from traces-shared.ts - -// --------------------------------------------------------------------------- -// WHERE clause builder -// --------------------------------------------------------------------------- - -interface WhereClauseParams { - orgId: string - startTime: string - endTime: string - serviceName?: string - spanName?: string - rootOnly?: boolean - errorsOnly?: boolean - environments?: readonly string[] - commitShas?: readonly string[] - attributeFilters?: readonly AttributeFilter[] - resourceAttributeFilters?: readonly AttributeFilter[] -} - -function buildWhereFragments(params: WhereClauseParams, useTraceListMv: boolean): SqlFragment[] { - const clauses: SqlFragment[] = [ - eq("OrgId", str(params.orgId)), - ] - - if (params.serviceName) { - clauses.push(eq("ServiceName", str(params.serviceName))) - } - if (params.spanName) { - clauses.push(eq("SpanName", str(params.spanName))) - } - - clauses.push( - gte("Timestamp", str(params.startTime)), - lte("Timestamp", str(params.endTime)), - ) - - // Entry point spans: Server/Consumer spans or true root spans - clauses.push( - when(!!params.rootOnly && !useTraceListMv, raw("(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '')")), - ) - if (params.errorsOnly) { - if (useTraceListMv) { - clauses.push(raw("HasError = 1")) - } else { - clauses.push(raw("StatusCode = 'Error'")) - } - } - if (params.environments?.length) { - if (useTraceListMv) { - clauses.push(inList("DeploymentEnv", params.environments.map(str))) - } else { - clauses.push(inList("ResourceAttributes['deployment.environment']", params.environments.map(str))) - } - } - if (params.commitShas?.length) { - clauses.push(inList("ResourceAttributes['deployment.commit_sha']", params.commitShas.map(str))) - } - - if (params.attributeFilters) { - for (const af of params.attributeFilters) { - clauses.push(attrFilter(af, useTraceListMv, "SpanAttributes", TRACE_LIST_MV_ATTR_MAP)) - } - } - - if (params.resourceAttributeFilters) { - for (const rf of params.resourceAttributeFilters) { - clauses.push(attrFilter(rf, useTraceListMv, "ResourceAttributes", TRACE_LIST_MV_RESOURCE_MAP)) - } - } - - return clauses -} - -// --------------------------------------------------------------------------- -// Timeseries SQL builder -// --------------------------------------------------------------------------- - -export interface BuildTracesTimeseriesSQLParams { - orgId: string - startTime: string - endTime: string - bucketSeconds: number - metric: TracesMetric - needsSampling: boolean - serviceName?: string - spanName?: string - rootOnly?: boolean - errorsOnly?: boolean - groupBy?: readonly string[] - groupByAttributeKeys?: readonly string[] - environments?: readonly string[] - commitShas?: readonly string[] - attributeFilters?: readonly AttributeFilter[] - resourceAttributeFilters?: readonly AttributeFilter[] - apdexThresholdMs?: number -} - -export function buildTracesTimeseriesSQL(params: BuildTracesTimeseriesSQLParams): string { - const apdexThresholdMs = params.apdexThresholdMs ?? 500 - const useTraceListMv = canUseTraceListMv(params) - const tableName = useTraceListMv ? "trace_list_mv" : "traces" - - return compileQuery({ - select: [ - as_(toStartOfInterval("Timestamp", params.bucketSeconds), "bucket"), - groupNameFragment(params.groupBy, params.groupByAttributeKeys, useTraceListMv), - ...metricSelectFragments(params.metric, params.needsSampling, apdexThresholdMs), - ], - from: ident(tableName), - where: buildWhereFragments(params, useTraceListMv), - groupBy: [raw("bucket, groupName")], - orderBy: [raw("bucket ASC, groupName ASC")], - format: "JSON", - }) -} - -// --------------------------------------------------------------------------- -// Breakdown SQL builder -// --------------------------------------------------------------------------- - -export interface BuildTracesBreakdownSQLParams { - orgId: string - startTime: string - endTime: string - metric: TracesMetric - groupBy: string - groupByAttributeKey?: string - limit?: number - serviceName?: string - spanName?: string - rootOnly?: boolean - errorsOnly?: boolean - environments?: readonly string[] - commitShas?: readonly string[] - attributeFilters?: readonly AttributeFilter[] - resourceAttributeFilters?: readonly AttributeFilter[] - apdexThresholdMs?: number -} - -// --------------------------------------------------------------------------- -// List SQL builder -// --------------------------------------------------------------------------- - -export interface TracesListRow { - readonly traceId: string - readonly timestamp: string | Date - readonly spanId: string - readonly serviceName: string - readonly spanName: string - readonly durationMs: number - readonly statusCode: string - readonly spanKind: string - readonly hasError: number - readonly spanAttributes: Record - readonly resourceAttributes: Record -} - -export interface BuildTracesListSQLParams { - orgId: string - startTime: string - endTime: string - limit?: number - serviceName?: string - spanName?: string - rootOnly?: boolean - errorsOnly?: boolean - environments?: readonly string[] - commitShas?: readonly string[] - attributeFilters?: readonly AttributeFilter[] - resourceAttributeFilters?: readonly AttributeFilter[] -} - -export function buildTracesListSQL(params: BuildTracesListSQLParams): string { - const limit = params.limit ?? 100 - // List queries always use the raw traces table (not MV) to get full attributes - const useTraceListMv = false - - return compileQuery({ - select: [ - as_(ident("TraceId"), "traceId"), - as_(ident("Timestamp"), "timestamp"), - as_(ident("SpanId"), "spanId"), - as_(ident("ServiceName"), "serviceName"), - as_(ident("SpanName"), "spanName"), - as_(raw("Duration / 1000000"), "durationMs"), - as_(ident("StatusCode"), "statusCode"), - as_(ident("SpanKind"), "spanKind"), - as_(raw("if(StatusCode = 'Error', 1, 0)"), "hasError"), - as_(ident("SpanAttributes"), "spanAttributes"), - as_(ident("ResourceAttributes"), "resourceAttributes"), - ], - from: ident("traces"), - where: buildWhereFragments(params, useTraceListMv), - groupBy: [], - orderBy: [raw("Timestamp DESC")], - limit: raw(String(Math.round(limit))), - format: "JSON", - }) -} - -// --------------------------------------------------------------------------- -// Breakdown SQL builder -// --------------------------------------------------------------------------- - -export function buildTracesBreakdownSQL(params: BuildTracesBreakdownSQLParams): string { - const apdexThresholdMs = params.apdexThresholdMs ?? 500 - const limit = params.limit ?? 10 - const useTraceListMv = canUseTraceListMv(params) - const tableName = useTraceListMv ? "trace_list_mv" : "traces" - - return compileQuery({ - select: [ - breakdownGroupFragment(params.groupBy, params.groupByAttributeKey), - ...breakdownMetricSelectFragments(params.metric, apdexThresholdMs), - ], - from: ident(tableName), - where: buildWhereFragments(params, useTraceListMv), - groupBy: [ident("name")], - orderBy: [raw("count DESC")], - limit: raw(String(Math.round(limit))), - format: "JSON", - }) -} From 2f3d3f180a2c91ff7aa55cfa45a7c17913e79663 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 13:44:06 +0200 Subject: [PATCH 09/11] add type tests --- bun.lock | 1 + packages/query-engine/package.json | 1 + packages/query-engine/src/ch/expr.test-d.ts | 193 ++++++++++ .../src/ch/queries/query-helpers.ts | 130 +++++++ .../query-engine/src/ch/queries/traces.ts | 125 +------ packages/query-engine/src/ch/query.test-d.ts | 347 ++++++++++++++++++ packages/query-engine/src/ch/types.test-d.ts | 93 +++++ 7 files changed, 779 insertions(+), 111 deletions(-) create mode 100644 packages/query-engine/src/ch/expr.test-d.ts create mode 100644 packages/query-engine/src/ch/queries/query-helpers.ts create mode 100644 packages/query-engine/src/ch/query.test-d.ts create mode 100644 packages/query-engine/src/ch/types.test-d.ts diff --git a/bun.lock b/bun.lock index d92fe615f4..fe2f647583 100644 --- a/bun.lock +++ b/bun.lock @@ -303,6 +303,7 @@ "@effect/language-service": "catalog:effect", "@types/bun": "^1.3.8", "@types/node": "catalog:tooling", + "expect-type": "^1.3.0", "typescript": "catalog:tooling", }, }, diff --git a/packages/query-engine/package.json b/packages/query-engine/package.json index 097186aa66..1c2aa07388 100644 --- a/packages/query-engine/package.json +++ b/packages/query-engine/package.json @@ -23,6 +23,7 @@ "@effect/language-service": "catalog:effect", "@types/bun": "^1.3.8", "@types/node": "catalog:tooling", + "expect-type": "^1.3.0", "typescript": "catalog:tooling" } } diff --git a/packages/query-engine/src/ch/expr.test-d.ts b/packages/query-engine/src/ch/expr.test-d.ts new file mode 100644 index 0000000000..1932bb9c46 --- /dev/null +++ b/packages/query-engine/src/ch/expr.test-d.ts @@ -0,0 +1,193 @@ +// --------------------------------------------------------------------------- +// Type-level tests: Expression type safety +// --------------------------------------------------------------------------- + +import { expectTypeOf } from "expect-type" +import * as CH from "./index" +import type { Expr, Condition, ColumnRef } from "./expr" +import type { ParamMarker } from "./param" +import type { CHString, CHFloat64, CHMap } from "./types" +import { makeColumnRef } from "./expr" + +// --------------------------------------------------------------------------- +// Literal expressions +// --------------------------------------------------------------------------- + +expectTypeOf(CH.lit("hello")).toMatchTypeOf>() +expectTypeOf(CH.lit(42)).toMatchTypeOf>() + +// --------------------------------------------------------------------------- +// Aggregate functions — return types +// --------------------------------------------------------------------------- + +expectTypeOf(CH.count()).toMatchTypeOf>() +expectTypeOf(CH.avg(CH.lit(1))).toMatchTypeOf>() +expectTypeOf(CH.sum(CH.lit(1))).toMatchTypeOf>() +expectTypeOf(CH.uniq(CH.lit("x"))).toMatchTypeOf>() +expectTypeOf(CH.countIf(CH.lit(1).gt(0))).toMatchTypeOf>() + +// min/max preserve generic type +expectTypeOf(CH.min(CH.lit("a"))).toMatchTypeOf>() +expectTypeOf(CH.max(CH.lit(1))).toMatchTypeOf>() +expectTypeOf(CH.min(CH.lit(1))).toMatchTypeOf>() + +// any_ preserves generic +expectTypeOf(CH.any(CH.lit("x"))).toMatchTypeOf>() +expectTypeOf(CH.any(CH.lit(1))).toMatchTypeOf>() + +// groupUniqArray wraps in ReadonlyArray +expectTypeOf(CH.groupUniqArray(CH.lit("x"))).toMatchTypeOf>>() +expectTypeOf(CH.groupUniqArray(CH.lit(1))).toMatchTypeOf>>() + +// quantile returns Expr +expectTypeOf(CH.quantile(0.95)(CH.lit(1))).toMatchTypeOf>() + +// --------------------------------------------------------------------------- +// ClickHouse functions — return types +// --------------------------------------------------------------------------- + +expectTypeOf(CH.toStartOfInterval(CH.lit("ts"), 60)).toMatchTypeOf>() +expectTypeOf(CH.if_(CH.lit(1).gt(0), CH.lit("yes"), CH.lit("no"))).toMatchTypeOf>() +expectTypeOf(CH.coalesce(CH.lit("a"), CH.lit("b"))).toMatchTypeOf>() +expectTypeOf(CH.concat(CH.lit("a"), CH.lit("b"))).toMatchTypeOf>() +expectTypeOf(CH.round_(CH.lit(1), 2)).toMatchTypeOf>() +expectTypeOf(CH.intDiv(CH.lit(10), 3)).toMatchTypeOf>() +expectTypeOf(CH.toString(CH.lit(1))).toMatchTypeOf>() +expectTypeOf(CH.toFloat64OrZero(CH.lit("3.14"))).toMatchTypeOf>() +expectTypeOf(CH.length(CH.lit("hello"))).toMatchTypeOf>() +expectTypeOf(CH.position(CH.lit("hello"), "ell")).toMatchTypeOf>() + +// --------------------------------------------------------------------------- +// Comparison operators — return Condition +// --------------------------------------------------------------------------- + +const strExpr = CH.lit("hello") +const numExpr = CH.lit(42) + +expectTypeOf(strExpr.eq("world")).toMatchTypeOf() +expectTypeOf(strExpr.neq("world")).toMatchTypeOf() +expectTypeOf(numExpr.gt(0)).toMatchTypeOf() +expectTypeOf(numExpr.gte(0)).toMatchTypeOf() +expectTypeOf(numExpr.lt(100)).toMatchTypeOf() +expectTypeOf(numExpr.lte(100)).toMatchTypeOf() + +// Comparisons also accept Expr of same type +expectTypeOf(strExpr.eq(CH.lit("world"))).toMatchTypeOf() +expectTypeOf(numExpr.gt(CH.lit(0))).toMatchTypeOf() + +// IN / NOT IN return Condition +expectTypeOf(strExpr.in_("a", "b")).toMatchTypeOf() +expectTypeOf(strExpr.notIn("a", "b")).toMatchTypeOf() + +// Condition combinators return Condition +const cond = strExpr.eq("x") +expectTypeOf(cond.and(numExpr.gt(0))).toMatchTypeOf() +expectTypeOf(cond.or(numExpr.lt(10))).toMatchTypeOf() + +// --------------------------------------------------------------------------- +// Arithmetic — only valid for Expr +// --------------------------------------------------------------------------- + +expectTypeOf(numExpr.div(2)).toMatchTypeOf>() +expectTypeOf(numExpr.mul(2)).toMatchTypeOf>() +expectTypeOf(numExpr.add(1)).toMatchTypeOf>() +expectTypeOf(numExpr.sub(1)).toMatchTypeOf>() + +// Arithmetic with Expr argument +expectTypeOf(numExpr.div(CH.lit(2))).toMatchTypeOf>() + +// @ts-expect-error — .div() requires Expr, not Expr +strExpr.div(2) + +// @ts-expect-error — .mul() requires Expr +strExpr.mul(2) + +// @ts-expect-error — .add() requires Expr +strExpr.add(1) + +// @ts-expect-error — .sub() requires Expr +strExpr.sub(1) + +// --------------------------------------------------------------------------- +// String operations — only valid for Expr +// --------------------------------------------------------------------------- + +expectTypeOf(strExpr.like("%test%")).toMatchTypeOf() +expectTypeOf(strExpr.ilike("%test%")).toMatchTypeOf() +expectTypeOf(strExpr.notLike("%test%")).toMatchTypeOf() + +// @ts-expect-error — .like() requires Expr +numExpr.like("%test%") + +// @ts-expect-error — .ilike() requires Expr +numExpr.ilike("%test%") + +// @ts-expect-error — .notLike() requires Expr +numExpr.notLike("%test%") + +// --------------------------------------------------------------------------- +// ColumnRef .get() — only valid for Map columns +// --------------------------------------------------------------------------- + +const mapRef = makeColumnRef<"Attrs", CHMap>("Attrs") +expectTypeOf(mapRef.get("key")).toMatchTypeOf>() + +const strRef = makeColumnRef<"Name", CHString>("Name") +// @ts-expect-error — .get() requires a Map column type +strRef.get("key") + +const numRef = makeColumnRef<"Score", CHFloat64>("Score") +// @ts-expect-error — .get() requires a Map column type +numRef.get("key") + +// --------------------------------------------------------------------------- +// Type mismatch in comparisons +// --------------------------------------------------------------------------- + +// @ts-expect-error — cannot compare Expr with number +strExpr.eq(42) + +// @ts-expect-error — cannot compare Expr with string +numExpr.eq("hello") + +// --------------------------------------------------------------------------- +// Aggregate function input constraints +// --------------------------------------------------------------------------- + +// @ts-expect-error — avg requires Expr +CH.avg(CH.lit("hello")) + +// @ts-expect-error — sum requires Expr +CH.sum(CH.lit("hello")) + +// --------------------------------------------------------------------------- +// Param type safety +// --------------------------------------------------------------------------- + +expectTypeOf(CH.param.string("orgId")).toMatchTypeOf>() +expectTypeOf(CH.param.string("orgId")).toMatchTypeOf>() + +expectTypeOf(CH.param.int("limit")).toMatchTypeOf>() +expectTypeOf(CH.param.int("limit")).toMatchTypeOf>() + +expectTypeOf(CH.param.dateTime("start")).toMatchTypeOf>() +expectTypeOf(CH.param.dateTime("start")).toMatchTypeOf>() + +// Param name is captured as a literal type +expectTypeOf(CH.param.string("orgId")._paramName).toEqualTypeOf<"orgId">() +expectTypeOf(CH.param.int("limit")._paramName).toEqualTypeOf<"limit">() + +// --------------------------------------------------------------------------- +// mapContains / mapGet / inList — return types +// --------------------------------------------------------------------------- + +expectTypeOf(CH.mapContains(mapRef, "key")).toMatchTypeOf() +expectTypeOf(CH.mapGet(mapRef, "key")).toMatchTypeOf>() +expectTypeOf(CH.inList(strExpr, ["a", "b"])).toMatchTypeOf() + +// --------------------------------------------------------------------------- +// Array constructors +// --------------------------------------------------------------------------- + +expectTypeOf(CH.arrayOf(CH.lit("a"), CH.lit("b"))).toMatchTypeOf>>() +expectTypeOf(CH.arrayOf(CH.lit(1), CH.lit(2))).toMatchTypeOf>>() diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts new file mode 100644 index 0000000000..3f786be873 --- /dev/null +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -0,0 +1,130 @@ +// --------------------------------------------------------------------------- +// Shared query helpers +// +// Reusable expression builders and WHERE condition helpers used across +// traces, alerts, services, and metrics queries. +// --------------------------------------------------------------------------- + +import type { AttributeFilter } from "../../query-engine" +import * as CH from "../expr" +import { param } from "../param" +import type { ColumnAccessor } from "../query" +import type { Traces } from "../tables" +import { buildAttrFilterCondition } from "../../traces-shared" + +// --------------------------------------------------------------------------- +// APDEX expressions +// --------------------------------------------------------------------------- + +/** + * Build the standard APDEX aggregation expressions (satisfiedCount, + * toleratingCount, apdexScore) from a duration expression and threshold. + * + * @param durationMs - An expression representing span duration in milliseconds + * (typically `$.Duration.div(1000000)`) + * @param thresholdMs - The APDEX "T" threshold in milliseconds + */ +export function apdexExprs(durationMs: CH.Expr, thresholdMs: number) { + const satisfied = CH.countIf(durationMs.lt(thresholdMs)) + const tolerating = CH.countIf(durationMs.gte(thresholdMs).and(durationMs.lt(thresholdMs * 4))) + return { + satisfiedCount: satisfied, + toleratingCount: tolerating, + apdexScore: CH.if_( + CH.count().gt(0), + CH.round_( + satisfied.add(tolerating.mul(0.5)).div(CH.count()), + 4, + ), + CH.lit(0), + ), + } +} + +// --------------------------------------------------------------------------- +// Traces base WHERE conditions +// --------------------------------------------------------------------------- + +interface TracesMatchModes { + serviceName?: "contains" + spanName?: "contains" + deploymentEnv?: "contains" +} + +export interface TracesBaseWhereOpts { + serviceName?: string + spanName?: string + rootOnly?: boolean + errorsOnly?: boolean + environments?: readonly string[] + commitShas?: readonly string[] + attributeFilters?: readonly AttributeFilter[] + resourceAttributeFilters?: readonly AttributeFilter[] + matchModes?: TracesMatchModes + minDurationMs?: number + maxDurationMs?: number +} + +/** + * Build the WHERE conditions shared between traces queries and alert queries: + * OrgId, Timestamp range, serviceName, spanName, rootOnly, errorsOnly, + * environments, commitShas, attribute filters, duration filters, and + * optional "contains" match modes. + * + * Alert queries omit matchModes and duration filters — they just don't pass them. + */ +export function tracesBaseWhereConditions( + $: ColumnAccessor, + opts: TracesBaseWhereOpts, +): Array { + const mm = opts.matchModes + const conditions: Array = [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (v: string) => + mm?.serviceName === "contains" + ? CH.positionCaseInsensitive($.ServiceName, CH.lit(v)).gt(0) + : $.ServiceName.eq(v), + ), + CH.when(opts.spanName, (v: string) => + mm?.spanName === "contains" + ? CH.positionCaseInsensitive($.SpanName, CH.lit(v)).gt(0) + : $.SpanName.eq(v), + ), + CH.whenTrue(!!opts.rootOnly, () => + $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), + ), + CH.whenTrue(!!opts.errorsOnly, () => $.StatusCode.eq("Error")), + ] + + if (opts.minDurationMs != null) { + conditions.push($.Duration.gte(opts.minDurationMs * 1000000)) + } + if (opts.maxDurationMs != null) { + conditions.push($.Duration.lte(opts.maxDurationMs * 1000000)) + } + + if (opts.environments?.length) { + if (mm?.deploymentEnv === "contains" && opts.environments.length === 1) { + conditions.push(CH.positionCaseInsensitive($.ResourceAttributes.get("deployment.environment"), CH.lit(opts.environments[0])).gt(0)) + } else { + conditions.push(CH.inList($.ResourceAttributes.get("deployment.environment"), opts.environments)) + } + } + if (opts.commitShas?.length) { + conditions.push(CH.inList($.ResourceAttributes.get("deployment.commit_sha"), opts.commitShas)) + } + if (opts.attributeFilters) { + for (const af of opts.attributeFilters) { + conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) + } + } + if (opts.resourceAttributeFilters) { + for (const rf of opts.resourceAttributeFilters) { + conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) + } + } + + return conditions +} diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index c46afbb7c1..dd9806d63a 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -9,10 +9,8 @@ import * as CH from "../expr" import { param } from "../param" import { from, type ColumnAccessor } from "../query" import { Traces } from "../tables" -import { - METRIC_NEEDS, - buildAttrFilterCondition, -} from "../../traces-shared" +import { METRIC_NEEDS } from "../../traces-shared" +import { apdexExprs, tracesBaseWhereConditions, type TracesBaseWhereOpts } from "./query-helpers" // --------------------------------------------------------------------------- // Metric SELECT expressions @@ -28,43 +26,22 @@ function metricSelectExprs( const needs = allMetrics ? new Set(["count", "avg_duration", "quantiles", "error_rate", "apdex"]) : new Set(METRIC_NEEDS[metric]) - const t = String(apdexThresholdMs) + const durationMs = $.Duration.div(1000000) + + const apdex = needs.has("apdex") + ? apdexExprs(durationMs, apdexThresholdMs) + : { satisfiedCount: CH.lit(0), toleratingCount: CH.lit(0), apdexScore: CH.lit(0) } return { count: CH.count(), - avgDuration: needs.has("avg_duration") - ? CH.avg($.Duration).div(1000000) - : CH.lit(0), - p50Duration: needs.has("quantiles") - ? CH.quantile(0.5)($.Duration).div(1000000) - : CH.lit(0), - p95Duration: needs.has("quantiles") - ? CH.quantile(0.95)($.Duration).div(1000000) - : CH.lit(0), - p99Duration: needs.has("quantiles") - ? CH.quantile(0.99)($.Duration).div(1000000) - : CH.lit(0), + avgDuration: needs.has("avg_duration") ? CH.avg(durationMs) : CH.lit(0), + p50Duration: needs.has("quantiles") ? CH.quantile(0.5)($.Duration).div(1000000) : CH.lit(0), + p95Duration: needs.has("quantiles") ? CH.quantile(0.95)($.Duration).div(1000000) : CH.lit(0), + p99Duration: needs.has("quantiles") ? CH.quantile(0.99)($.Duration).div(1000000) : CH.lit(0), errorRate: needs.has("error_rate") ? CH.if_(CH.count().gt(0), CH.countIf($.StatusCode.eq("Error")).mul(100.0).div(CH.count()), CH.lit(0)) : CH.lit(0), - satisfiedCount: needs.has("apdex") - ? CH.countIf($.Duration.div(1000000).lt(Number(t))) - : CH.lit(0), - toleratingCount: needs.has("apdex") - ? CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))) - : CH.lit(0), - apdexScore: needs.has("apdex") - ? CH.if_( - CH.count().gt(0), - CH.round_( - CH.countIf($.Duration.div(1000000).lt(Number(t))) - .add(CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))).mul(0.5)) - .div(CH.count()), - 4, - ), - CH.lit(0), - ) - : CH.lit(0), + ...apdex, sampledSpanCount: needsSampling ? CH.countIf($.TraceState.like("%th:%")) : CH.lit(0), @@ -170,88 +147,14 @@ function buildWhereConditions( $: ColumnAccessor, opts: TracesQueryOpts, ): Array { - const mm = opts.matchModes - const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTime("startTime")), - $.Timestamp.lte(param.dateTime("endTime")), - CH.when(opts.serviceName, (v: string) => - mm?.serviceName === "contains" - ? CH.positionCaseInsensitive($.ServiceName, CH.lit(v)).gt(0) - : $.ServiceName.eq(v), - ), - CH.when(opts.spanName, (v: string) => - mm?.spanName === "contains" - ? CH.positionCaseInsensitive($.SpanName, CH.lit(v)).gt(0) - : $.SpanName.eq(v), - ), - CH.whenTrue(!!opts.rootOnly, () => - $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), - ), - ] - - // Duration filters (Duration column is nanoseconds) - if (opts.minDurationMs != null) { - conditions.push($.Duration.gte(opts.minDurationMs * 1000000)) - } - if (opts.maxDurationMs != null) { - conditions.push($.Duration.lte(opts.maxDurationMs * 1000000)) - } - - if (opts.errorsOnly) { - conditions.push($.StatusCode.eq("Error")) - } - - if (opts.environments?.length) { - if (mm?.deploymentEnv === "contains" && opts.environments.length === 1) { - conditions.push(CH.positionCaseInsensitive($.ResourceAttributes.get("deployment.environment"), CH.lit(opts.environments[0])).gt(0)) - } else { - conditions.push(CH.inList($.ResourceAttributes.get("deployment.environment"), opts.environments)) - } - } - - if (opts.commitShas?.length) { - conditions.push(CH.inList($.ResourceAttributes.get("deployment.commit_sha"), opts.commitShas)) - } - - if (opts.attributeFilters) { - for (const af of opts.attributeFilters) { - conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) - } - } - - if (opts.resourceAttributeFilters) { - for (const rf of opts.resourceAttributeFilters) { - conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) - } - } - - return conditions + return tracesBaseWhereConditions($, opts) } // --------------------------------------------------------------------------- // Shared options interface // --------------------------------------------------------------------------- -interface TracesMatchModes { - serviceName?: "contains" - spanName?: "contains" - deploymentEnv?: "contains" -} - -interface TracesQueryOpts { - serviceName?: string - spanName?: string - rootOnly?: boolean - errorsOnly?: boolean - environments?: readonly string[] - commitShas?: readonly string[] - minDurationMs?: number - maxDurationMs?: number - matchModes?: TracesMatchModes - attributeFilters?: readonly AttributeFilter[] - resourceAttributeFilters?: readonly AttributeFilter[] -} +interface TracesQueryOpts extends TracesBaseWhereOpts {} // --------------------------------------------------------------------------- // Timeseries query diff --git a/packages/query-engine/src/ch/query.test-d.ts b/packages/query-engine/src/ch/query.test-d.ts new file mode 100644 index 0000000000..6bb58740ad --- /dev/null +++ b/packages/query-engine/src/ch/query.test-d.ts @@ -0,0 +1,347 @@ +// --------------------------------------------------------------------------- +// Type-level tests: Query builder, joins, subqueries, union, compilation +// --------------------------------------------------------------------------- + +import { expectTypeOf } from "expect-type" +import * as CH from "./index" +import type { InferQueryOutput } from "./query" +import type { InferUnionOutput } from "./union" +import type { CompiledQuery } from "./compile" +import type { Expr } from "./expr" + +// --------------------------------------------------------------------------- +// Test fixtures +// --------------------------------------------------------------------------- + +const Users = CH.table("users", { + Id: CH.string, + Name: CH.string, + Age: CH.uint64, + Score: CH.float64, + Attrs: CH.map(CH.string, CH.string), + Tags: CH.array(CH.string), + CreatedAt: CH.dateTime64, +}) + +const Orders = CH.table("orders", { + Id: CH.string, + UserId: CH.string, + Amount: CH.uint64, + Status: CH.string, +}) + +const Tags = CH.table("tags", { + Id: CH.string, + UserId: CH.string, + Label: CH.string, +}) + +// --------------------------------------------------------------------------- +// Select — callback overload infers Output from returned record +// --------------------------------------------------------------------------- + +const q1 = CH.from(Users).select(($) => ({ + bucket: CH.toStartOfInterval($.CreatedAt, 60), + count: CH.count(), + avgScore: CH.avg($.Score), +})) + +type Q1Output = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly bucket: string + readonly count: number + readonly avgScore: number +}>() + +// --------------------------------------------------------------------------- +// Select — shorthand overload infers Output from column names +// --------------------------------------------------------------------------- + +const q2 = CH.from(Users).select("Id", "Name") + +type Q2Output = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly Id: string + readonly Name: string +}>() + +const q3 = CH.from(Users).select("Id", "Age", "Score") + +type Q3Output = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly Id: string + readonly Age: number + readonly Score: number +}>() + +// --------------------------------------------------------------------------- +// Select — shorthand rejects invalid column names +// --------------------------------------------------------------------------- + +// @ts-expect-error — "NonExistent" is not a column on Users +CH.from(Users).select("Id", "NonExistent") + +// --------------------------------------------------------------------------- +// Column accessor types inside select callback +// --------------------------------------------------------------------------- + +CH.from(Users).select(($) => { + expectTypeOf($.Id).toMatchTypeOf>() + expectTypeOf($.Age).toMatchTypeOf>() + expectTypeOf($.Score).toMatchTypeOf>() + expectTypeOf($.Attrs).toMatchTypeOf>>() + expectTypeOf($.Tags).toMatchTypeOf>>() + expectTypeOf($.CreatedAt).toMatchTypeOf>() + return { id: $.Id } +}) + +// --------------------------------------------------------------------------- +// Chaining preserves Output type through where/groupBy/orderBy/limit/format +// --------------------------------------------------------------------------- + +const full = CH.from(Users) + .select(($) => ({ + name: $.Name, + count: CH.count(), + })) + .where(($) => [$.Id.eq("test")]) + .groupBy("name") + .orderBy(["count", "desc"]) + .limit(10) + .format("JSON") + +type FullOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly name: string + readonly count: number +}>() + +// --------------------------------------------------------------------------- +// groupBy and orderBy only accept keys from Output +// --------------------------------------------------------------------------- + +const qWithSelect = CH.from(Users).select(($) => ({ + name: $.Name, + count: CH.count(), +})) + +// Valid — keys exist in Output +qWithSelect.groupBy("name", "count") +qWithSelect.orderBy(["name", "asc"], ["count", "desc"]) + +// @ts-expect-error — "bogus" is not a key of Output +qWithSelect.groupBy("bogus") + +// @ts-expect-error — "bogus" is not a key of Output +qWithSelect.orderBy(["bogus", "asc"]) + +// --------------------------------------------------------------------------- +// innerJoin — adds typed columns under alias +// --------------------------------------------------------------------------- + +const joined = CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + +const jq = joined.select(($) => ({ + userName: $.Name, + orderAmount: $.o.Amount, + orderStatus: $.o.Status, +})) + +type JQOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly userName: string + readonly orderAmount: number + readonly orderStatus: string +}>() + +// --------------------------------------------------------------------------- +// leftJoin — wraps joined columns with | null +// --------------------------------------------------------------------------- + +const leftJoined = CH.from(Users) + .leftJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + +const ljq = leftJoined.select(($) => ({ + userName: $.Name, + orderAmount: $.o.Amount, + orderStatus: $.o.Status, +})) + +type LJQOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly userName: string + readonly orderAmount: number | null + readonly orderStatus: string | null +}>() + +// --------------------------------------------------------------------------- +// Multiple joins accumulate +// --------------------------------------------------------------------------- + +const multi = CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + .innerJoin(Tags, "t", (u, t) => u.Id.eq(t.UserId)) + .select(($) => ({ + name: $.Name, + amount: $.o.Amount, + label: $.t.Label, + })) + +type MultiOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly name: string + readonly amount: number + readonly label: string +}>() + +// --------------------------------------------------------------------------- +// crossJoin — no nullable wrapping +// --------------------------------------------------------------------------- + +const crossed = CH.from(Users) + .crossJoin(Orders, "o") + .select(($) => ({ + userName: $.Name, + orderAmount: $.o.Amount, + })) + +type CrossOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly userName: string + readonly orderAmount: number +}>() + +// --------------------------------------------------------------------------- +// innerJoinQuery — join with subquery +// --------------------------------------------------------------------------- + +const subquery = CH.from(Orders).select(($) => ({ + userId: $.UserId, + totalAmount: CH.sum($.Amount), +})) + +const joinedSub = CH.from(Users) + .innerJoinQuery(subquery, "agg", (u, agg) => u.Id.eq(agg.userId)) + .select(($) => ({ + name: $.Name, + total: $.agg.totalAmount, + })) + +type JoinedSubOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly name: string + readonly total: number +}>() + +// --------------------------------------------------------------------------- +// leftJoinQuery — nullable subquery join +// --------------------------------------------------------------------------- + +const leftJoinedSub = CH.from(Users) + .leftJoinQuery(subquery, "agg", (u, agg) => u.Id.eq(agg.userId)) + .select(($) => ({ + name: $.Name, + total: $.agg.totalAmount, + })) + +type LeftJoinedSubOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly name: string + readonly total: number | null +}>() + +// --------------------------------------------------------------------------- +// fromQuery — type-safe FROM subquery +// --------------------------------------------------------------------------- + +const inner = CH.from(Users).select(($) => ({ + userId: $.Id, + userName: $.Name, + userAge: $.Age, +})) + +const outer = CH.fromQuery(inner, "sub").select(($) => ({ + id: $.userId, + name: $.userName, +})) + +type OuterOutput = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly id: string + readonly name: string +}>() + +// --------------------------------------------------------------------------- +// unionAll — preserves shared Output type +// --------------------------------------------------------------------------- + +const uq1 = CH.from(Users).select(($) => ({ + name: $.Name, + count: CH.count(), +})) + +const uq2 = CH.from(Users).select(($) => ({ + name: $.Id, + count: CH.count(), +})) + +const union = CH.unionAll(uq1, uq2) +type UnionOutput = InferUnionOutput +expectTypeOf().toEqualTypeOf<{ + readonly name: string + readonly count: number +}>() + +// Union orderBy accepts Output keys +union.orderBy(["count", "desc"]) + +// @ts-expect-error — "bogus" is not in union Output +union.orderBy(["bogus", "asc"]) + +// --------------------------------------------------------------------------- +// unionAll — mismatched Output types should fail +// --------------------------------------------------------------------------- + +const qA = CH.from(Users).select(($) => ({ + name: $.Name, + count: CH.count(), +})) + +const qB = CH.from(Users).select(($) => ({ + name: $.Name, + extra: $.Id, +})) + +// @ts-expect-error — Output types don't match (count vs extra) +CH.unionAll(qA, qB) + +// --------------------------------------------------------------------------- +// compileCH — output type matches query Output +// --------------------------------------------------------------------------- + +const compileTarget = CH.from(Users).select(($) => ({ + id: $.Id, + age: $.Age, +})) + +const compiled = CH.compile(compileTarget, {}) + +expectTypeOf(compiled).toMatchTypeOf< + CompiledQuery<{ readonly id: string; readonly age: number }> +>() + +// castRows returns correctly typed array +expectTypeOf(compiled.castRows([])).toEqualTypeOf< + ReadonlyArray<{ readonly id: string; readonly age: number }> +>() + +// --------------------------------------------------------------------------- +// InferQueryOutput utility type +// --------------------------------------------------------------------------- + +type Extracted = InferQueryOutput +expectTypeOf().toEqualTypeOf<{ + readonly id: string + readonly age: number +}>() diff --git a/packages/query-engine/src/ch/types.test-d.ts b/packages/query-engine/src/ch/types.test-d.ts new file mode 100644 index 0000000000..8394401742 --- /dev/null +++ b/packages/query-engine/src/ch/types.test-d.ts @@ -0,0 +1,93 @@ +// --------------------------------------------------------------------------- +// Type-level tests: Phantom type inference +// --------------------------------------------------------------------------- + +import { expectTypeOf } from "expect-type" +import type { + CHString, + CHUInt8, + CHUInt16, + CHUInt32, + CHUInt64, + CHInt32, + CHFloat64, + CHDateTime, + CHDateTime64, + CHBool, + CHMap, + CHArray, + CHNullable, + InferTS, + OutputToColumnDefs, + NullableColumnDefs, +} from "./types" + +// --------------------------------------------------------------------------- +// Primitive type inference +// --------------------------------------------------------------------------- + +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() + +// --------------------------------------------------------------------------- +// Compound type inference +// --------------------------------------------------------------------------- + +expectTypeOf>>().toEqualTypeOf>() +expectTypeOf>>().toEqualTypeOf>() +expectTypeOf>>().toEqualTypeOf>() +expectTypeOf>>().toEqualTypeOf>() +expectTypeOf>>().toEqualTypeOf() +expectTypeOf>>().toEqualTypeOf() + +// Nested compound types +expectTypeOf>>>().toEqualTypeOf< + ReadonlyArray> +>() +expectTypeOf>>>().toEqualTypeOf | null>() + +// --------------------------------------------------------------------------- +// OutputToColumnDefs — converts output record to synthetic column defs +// --------------------------------------------------------------------------- + +type TestOutput = { name: string; count: number; active: boolean } +type Converted = OutputToColumnDefs + +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() + +// --------------------------------------------------------------------------- +// NullableColumnDefs — wraps each column type with | null +// --------------------------------------------------------------------------- + +type TestCols = { Name: CHString; Score: CHFloat64; Active: CHBool } +type Nulled = NullableColumnDefs + +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() + +// --------------------------------------------------------------------------- +// Negative cases — wrong type inferences should fail +// --------------------------------------------------------------------------- + +// @ts-expect-error — CHString infers string, not number +expectTypeOf>().toEqualTypeOf() + +// @ts-expect-error — CHFloat64 infers number, not string +expectTypeOf>().toEqualTypeOf() + +// @ts-expect-error — CHBool infers boolean, not number +expectTypeOf>().toEqualTypeOf() + +// @ts-expect-error — CHNullable infers string | null, not just string +expectTypeOf>>().toEqualTypeOf() From 4a07f1ee2e1c0faf414f1a148744ad004100ad51 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 14:14:08 +0200 Subject: [PATCH 10/11] fix stuff --- apps/api/src/services/QueryEngineService.ts | 193 ++++++++---------- apps/api/src/services/TinybirdService.ts | 10 +- packages/domain/src/http/query-engine.ts | 5 +- packages/domain/src/http/tinybird.ts | 2 +- .../query-engine/src/ch/queries/alerts.ts | 182 +++-------------- .../query-engine/src/ch/queries/metrics.ts | 125 ++---------- .../src/ch/queries/query-helpers.ts | 58 +++++- .../query-engine/src/ch/queries/services.ts | 16 +- .../query-engine/src/ch/queries/traces.ts | 4 +- 9 files changed, 195 insertions(+), 400 deletions(-) diff --git a/apps/api/src/services/QueryEngineService.ts b/apps/api/src/services/QueryEngineService.ts index 73461ea698..fcc9099708 100644 --- a/apps/api/src/services/QueryEngineService.ts +++ b/apps/api/src/services/QueryEngineService.ts @@ -498,6 +498,61 @@ function resolveAttributeScope( return scope === "resource" ? "resource" : "span" } +type AttrFilterArray = Array<{ + key: string + value?: string + mode: "equals" | "exists" | "gt" | "gte" | "lt" | "lte" | "contains" +}> + +function extractTracesOpts(filters: Record | undefined) { + return { + serviceName: filters?.serviceName as string | undefined, + spanName: filters?.spanName as string | undefined, + rootOnly: filters?.rootSpansOnly as boolean | undefined, + errorsOnly: filters?.errorsOnly as boolean | undefined, + environments: filters?.environments as string[] | undefined, + commitShas: filters?.commitShas as string[] | undefined, + minDurationMs: filters?.minDurationMs as number | undefined, + maxDurationMs: filters?.maxDurationMs as number | undefined, + matchModes: filters?.matchModes as { + serviceName?: "contains" + spanName?: "contains" + deploymentEnv?: "contains" + } | undefined, + attributeFilters: filters?.attributeFilters as AttrFilterArray | undefined, + resourceAttributeFilters: filters?.resourceAttributeFilters as AttrFilterArray | undefined, + groupByAttributeKeys: filters?.groupByAttributeKeys as string[] | undefined, + } +} + +function shapeMetricsGroupRows( + rows: ReadonlyArray, + valueExtractor: (row: T) => number, + groupBy: readonly string[] | undefined, + groupByAttributeKey: string | undefined, + fillOptions: BucketFillOptions | undefined, +): Array { + if (groupBy?.includes("none") || !groupBy?.length) { + return groupTimeSeriesRows( + rows.map((row) => ({ bucket: row.bucket, groupName: "all" as const, value: valueExtractor(row) })), + (r) => r.value, + fillOptions, + ) + } + if (groupByAttributeKey) { + return groupTimeSeriesRows( + rows.map((row) => ({ bucket: row.bucket, groupName: row.attributeValue || "(empty)", value: valueExtractor(row) })), + (r) => r.value, + fillOptions, + ) + } + return groupTimeSeriesRows( + rows.map((row) => ({ bucket: row.bucket, groupName: row.serviceName, value: valueExtractor(row) })), + (r) => r.value, + fillOptions, + ) +} + const isScalarAlertQuery = ( query: QuerySpec, ): query is Extract => { @@ -550,22 +605,15 @@ export const makeQueryEngineExecute = (tinybird: QueryEngineTinybird) => : undefined if (request.query.source === "traces" && request.query.kind === "timeseries") { + const opts = extractTracesOpts(request.query.filters as Record) const rows = yield* executeCHQuery( tinybird, tenant, CH.tracesTimeseriesQuery({ + ...opts, metric: request.query.metric, needsSampling: false, - serviceName: request.query.filters?.serviceName, - spanName: request.query.filters?.spanName, - rootOnly: request.query.filters?.rootSpansOnly, - errorsOnly: request.query.filters?.errorsOnly, groupBy: request.query.groupBy as string[] | undefined, - groupByAttributeKeys: request.query.filters?.groupByAttributeKeys as string[] | undefined, - environments: request.query.filters?.environments as string[] | undefined, - commitShas: request.query.filters?.commitShas as string[] | undefined, - attributeFilters: request.query.filters?.attributeFilters as Array<{ key: string; value?: string; mode: "equals" | "exists" }> | undefined, - resourceAttributeFilters: request.query.filters?.resourceAttributeFilters as Array<{ key: string; value?: string; mode: "equals" | "exists" }> | undefined, apdexThresholdMs: request.query.metric === "apdex" ? request.query.apdexThresholdMs : undefined, }), { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime, bucketSeconds: bucketSeconds! }, @@ -645,35 +693,13 @@ export const makeQueryEngineExecute = (tinybird: QueryEngineTinybird) => const rateValueField = request.query.metric === "rate" ? "rateValue" : "increaseValue" - const data = (request.query.groupBy?.includes("none") || !request.query.groupBy?.length) - ? groupTimeSeriesRows( - rateResult.map((row) => ({ - bucket: row.bucket, - groupName: "all" as const, - value: Number(row[rateValueField]), - })), - (row) => row.value, - fillOptions, - ) - : groupByAttributeKey - ? groupTimeSeriesRows( - rateResult.map((row) => ({ - bucket: row.bucket, - groupName: row.attributeValue || "(empty)", - value: Number(row[rateValueField]), - })), - (row) => row.value, - fillOptions, - ) - : groupTimeSeriesRows( - rateResult.map((row) => ({ - bucket: row.bucket, - groupName: row.serviceName, - value: Number(row[rateValueField]), - })), - (row) => row.value, - fillOptions, - ) + const data = shapeMetricsGroupRows( + rateResult, + (row) => Number(row[rateValueField]), + request.query.groupBy, + groupByAttributeKey, + fillOptions, + ) return new QueryEngineExecuteResponse({ result: { @@ -719,25 +745,13 @@ export const makeQueryEngineExecute = (tinybird: QueryEngineTinybird) => (row) => row.value, fillOptions, ) - : groupByAttributeKey - ? groupTimeSeriesRows( - result.map((row) => ({ - bucket: row.bucket, - groupName: row.attributeValue || "(empty)", - value: Number(row[valueField]), - })), - (row) => row.value, - fillOptions, - ) - : groupTimeSeriesRows( - result.map((row) => ({ - bucket: row.bucket, - groupName: row.serviceName, - value: Number(row[valueField]), - })), - (row) => row.value, - fillOptions, - ) + : shapeMetricsGroupRows( + result, + (row) => Number(row[valueField]), + request.query.groupBy, + groupByAttributeKey, + fillOptions, + ) return new QueryEngineExecuteResponse({ result: { @@ -749,25 +763,19 @@ export const makeQueryEngineExecute = (tinybird: QueryEngineTinybird) => } if (request.query.source === "traces" && request.query.kind === "breakdown") { + const opts = extractTracesOpts(request.query.filters as Record) const rows = yield* executeCHQuery( tinybird, tenant, CH.tracesBreakdownQuery({ + ...opts, metric: request.query.metric, groupBy: request.query.groupBy, groupByAttributeKey: request.query.groupBy === "attribute" - ? (request.query.filters?.groupByAttributeKeys as string[] | undefined)?.[0] + ? opts.groupByAttributeKeys?.[0] : undefined, limit: request.query.limit, - serviceName: request.query.filters?.serviceName, - spanName: request.query.filters?.spanName, - rootOnly: request.query.filters?.rootSpansOnly, - errorsOnly: request.query.filters?.errorsOnly, - environments: request.query.filters?.environments as string[] | undefined, - commitShas: request.query.filters?.commitShas as string[] | undefined, - attributeFilters: request.query.filters?.attributeFilters as Array<{ key: string; value?: string; mode: "equals" | "exists" }> | undefined, - resourceAttributeFilters: request.query.filters?.resourceAttributeFilters as Array<{ key: string; value?: string; mode: "equals" | "exists" }> | undefined, apdexThresholdMs: request.query.metric === "apdex" ? request.query.apdexThresholdMs : undefined, }), { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime }, @@ -850,38 +858,22 @@ export const makeQueryEngineExecute = (tinybird: QueryEngineTinybird) => } if (request.query.source === "traces" && request.query.kind === "list") { - type AttrFilterArray = Array<{ key: string; value?: string; mode: "equals" | "exists" | "gt" | "gte" | "lt" | "lte" | "contains" }> - const filters = request.query.filters + const opts = extractTracesOpts(request.query.filters as Record) // Graceful limit clamping: cap at 200, auto-reduce to 50 when no indexed filters - const hasIndexedFilter = !!(filters?.serviceName || filters?.spanName || filters?.errorsOnly || filters?.rootSpansOnly) + const hasIndexedFilter = !!(opts.serviceName || opts.spanName || opts.errorsOnly || opts.rootOnly) const maxLimit = hasIndexedFilter ? 200 : 50 const clampedLimit = Math.min(request.query.limit ?? 25, maxLimit) - const sharedOpts = { - limit: clampedLimit, - offset: request.query.offset, - serviceName: filters?.serviceName, - spanName: filters?.spanName, - rootOnly: filters?.rootSpansOnly, - errorsOnly: filters?.errorsOnly, - environments: filters?.environments as string[] | undefined, - commitShas: filters?.commitShas as string[] | undefined, - minDurationMs: filters?.minDurationMs, - maxDurationMs: filters?.maxDurationMs, - matchModes: filters?.matchModes as { serviceName?: "contains"; spanName?: "contains"; deploymentEnv?: "contains" } | undefined, - attributeFilters: filters?.attributeFilters as AttrFilterArray | undefined, - resourceAttributeFilters: filters?.resourceAttributeFilters as AttrFilterArray | undefined, - columns: (request.query as { columns?: readonly string[] }).columns as string[] | undefined, - } - - // Always use tracesListQuery — it handles MV usage when rootOnly=true - // and returns a consistent field shape for dashboard widgets. - // (tracesRootListQuery has a different output shape for the built-in trace list page) const rows = yield* executeCHQuery( tinybird, tenant, - CH.tracesListQuery(sharedOpts), + CH.tracesListQuery({ + ...opts, + limit: clampedLimit, + offset: request.query.offset, + columns: (request.query as { columns?: readonly string[] }).columns as string[] | undefined, + }), { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime }, "Failed to execute traces list query", ) @@ -972,19 +964,12 @@ export const makeQueryEngineEvaluate = (tinybird: QueryEngineTinybird) => let observations: ReadonlyArray if (request.query.source === "traces") { - type AttrFilterArray = Array<{ key: string; value?: string; mode: "equals" | "exists" | "gt" | "gte" | "lt" | "lte" | "contains" }> + const opts = extractTracesOpts(request.query.filters as Record) const rows = yield* executeCHQuery( tinybird, tenant, CH.alertTracesAggregateQuery({ - serviceName: request.query.filters?.serviceName, - spanName: request.query.filters?.spanName, - rootOnly: request.query.filters?.rootSpansOnly, - errorsOnly: request.query.filters?.errorsOnly, - environments: request.query.filters?.environments as string[] | undefined, - commitShas: request.query.filters?.commitShas as string[] | undefined, - attributeFilters: request.query.filters?.attributeFilters as AttrFilterArray | undefined, - resourceAttributeFilters: request.query.filters?.resourceAttributeFilters as AttrFilterArray | undefined, + ...opts, apdexThresholdMs: request.query.metric === "apdex" ? request.query.apdexThresholdMs : undefined, }), { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime }, @@ -1097,19 +1082,13 @@ export const makeQueryEngineEvaluateGrouped = (tinybird: QueryEngineTinybird) => if (groupBy === "service") { if (request.query.source === "traces") { - type AttrFilterArray = Array<{ key: string; value?: string; mode: "equals" | "exists" | "gt" | "gte" | "lt" | "lte" | "contains" }> const tracesQuery = request.query as Extract + const opts = extractTracesOpts(request.query.filters as Record) const rows = yield* executeCHQuery( tinybird, tenant, CH.alertTracesAggregateByServiceQuery({ - spanName: request.query.filters?.spanName, - rootOnly: request.query.filters?.rootSpansOnly, - errorsOnly: request.query.filters?.errorsOnly, - environments: request.query.filters?.environments as string[] | undefined, - commitShas: request.query.filters?.commitShas as string[] | undefined, - attributeFilters: request.query.filters?.attributeFilters as AttrFilterArray | undefined, - resourceAttributeFilters: request.query.filters?.resourceAttributeFilters as AttrFilterArray | undefined, + ...opts, apdexThresholdMs: tracesQuery.metric === "apdex" ? tracesQuery.apdexThresholdMs : undefined, }), { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime }, diff --git a/apps/api/src/services/TinybirdService.ts b/apps/api/src/services/TinybirdService.ts index 6fc9ef522c..df6acab35f 100644 --- a/apps/api/src/services/TinybirdService.ts +++ b/apps/api/src/services/TinybirdService.ts @@ -49,7 +49,7 @@ export class TinybirdService extends ServiceMap.Service + const toTinybirdQueryError = (pipe: string, error: unknown) => new TinybirdQueryError({ message: error instanceof Error ? error.message : "Tinybird query failed", pipe, @@ -75,7 +75,7 @@ export class TinybirdService extends ServiceMap.Service toTinybirdQueryError(pipe as TinybirdQueryRequest["pipe"], error))) + .pipe(Effect.mapError((error) => toTinybirdQueryError(pipe, error))) if (Option.isSome(override)) { yield* Effect.annotateCurrentSpan("clientSource", "org_override") @@ -101,7 +101,7 @@ export class TinybirdService extends ServiceMap.Service client.sql(sql), - catch: (error) => toTinybirdQueryError(pipe as TinybirdQueryRequest["pipe"], error), + catch: (error) => toTinybirdQueryError(pipe, error), }).pipe( Effect.tapError((error) => Effect.logError("TinybirdService.executeSql failed", { pipe, error: String(error) }), @@ -147,10 +147,10 @@ export class TinybirdService extends ServiceMap.Service()( "@maple/http/errors/QueryEngineValidationError", { @@ -20,7 +17,7 @@ export class QueryEngineExecutionError extends Schema.TaggedErrorClass, apdexThresholdMs: number) { - const t = apdexThresholdMs return { count: CH.count(), avgDuration: CH.avg($.Duration).div(1000000), @@ -62,18 +52,7 @@ function alertTracesSelectExprs($: ColumnAccessor, apdexT p95Duration: CH.quantile(0.95)($.Duration).div(1000000), p99Duration: CH.quantile(0.99)($.Duration).div(1000000), errorRate: CH.if_(CH.count().gt(0), CH.countIf($.StatusCode.eq("Error")).mul(100.0).div(CH.count()), CH.lit(0)), - satisfiedCount: CH.countIf($.Duration.div(1000000).lt(t)), - toleratingCount: CH.countIf($.Duration.div(1000000).gte(t).and($.Duration.div(1000000).lt(t * 4))), - apdexScore: CH.if_( - CH.count().gt(0), - CH.round_( - CH.countIf($.Duration.div(1000000).lt(t)) - .add(CH.countIf($.Duration.div(1000000).gte(t).and($.Duration.div(1000000).lt(t * 4))).mul(0.5)) - .div(CH.count()), - 4, - ), - CH.lit(0), - ), + ...apdexExprs($.Duration.div(1000000), apdexThresholdMs), } } @@ -81,36 +60,7 @@ function alertTracesWhereConditions( $: ColumnAccessor, opts: AlertTracesOpts, ): Array { - const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTime("startTime")), - $.Timestamp.lte(param.dateTime("endTime")), - CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), - CH.when(opts.spanName, (v: string) => $.SpanName.eq(v)), - CH.whenTrue(!!opts.rootOnly, () => - $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), - ), - CH.whenTrue(!!opts.errorsOnly, () => $.StatusCode.eq("Error")), - ] - - if (opts.environments?.length) { - conditions.push(CH.inList($.ResourceAttributes.get("deployment.environment"), opts.environments)) - } - if (opts.commitShas?.length) { - conditions.push(CH.inList($.ResourceAttributes.get("deployment.commit_sha"), opts.commitShas)) - } - if (opts.attributeFilters) { - for (const af of opts.attributeFilters) { - conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) - } - } - if (opts.resourceAttributeFilters) { - for (const rf of opts.resourceAttributeFilters) { - conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) - } - } - - return conditions + return tracesBaseWhereConditions($, opts) } export function alertTracesAggregateQuery( @@ -162,100 +112,36 @@ export interface AlertMetricsAggregateByServiceOutput extends AlertMetricsAggreg } -const VALUE_TABLES = { - sum: MetricsSum, - gauge: MetricsGauge, -} as const - -const HISTOGRAM_TABLES = { - histogram: MetricsHistogram, - exponential_histogram: MetricsExpHistogram, -} as const - -function buildValueMetricsAggregate( - opts: AlertMetricsOpts, -) { - const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] - - return from(tbl as typeof MetricsSum) - .select(($) => ({ - avgValue: CH.avg($.Value), - minValue: CH.min_($.Value), - maxValue: CH.max_($.Value), - sumValue: CH.sum($.Value), - dataPointCount: CH.count(), - })) - .where(($) => [ - $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), - $.TimeUnix.gte(param.dateTime("startTime")), - $.TimeUnix.lte(param.dateTime("endTime")), - CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), - ]) - .format("JSON") -} - -function buildHistogramMetricsAggregate( - opts: AlertMetricsOpts, -) { - const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] - - return from(tbl as typeof MetricsHistogram) - .select(($) => ({ - avgValue: CH.if_(CH.sum($.Count).gt(0), CH.sum($.Sum).div(CH.sum($.Count)), CH.lit(0)), - minValue: CH.min_($.Min), - maxValue: CH.max_($.Max), - sumValue: CH.sum($.Sum), - dataPointCount: CH.sum($.Count), - })) - .where(($) => [ - $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), - $.TimeUnix.gte(param.dateTime("startTime")), - $.TimeUnix.lte(param.dateTime("endTime")), - CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), - ]) - .format("JSON") +function metricsAlertWhere(opts: AlertMetricsOpts) { + return ($: ColumnAccessor) => [ + $.MetricName.eq(param.string("metricName")), + $.OrgId.eq(param.string("orgId")), + $.TimeUnix.gte(param.dateTime("startTime")), + $.TimeUnix.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), + ] as Array } -function buildValueMetricsAggregateByService( +export function alertMetricsAggregateQuery( opts: AlertMetricsOpts, ) { - const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] + const { tbl, isHistogram } = resolveMetricTable(opts.metricType) return from(tbl as typeof MetricsSum) - .select(($) => ({ - serviceName: $.ServiceName, - avgValue: CH.avg($.Value), - minValue: CH.min_($.Value), - maxValue: CH.max_($.Value), - sumValue: CH.sum($.Value), - dataPointCount: CH.count(), - })) - .where(($) => [ - $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), - $.TimeUnix.gte(param.dateTime("startTime")), - $.TimeUnix.lte(param.dateTime("endTime")), - ]) - .groupBy("serviceName") - .orderBy(["dataPointCount", "desc"]) + .select(($) => metricsSelectExprs($, isHistogram)) + .where(metricsAlertWhere(opts)) .format("JSON") } -function buildHistogramMetricsAggregateByService( +export function alertMetricsAggregateByServiceQuery( opts: AlertMetricsOpts, ) { - const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] + const { tbl, isHistogram } = resolveMetricTable(opts.metricType) - return from(tbl as typeof MetricsHistogram) + return from(tbl as typeof MetricsSum) .select(($) => ({ serviceName: $.ServiceName, - avgValue: CH.if_(CH.sum($.Count).gt(0), CH.sum($.Sum).div(CH.sum($.Count)), CH.lit(0)), - minValue: CH.min_($.Min), - maxValue: CH.max_($.Max), - sumValue: CH.sum($.Sum), - dataPointCount: CH.sum($.Count), + ...metricsSelectExprs($, isHistogram), })) .where(($) => [ $.MetricName.eq(param.string("metricName")), @@ -268,20 +154,6 @@ function buildHistogramMetricsAggregateByService( .format("JSON") } -export function alertMetricsAggregateQuery( - opts: AlertMetricsOpts, -) { - const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" - return isHistogram ? buildHistogramMetricsAggregate(opts) : buildValueMetricsAggregate(opts) -} - -export function alertMetricsAggregateByServiceQuery( - opts: AlertMetricsOpts, -) { - const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" - return isHistogram ? buildHistogramMetricsAggregateByService(opts) : buildValueMetricsAggregateByService(opts) -} - // --------------------------------------------------------------------------- // Logs alert aggregate // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index c67503a66e..5ac32890d8 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -19,20 +19,7 @@ import { MetricsExpHistogram, } from "../tables" import { compileCH } from "../compile" - -// --------------------------------------------------------------------------- -// Table lookup -// --------------------------------------------------------------------------- - -const VALUE_TABLES = { - sum: MetricsSum, - gauge: MetricsGauge, -} as const - -const HISTOGRAM_TABLES = { - histogram: MetricsHistogram, - exponential_histogram: MetricsExpHistogram, -} as const +import { resolveMetricTable, metricsSelectExprs } from "./query-helpers" // --------------------------------------------------------------------------- // Shared options & output types @@ -67,18 +54,7 @@ export interface MetricsTimeseriesOutput { export function metricsTimeseriesQuery( opts: MetricsTimeseriesOpts, ) { - const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" - - if (isHistogram) { - return buildHistogramTimeseries(opts) - } - return buildValueTimeseries(opts) -} - -function buildValueTimeseries( - opts: MetricsTimeseriesOpts, -) { - const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] + const { tbl, isHistogram } = resolveMetricTable(opts.metricType) const q = from(tbl as typeof MetricsSum) .select(($) => ({ @@ -87,48 +63,7 @@ function buildValueTimeseries( attributeValue: opts.groupByAttributeKey ? $.Attributes.get(opts.groupByAttributeKey) : CH.lit(""), - avgValue: CH.avg($.Value), - minValue: CH.min_($.Value), - maxValue: CH.max_($.Value), - sumValue: CH.sum($.Value), - dataPointCount: CH.count(), - })) - .where(($) => [ - $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), - $.TimeUnix.gte(param.dateTime("startTime")), - $.TimeUnix.lte(param.dateTime("endTime")), - CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), - CH.when(opts.attributeKey, (k: string) => - $.Attributes.get(k).eq(opts.attributeValue ?? ""), - ), - ]) - - return (opts.groupByAttributeKey - ? q.groupBy("bucket", "serviceName", "attributeValue") - : q.groupBy("bucket", "serviceName") - ) - .orderBy(["bucket", "asc"]) - .format("JSON") -} - -function buildHistogramTimeseries( - opts: MetricsTimeseriesOpts, -) { - const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] - - const q = from(tbl as typeof MetricsHistogram) - .select(($) => ({ - bucket: CH.toStartOfInterval($.TimeUnix, param.int("bucketSeconds")), - serviceName: $.ServiceName, - attributeValue: opts.groupByAttributeKey - ? $.Attributes.get(opts.groupByAttributeKey) - : CH.lit(""), - avgValue: CH.if_(CH.sum($.Count).gt(0), CH.sum($.Sum).div(CH.sum($.Count)), CH.lit(0)), - minValue: CH.min_($.Min), - maxValue: CH.max_($.Max), - sumValue: CH.sum($.Sum), - dataPointCount: CH.sum($.Count), + ...metricsSelectExprs($, isHistogram), })) .where(($) => [ $.MetricName.eq(param.string("metricName")), @@ -258,53 +193,19 @@ export interface MetricsBreakdownOutput { export function metricsBreakdownQuery( opts: MetricsBreakdownOpts, ) { - const isHistogram = opts.metricType === "histogram" || opts.metricType === "exponential_histogram" + const { tbl, isHistogram } = resolveMetricTable(opts.metricType) const limit = opts.limit ?? 10 - if (isHistogram) { - return buildHistogramBreakdown(opts, limit) - } - return buildValueBreakdown(opts, limit) -} - -function buildValueBreakdown( - opts: MetricsBreakdownOpts, - limit: number, -) { - const tbl = VALUE_TABLES[opts.metricType as keyof typeof VALUE_TABLES] - return from(tbl as typeof MetricsSum) - .select(($) => ({ - name: $.ServiceName, - avgValue: CH.avg($.Value), - sumValue: CH.sum($.Value), - count: CH.count(), - })) - .where(($) => [ - $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), - $.TimeUnix.gte(param.dateTime("startTime")), - $.TimeUnix.lte(param.dateTime("endTime")), - ]) - .groupBy("name") - .orderBy(["count", "desc"]) - .limit(limit) - .format("JSON") -} - -function buildHistogramBreakdown( - opts: MetricsBreakdownOpts, - limit: number, -) { - const tbl = HISTOGRAM_TABLES[opts.metricType as keyof typeof HISTOGRAM_TABLES] - - return from(tbl as typeof MetricsHistogram) - .select(($) => ({ - name: $.ServiceName, - avgValue: CH.if_(CH.sum($.Count).gt(0), CH.sum($.Sum).div(CH.sum($.Count)), CH.lit(0)), - sumValue: CH.sum($.Sum), - count: CH.sum($.Count), - })) + .select(($) => { + const exprs = metricsSelectExprs($, isHistogram) + return { + name: $.ServiceName, + avgValue: exprs.avgValue, + sumValue: exprs.sumValue, + count: exprs.dataPointCount, + } + }) .where(($) => [ $.MetricName.eq(param.string("metricName")), $.OrgId.eq(param.string("orgId")), diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index 3f786be873..499b81c30e 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -5,11 +5,17 @@ // traces, alerts, services, and metrics queries. // --------------------------------------------------------------------------- -import type { AttributeFilter } from "../../query-engine" +import type { AttributeFilter, MetricType } from "../../query-engine" import * as CH from "../expr" import { param } from "../param" import type { ColumnAccessor } from "../query" import type { Traces } from "../tables" +import { + MetricsSum, + MetricsGauge, + MetricsHistogram, + MetricsExpHistogram, +} from "../tables" import { buildAttrFilterCondition } from "../../traces-shared" // --------------------------------------------------------------------------- @@ -128,3 +134,53 @@ export function tracesBaseWhereConditions( return conditions } + +// --------------------------------------------------------------------------- +// Metrics table lookup + SELECT factory +// --------------------------------------------------------------------------- + +export const VALUE_TABLES = { + sum: MetricsSum, + gauge: MetricsGauge, +} as const + +export const HISTOGRAM_TABLES = { + histogram: MetricsHistogram, + exponential_histogram: MetricsExpHistogram, +} as const + +export function resolveMetricTable(metricType: MetricType) { + const isHistogram = metricType === "histogram" || metricType === "exponential_histogram" + const tbl = isHistogram + ? HISTOGRAM_TABLES[metricType as keyof typeof HISTOGRAM_TABLES] + : VALUE_TABLES[metricType as keyof typeof VALUE_TABLES] + return { tbl, isHistogram } +} + +/** + * Build the standard metrics aggregation SELECT expressions. + * For value tables (sum/gauge): operates on $.Value column. + * For histogram tables: operates on $.Sum, $.Count, $.Min, $.Max columns. + */ +export function metricsSelectExprs( + $: ColumnAccessor, + isHistogram: boolean, +) { + if (isHistogram) { + const $h = $ as unknown as ColumnAccessor + return { + avgValue: CH.if_(CH.sum($h.Count).gt(0), CH.sum($h.Sum).div(CH.sum($h.Count)), CH.lit(0)), + minValue: CH.min_($h.Min), + maxValue: CH.max_($h.Max), + sumValue: CH.sum($h.Sum), + dataPointCount: CH.sum($h.Count), + } + } + return { + avgValue: CH.avg($.Value), + minValue: CH.min_($.Value), + maxValue: CH.max_($.Value), + sumValue: CH.sum($.Value), + dataPointCount: CH.count(), + } +} diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index 5902165039..9ac1b10fc0 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -9,6 +9,7 @@ import { param } from "../param" import { from, type ColumnAccessor } from "../query" import { unionAll, type CHUnionQuery } from "../union" import { ServiceOverviewSpans, ServiceUsage, Traces } from "../tables" +import { apdexExprs } from "./query-helpers" // --------------------------------------------------------------------------- // Service overview @@ -125,24 +126,13 @@ export interface ServiceApdexTimeseriesOutput { export function serviceApdexTimeseriesQuery( opts: ServiceApdexTimeseriesOpts, ) { - const t = String(opts.apdexThresholdMs ?? 500) + const thresholdMs = opts.apdexThresholdMs ?? 500 return from(Traces) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, param.int("bucketSeconds")), totalCount: CH.count(), - satisfiedCount: CH.countIf($.Duration.div(1000000).lt(Number(t))), - toleratingCount: CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))), - apdexScore: CH.if_( - CH.count().gt(0), - CH.round_( - CH.countIf($.Duration.div(1000000).lt(Number(t))) - .add(CH.countIf($.Duration.div(1000000).gte(Number(t)).and($.Duration.div(1000000).lt(Number(t) * 4))).mul(0.5)) - .div(CH.count()), - 4, - ), - CH.lit(0), - ), + ...apdexExprs($.Duration.div(1000000), thresholdMs), })) .where(($) => [ $.SpanKind.in_("Server", "Consumer").or($.ParentSpanId.eq("")), diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index dd9806d63a..a3c61c493a 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -4,7 +4,7 @@ // DSL-based query definitions for traces timeseries, breakdown, and list. // --------------------------------------------------------------------------- -import type { TracesMetric, AttributeFilter } from "../../query-engine" +import type { TracesMetric } from "../../query-engine" import * as CH from "../expr" import { param } from "../param" import { from, type ColumnAccessor } from "../query" @@ -34,7 +34,7 @@ function metricSelectExprs( return { count: CH.count(), - avgDuration: needs.has("avg_duration") ? CH.avg(durationMs) : CH.lit(0), + avgDuration: needs.has("avg_duration") ? CH.avg($.Duration).div(1000000) : CH.lit(0), p50Duration: needs.has("quantiles") ? CH.quantile(0.5)($.Duration).div(1000000) : CH.lit(0), p95Duration: needs.has("quantiles") ? CH.quantile(0.95)($.Duration).div(1000000) : CH.lit(0), p99Duration: needs.has("quantiles") ? CH.quantile(0.99)($.Duration).div(1000000) : CH.lit(0), From 758829715b4de2171bcde1da21ea0971ea0b0078 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 6 Apr 2026 14:25:11 +0200 Subject: [PATCH 11/11] g --- apps/web/src/api/tinybird/logs.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/src/api/tinybird/logs.ts b/apps/web/src/api/tinybird/logs.ts index fcca4d5965..1c040f77f5 100644 --- a/apps/web/src/api/tinybird/logs.ts +++ b/apps/web/src/api/tinybird/logs.ts @@ -68,8 +68,8 @@ function transformLog(raw: ListLogsOutput): Log { severityNumber: Number(raw.severityNumber), serviceName: raw.serviceName, body: raw.body, - traceId: toTraceId(raw.traceId), - spanId: toSpanId(raw.spanId), + traceId: raw.traceId ? toTraceId(raw.traceId) : ("" as TraceId), + spanId: raw.spanId ? toSpanId(raw.spanId) : ("" as SpanId), logAttributes: parseAttributes(raw.logAttributes), resourceAttributes: parseAttributes(raw.resourceAttributes), }