diff --git a/src/search/src/datafusion/exec.rs b/src/search/src/datafusion/exec.rs index 4983170b52..f053725ef5 100644 --- a/src/search/src/datafusion/exec.rs +++ b/src/search/src/datafusion/exec.rs @@ -786,6 +786,97 @@ mod tests { Ok(()) } + // ── tmp/code.md B4 — the function catalog served to the query editor ───── + // + // registered_function_names() is the authoritative list. Two gaps were + // found reviewing it: JSON functions are registered by a SEPARATE call that + // the snapshot context never made, and functions provided by SQL rewriters + // never appear in any registry at all. + + #[test] + fn registered_function_names_includes_o2_udfs() { + let names = registered_function_names(); + for expected in [ + "str_match", + "match_all", + "histogram", + "spath", + "arrcount", + "re_match", + "cast_to_timestamp", + ] { + assert!( + names.iter().any(|n| n == expected), + "registry is missing the O2 UDF `{expected}`" + ); + } + } + + #[test] + fn registered_function_names_includes_datafusion_builtins() { + let names = registered_function_names(); + for expected in ["date_trunc", "coalesce", "concat", "regexp_replace", "abs"] { + assert!( + names.iter().any(|n| n == expected), + "registry is missing the DataFusion builtin `{expected}`" + ); + } + } + + #[test] + fn registered_function_names_includes_json_functions() { + // Production contexts call datafusion_functions_json::register_all + // separately (see flight.rs). Without that call here the whole json_* + // family is absent from the catalog the editor is served. + let names = registered_function_names(); + for expected in ["json_get", "json_get_str", "json_length"] { + assert!( + names.iter().any(|n| n == expected), + "registry is missing the JSON function `{expected}` — is datafusion_functions_json::register_all wired into the snapshot context?" + ); + } + } + + #[test] + fn registered_function_names_is_sorted_and_deduped() { + let names = registered_function_names(); + assert!(!names.is_empty()); + let mut sorted = names.to_vec(); + sorted.sort(); + sorted.dedup(); + assert_eq!(names, sorted.as_slice(), "names must be sorted and deduplicated"); + } + + #[test] + fn rewriter_aliases_are_exposed_for_the_catalog() { + // match_all_raw / match_all_raw_ignore_case are valid user-facing SQL + // (sql/rewriter/match_all_raw.rs rewrites them to match_all before + // planning) but appear in NO registry. A catalog built only from the + // registry would silently drop two functions users rely on today. + let aliases = crate::sql::rewriter::REWRITER_FUNCTION_ALIASES; + for expected in ["match_all_raw", "match_all_raw_ignore_case"] { + assert!( + aliases.contains(&expected), + "rewriter alias `{expected}` is not exported for the catalog" + ); + } + } + + #[test] + fn catalog_function_names_unions_registry_and_rewriter_aliases() { + let catalog = catalog_function_names(); + assert!(catalog.iter().any(|n| n == "match_all"), "registry entry missing"); + assert!( + catalog.iter().any(|n| n == "match_all_raw"), + "rewriter alias missing from the catalog union" + ); + assert!(catalog.iter().any(|n| n == "json_get"), "json function missing"); + let mut sorted = catalog.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(catalog, sorted, "catalog must be sorted and deduplicated"); + } + #[test] fn test_table_builder_new() { let builder = TableBuilder::new(); diff --git a/web/src/composables/useSuggestions.spec.ts b/web/src/composables/useSuggestions.spec.ts index 9baa700925..81e28c711a 100644 --- a/web/src/composables/useSuggestions.spec.ts +++ b/web/src/composables/useSuggestions.spec.ts @@ -410,3 +410,87 @@ describe("catalog wiring — suggestions come from the shared sqlCompletion modu expect(host.sortText < and.sortText).toBe(true); }); }); + +// ─── Phase 2 (tmp/code.md D1/N5 + B1) ──────────────────────────────────────── +// The composable is what Logs/Traces/Alerts actually consume, so the column +// type and the clause keywords have to survive the trip through it — not just +// exist in the catalog. + +describe("Phase 2 — column types reach the editor as detail (D1/N5)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("carries the column type into each field keyword's detail", () => { + const c = makeComposable(); + c.updateFieldKeywords([ + { name: "code", type: "Int64" }, + { name: "message", type: "Utf8" }, + ]); + const code = c.autoCompleteKeywords.value.find((k: any) => k.label === "code"); + const message = c.autoCompleteKeywords.value.find((k: any) => k.label === "message"); + expect(code.detail).toBe("Int64"); + expect(message.detail).toBe("Utf8"); + }); + + it("still excludes the timestamp column", () => { + const c = makeComposable(); + c.updateFieldKeywords([ + { name: "_timestamp", type: "Int64" }, + { name: "code", type: "Int64" }, + ]); + const labels = c.autoCompleteKeywords.value.map((k: any) => k.label); + expect(labels).not.toContain("_timestamp"); + expect(labels).toContain("code"); + }); + + it("tolerates fields with no type rather than emitting 'undefined'", () => { + const c = makeComposable(); + c.updateFieldKeywords([{ name: "code" }]); + const code = c.autoCompleteKeywords.value.find((k: any) => k.label === "code"); + expect(code.detail).toBeUndefined(); + }); + + it("carries types through updateAllKeywords too", () => { + const c = makeComposable(); + c.updateAllKeywords([{ name: "code", type: "Int64" }], []); + const code = c.autoCompleteKeywords.value.find((k: any) => k.label === "code"); + expect(code.detail).toBe("Int64"); + }); +}); + +describe("Phase 2 — SQL clause keywords are offered (B1)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("includes SELECT/FROM/WHERE in the base keyword list", async () => { + const c = makeComposable({ storedValues: [] }); + await run(c, "SELECT * FROM stream WHERE "); + const labels = c.effectiveKeywords.value.map((k: any) => k.label); + for (const kw of ["SELECT", "FROM", "WHERE", "GROUP BY", "ORDER BY", "LIMIT"]) { + expect(labels, `missing ${kw}`).toContain(kw); + } + }); + + it("keeps predicates alongside the clauses", async () => { + const c = makeComposable({ storedValues: [] }); + await run(c, "SELECT * FROM stream WHERE "); + const labels = c.effectiveKeywords.value.map((k: any) => k.label); + expect(labels).toContain("and"); + expect(labels).toContain("="); + }); + + it("suppresses clause keywords in value context", async () => { + const c = makeComposable({ storedValues: ["error"] }); + await run(c, "level = "); + const labels = c.effectiveKeywords.value.map((k: any) => k.label); + expect(labels).not.toContain("SELECT"); + expect(labels).toEqual(["error"]); + }); + + it("sorts fields ahead of clause keywords", async () => { + const c = makeComposable({ storedValues: [] }); + c.updateFieldKeywords([{ name: "host", type: "Utf8" }]); + await run(c, "SELECT * FROM stream WHERE "); + const host = c.effectiveKeywords.value.find((k: any) => k.label === "host"); + const select = c.effectiveKeywords.value.find((k: any) => k.label === "SELECT"); + expect(host.sortText < select.sortText).toBe(true); + }); +}); diff --git a/web/src/services/query_functions.spec.ts b/web/src/services/query_functions.spec.ts new file mode 100644 index 0000000000..82201a0a7b --- /dev/null +++ b/web/src/services/query_functions.spec.ts @@ -0,0 +1,52 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// TDD spec for tmp/code.md B4 — the query-function catalog endpoint. +// +// The frontend currently hand-maintains its list of SQL functions, which has +// no way of staying in step with what the backend actually registers. The +// server already computes the authoritative set; this service exposes it. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import http from "./http"; +import queryFunctions from "./query_functions"; + +vi.mock("./http", () => { + const mockClient = { get: vi.fn() }; + return { default: vi.fn(() => mockClient) }; +}); + +describe("query_functions service", () => { + const mockClient = (http as unknown as ReturnType)(); + + beforeEach(() => vi.clearAllMocks()); + + it("list() calls GET /api/{org}/query_functions", () => { + queryFunctions.list("myorg"); + expect(mockClient.get).toHaveBeenCalledWith("/api/myorg/query_functions"); + }); + + it("encodes an organisation identifier that needs it", () => { + queryFunctions.list("my org/1"); + expect(mockClient.get).toHaveBeenCalledWith( + `/api/${encodeURIComponent("my org/1")}/query_functions`, + ); + }); + + it("returns the client promise so callers can await it", async () => { + mockClient.get.mockResolvedValue({ data: { list: [] } }); + await expect(queryFunctions.list("myorg")).resolves.toEqual({ data: { list: [] } }); + }); +}); diff --git a/web/src/utils/query/serverFunctions.spec.ts b/web/src/utils/query/serverFunctions.spec.ts new file mode 100644 index 0000000000..e00cc0a3b9 --- /dev/null +++ b/web/src/utils/query/serverFunctions.spec.ts @@ -0,0 +1,143 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// TDD spec for tmp/code.md B4 — turning the server's function registry into +// completion entries and merging it with the hand-written O2 catalog. +// +// The server list is authoritative for WHAT exists (it is derived from the +// DataFusion registry, so it tracks the pinned fork, build features and +// per-org VRL transforms). The local catalog is authoritative for HOW an entry +// should be inserted, because only it knows which arguments are columns and +// which are literals — the distinction A3 was about. + +import { describe, it, expect } from "vitest"; +import { SQL_FUNCTIONS } from "./sqlCompletion"; +import { toCompletionEntries, mergeServerFunctions } from "./serverFunctions"; + +const serverFn = (over: Record = {}) => ({ + name: "date_trunc", + signature: "(precision, timestamp)", + doc: "Truncate a timestamp to a given precision.", + kind: "scalar", + ...over, +}); + +describe("toCompletionEntries — server payload to catalog entries", () => { + it("maps name, signature and doc onto the entry shape", () => { + const [e] = toCompletionEntries([serverFn()]); + expect(e.name).toBe("date_trunc"); + expect(e.label).toBe("date_trunc"); + expect(e.kind).toBe("Function"); + expect(e.detail).toBe("(precision, timestamp)"); + expect(e.documentation).toBe("Truncate a timestamp to a given precision."); + }); + + it("builds a snippet with one tab stop per declared argument", () => { + const [e] = toCompletionEntries([serverFn()]); + expect(e.insertText).toBe("date_trunc(${1:precision}, ${2:timestamp})"); + expect(e.insertTextRules).toBe("InsertAsSnippet"); + }); + + it("never reuses a tab stop index", () => { + const [e] = toCompletionEntries([ + serverFn({ name: "f", signature: "(a, b, c, d)" }), + ]); + const indices = [...e.insertText.matchAll(/\$\{(\d+):/g)].map((m) => m[1]); + expect(indices).toEqual(["1", "2", "3", "4"]); + }); + + it("emits empty parens for a zero-argument function", () => { + const [e] = toCompletionEntries([serverFn({ name: "now", signature: "()" })]); + expect(e.insertText).toBe("now()"); + }); + + it("falls back to a single tab stop when the signature is unparseable", () => { + const [e] = toCompletionEntries([serverFn({ name: "weird", signature: undefined })]); + expect(e.insertText).toBe("weird(${1:arg})"); + }); + + it("does not quote any server argument — arity is known, types are not", () => { + // Quoting is a per-function decision the server list cannot make. Guessing + // wrong is exactly the A3 bug (sum('field')), so never guess. + for (const e of toCompletionEntries([serverFn(), serverFn({ name: "concat" })])) { + expect(e.insertText, `${e.name} must not quote an argument`).not.toContain("'"); + } + }); + + it("flags deprecated entries from the server", () => { + const [e] = toCompletionEntries([serverFn({ deprecated: true })]); + expect(e.deprecated).toBe(true); + }); + + it("tolerates an empty or missing payload", () => { + expect(toCompletionEntries([])).toEqual([]); + expect(toCompletionEntries(undefined as any)).toEqual([]); + }); + + it("skips entries with no usable name", () => { + expect(toCompletionEntries([{ name: "" } as any, {} as any])).toEqual([]); + }); +}); + +describe("mergeServerFunctions — local catalog wins on insertion detail", () => { + it("keeps the local entry when the server reports the same function", () => { + // The local sum() knows its argument is a COLUMN; the server only knows + // the arity. Preferring the server entry would reintroduce sum('field'). + const merged = mergeServerFunctions(SQL_FUNCTIONS, [serverFn({ name: "sum" })]); + const sum = merged.filter((f) => f.name === "sum"); + expect(sum).toHaveLength(1); + expect(sum[0].insertText).toBe("sum(${1:field})"); + }); + + it("adds server functions the local catalog does not have", () => { + const merged = mergeServerFunctions(SQL_FUNCTIONS, [serverFn()]); + expect(merged.some((f) => f.name === "date_trunc")).toBe(true); + }); + + it("keeps every local entry even when the server list is empty", () => { + const merged = mergeServerFunctions(SQL_FUNCTIONS, []); + expect(merged).toHaveLength(SQL_FUNCTIONS.length); + }); + + it("returns the local catalog unchanged when the server call failed", () => { + expect(mergeServerFunctions(SQL_FUNCTIONS, undefined as any)).toEqual(SQL_FUNCTIONS); + }); + + it("produces no duplicate names", () => { + const merged = mergeServerFunctions(SQL_FUNCTIONS, [ + serverFn({ name: "sum" }), + serverFn(), + serverFn(), + ]); + const names = merged.map((f) => f.name); + expect(new Set(names).size).toBe(names.length); + }); + + it("matches case-insensitively so SUM does not duplicate sum", () => { + const merged = mergeServerFunctions(SQL_FUNCTIONS, [serverFn({ name: "SUM" })]); + expect(merged.filter((f) => f.name.toLowerCase() === "sum")).toHaveLength(1); + }); + + it("marks server-only entries kind Function so they get the right icon", () => { + const merged = mergeServerFunctions(SQL_FUNCTIONS, [serverFn()]); + expect(merged.find((f) => f.name === "date_trunc")!.kind).toBe("Function"); + }); + + it("sorts deterministically so the dropdown order is stable", () => { + const a = mergeServerFunctions(SQL_FUNCTIONS, [serverFn(), serverFn({ name: "coalesce" })]); + const b = mergeServerFunctions(SQL_FUNCTIONS, [serverFn({ name: "coalesce" }), serverFn()]); + expect(a.map((f) => f.name)).toEqual(b.map((f) => f.name)); + }); +}); diff --git a/web/src/utils/query/sqlCompletion.spec.ts b/web/src/utils/query/sqlCompletion.spec.ts index 7176689e40..ba834e8d18 100644 --- a/web/src/utils/query/sqlCompletion.spec.ts +++ b/web/src/utils/query/sqlCompletion.spec.ts @@ -22,7 +22,9 @@ import { describe, it, expect } from "vitest"; import { SQL_FUNCTIONS, SQL_KEYWORDS, + SQL_CLAUSE_KEYWORDS, buildCompletionItems, + buildFieldEntry, type SqlCompletionEntry, } from "./sqlCompletion"; @@ -614,3 +616,182 @@ describe("N2/D7 — resolveSuggestions / resolveKeywords", () => { expect(resolveKeywords("sql", custom)).toBe(custom); }); }); + +// ═══════════════════════════════════════════════════════════════════════════ +// PHASE 2 +// ═══════════════════════════════════════════════════════════════════════════ + +// ─────────────────────────────────────────────────────────────────────────── +// B1 — SQL clause keywords +// +// monaco's sql.contribution.js is TOKENIZER-ONLY (zero matches for +// registerCompletionItemProvider). Nothing ever offered SELECT/FROM/WHERE; the +// only reason they appeared to work was the word-based provider echoing text +// already in the buffer. +// ─────────────────────────────────────────────────────────────────────────── + +describe("B1 — the clause keyword catalog", () => { + const clause = (label: string) => SQL_CLAUSE_KEYWORDS.find((k) => k.label === label); + + const REQUIRED = [ + "SELECT", + "FROM", + "WHERE", + "GROUP BY", + "ORDER BY", + "HAVING", + "LIMIT", + "OFFSET", + "DISTINCT", + "AS", + "WITH", + "UNION", + "UNION ALL", + "INNER JOIN", + "LEFT JOIN", + "RIGHT JOIN", + "FULL OUTER JOIN", + "ON", + "CASE", + "CAST", + "OVER", + "PARTITION BY", + "ASC", + "DESC", + "NULLS FIRST", + "NULLS LAST", + "EXISTS", + ]; + + it.each(REQUIRED)("offers %s", (label) => { + expect(clause(label), `SQL_CLAUSE_KEYWORDS is missing ${label}`).toBeDefined(); + }); + + it("labels clause keywords in uppercase — the SQL convention", () => { + for (const k of SQL_CLAUSE_KEYWORDS) { + expect(k.label, `${k.name} should be uppercase`).toBe(k.label.toUpperCase()); + } + }); + + it("marks every clause keyword as kind Keyword", () => { + for (const k of SQL_CLAUSE_KEYWORDS) expect(k.kind, k.name).toBe("Keyword"); + }); + + it("gives every clause keyword a detail so the widget explains it", () => { + for (const k of SQL_CLAUSE_KEYWORDS) expect(k.detail, `${k.name} needs detail`).toBeTruthy(); + }); + + it("does not duplicate anything already in SQL_KEYWORDS", () => { + const predicates = new Set(SQL_KEYWORDS.map((k) => k.name.toLowerCase())); + for (const k of SQL_CLAUSE_KEYWORDS) { + expect(predicates.has(k.name.toLowerCase()), `${k.name} duplicates a predicate`).toBe(false); + } + }); + + it("has no duplicates within itself", () => { + const names = SQL_CLAUSE_KEYWORDS.map((k) => k.name); + expect(new Set(names).size).toBe(names.length); + }); + + it("expands CASE into a full snippet, not a bare word", () => { + const c = clause("CASE")!; + expect(c.insertText).toContain("WHEN"); + expect(c.insertText).toContain("END"); + expect(c.insertTextRules).toBe("InsertAsSnippet"); + }); + + it("expands JOINs with an ON clause tab stop", () => { + const j = clause("INNER JOIN")!; + expect(j.insertText).toContain("ON"); + expect(j.insertTextRules).toBe("InsertAsSnippet"); + }); + + it("gives multi-tab-stop clause snippets distinct indices", () => { + for (const k of SQL_CLAUSE_KEYWORDS) { + const indices = [...k.insertText.matchAll(/\$\{(\d+)[:}]/g)].map((m) => m[1]); + expect(new Set(indices).size, `${k.name} reuses a tab stop index`).toBe(indices.length); + } + }); + + it("declares InsertAsSnippet for every entry carrying a tab stop", () => { + for (const k of SQL_CLAUSE_KEYWORDS) { + if (k.insertText.includes("${")) { + expect(k.insertTextRules, `${k.name}`).toBe("InsertAsSnippet"); + } + } + }); + + it("sorts clause keywords after fields but they remain reachable", () => { + // Clause keywords are structural: useful, but never ahead of a column name. + for (const k of SQL_CLAUSE_KEYWORDS) { + expect(k.sortText, `${k.name} needs an explicit sortText`).toBeTruthy(); + } + }); +}); + +describe("B1 — clause keywords reach a SQL editor through the fallback", () => { + it("resolveKeywords serves predicates AND clauses for SQL", async () => { + const { resolveKeywords } = await import("./sqlCompletion"); + const resolved = resolveKeywords("sql", []) as SqlCompletionEntry[]; + const names = resolved.map((k) => k.name); + expect(names).toContain("SELECT"); + expect(names).toContain("GROUP BY"); + expect(names).toContain("and"); // predicates still there + }); + + it("does not leak SQL clauses into non-SQL editors", async () => { + const { resolveKeywords } = await import("./sqlCompletion"); + expect(resolveKeywords("json", [])).toEqual([]); + expect(resolveKeywords("promql", [])).toEqual([]); + }); + + it("still prefers caller-supplied keywords over the defaults", async () => { + const { resolveKeywords } = await import("./sqlCompletion"); + const custom = [{ name: "host", label: "host", kind: "Field", insertText: "host" }] as any; + expect(resolveKeywords("sql", custom)).toBe(custom); + }); +}); + +// ─────────────────────────────────────────────────────────────────────────── +// D1 / N5 — column type in `detail` +// +// The type is already carried on every field object (Utf8, Int64, …) and was +// discarded when building keywords. It is exactly what belongs in the suggest +// widget's inline column. +// ─────────────────────────────────────────────────────────────────────────── + +describe("D1/N5 — field entries surface the column type", () => { + it("puts the column type in detail", () => { + expect(buildFieldEntry({ name: "code", type: "Int64" }).detail).toBe("Int64"); + }); + + it("keeps the field name as label and insertText", () => { + const e = buildFieldEntry({ name: "k8s_pod_name", type: "Utf8" }); + expect(e.label).toBe("k8s_pod_name"); + expect(e.insertText).toBe("k8s_pod_name"); + }); + + it("marks it kind Field", () => { + expect(buildFieldEntry({ name: "code", type: "Int64" }).kind).toBe("Field"); + }); + + it("sorts fields ahead of functions and keywords", () => { + const field = buildFieldEntry({ name: "code", type: "Int64" }); + const clause = SQL_CLAUSE_KEYWORDS[0]; + expect(field.sortText! < clause.sortText!).toBe(true); + }); + + it("omits detail rather than inventing one when the type is unknown", () => { + expect(buildFieldEntry({ name: "code" } as any).detail).toBeUndefined(); + }); + + it("does not mark a plain field name as a snippet", () => { + expect(buildFieldEntry({ name: "code", type: "Utf8" }).insertTextRules).toBeUndefined(); + }); + + it("forwards the type through to monaco as detail", () => { + const [item] = build({ keywords: [buildFieldEntry({ name: "code", type: "Int64" })] }); + expect(item.detail).toBe("Int64"); + expect(item.kind).toBe(KINDS.Field); + }); +});