diff --git a/web/src/composables/metrics/useMetricsExplorerGrid.spec.ts b/web/src/composables/metrics/useMetricsExplorerGrid.spec.ts index 4f6656c2b0..d9eda6a02f 100644 --- a/web/src/composables/metrics/useMetricsExplorerGrid.spec.ts +++ b/web/src/composables/metrics/useMetricsExplorerGrid.spec.ts @@ -1146,16 +1146,21 @@ describe("useMetricsExplorerGrid", () => { // Round 1: the card's own `sum(rate(...))` — empty. // Round 2: the presence probe — the metric is right there. + // Rounds 3-4: the widened-window retries — still nothing to rate. await landRounds( grid.requestPreview(cardNamed(grid, "lat_seconds_count")), NO_SERIES, SERIES, + NO_SERIES, + NO_SERIES, ); expect(names(grid)).toContain("lat_seconds_count"); expect(grid.emptyHiddenCount.value).toBe(0); // Not "No data": the card says what is actually wrong with it. expect(grid.previews.value["lat_seconds_count"].sparse).toBe(true); + // No rung worked, so the drill-in has nothing better than the default. + expect(grid.previews.value["lat_seconds_count"].widenedRateWindow).toBe(null); }); it("so a histogram with real data never renders NO card at all", async () => { @@ -1169,7 +1174,13 @@ describe("useMetricsExplorerGrid", () => { expect(grid.cards.value.map((c: any) => c.name)).not.toContain("lat_seconds"); // the phantom base: correctly suppressed, so the components are all there is for (const member of ["lat_seconds_bucket", "lat_seconds_count"]) { - await landRounds(grid.requestPreview(cardNamed(grid, member)), NO_SERIES, SERIES); + await landRounds( + grid.requestPreview(cardNamed(grid, member)), + NO_SERIES, + SERIES, + NO_SERIES, + NO_SERIES, + ); } expect(names(grid)).toEqual( @@ -1189,10 +1200,64 @@ describe("useMetricsExplorerGrid", () => { // asked WITHOUT `rate()`, which is the whole point. expect(inFlight.map((q) => q.query)).toEqual(['count({__name__="lat_seconds_count"})']); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(SERIES)); + await flush(); + // The probe saying "samples exist" sends the card into the widened-window + // retries; landing the first one settles the preview. inFlight.splice(0, inFlight.length).forEach((q) => q.complete(SERIES)); await preview; }); + it("retries with a widened window and charts the data instead of giving up", async () => { + // The org's scrape_interval setting is a claim, not a measurement. Data + // arriving every 60s under a 15s setting gets a window that almost never + // holds the two samples `rate()` needs — the card said "too few samples" + // for a metric that charts perfectly well over a wider window. + const grid = await setup(); + const preview = grid.requestPreview(cardNamed(grid, "lat_seconds_count")); + + await flush(); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(NO_SERIES)); // rate: empty + await flush(); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(SERIES)); // probe: samples exist + await flush(); + + // 1h range, 50 points, 15s scrape → standard window [1m27s], first retry 4x. + expect(inFlight.map((q) => q.query).join()).toContain("[5m48s]"); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(SERIES)); + await preview; + + const settled = grid.previews.value["lat_seconds_count"]; + expect(settled.sparse).toBe(false); + expect(settled.results.some((r: any) => r.result.length)).toBe(true); + expect(names(grid)).toContain("lat_seconds_count"); + // Carried for the drill-in: the editor resolves `$__rate_interval` from + // the same overstated scrape interval, so it needs the window that + // actually worked. + expect(settled.widenedRateWindow).toBe("5m48s"); + }); + + it("escalates to a second, wider window before conceding sparseness", async () => { + const grid = await setup(); + const preview = grid.requestPreview(cardNamed(grid, "lat_seconds_count")); + + await flush(); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(NO_SERIES)); // rate + await flush(); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(SERIES)); // probe + await flush(); + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(NO_SERIES)); // 4x: still nothing + await flush(); + + expect(inFlight.map((q) => q.query).join()).toContain("[23m12s]"); // 16x + inFlight.splice(0, inFlight.length).forEach((q) => q.complete(SERIES)); + await preview; + + expect(grid.previews.value["lat_seconds_count"].sparse).toBe(false); + expect(grid.previews.value["lat_seconds_count"].widenedRateWindow).toBe("23m12s"); + expect(names(grid)).toContain("lat_seconds_count"); + }); + it("still hides a metric the probe agrees is empty", async () => { // The filter has to keep doing its job: an idle counter is still an idle // counter, and the grid must not fill up with no-data panels. diff --git a/web/src/composables/metrics/useMetricsExplorerGrid.ts b/web/src/composables/metrics/useMetricsExplorerGrid.ts index 3d2f1ad008..92d69a149b 100644 --- a/web/src/composables/metrics/useMetricsExplorerGrid.ts +++ b/web/src/composables/metrics/useMetricsExplorerGrid.ts @@ -43,6 +43,7 @@ import { computeRateWindow, computePercentileWindow, computeStepSeconds, + computeWidenedRateWindows, DEFAULT_SCRAPE_INTERVAL_SECONDS, getMetricDefaults, isRateBasedKind, @@ -106,7 +107,9 @@ export interface CardPreview { * The metric HAS samples in the window, but its rate-based default query could * not produce a single point from them — `rate()` needs two samples inside its * window, and there are not two to be had (a one-off scrape, or a scrape - * interval longer than the window). + * interval longer than the window). Raised only after retrying with widened + * windows (see `computeWidenedRateWindows`) also produced nothing — or the + * retries failed, in which case sparse is still the best answer in hand. * * Emphatically NOT "no data": the card stays visible and says what is actually * wrong, instead of being hidden as empty and taking a real, ingested metric @@ -114,6 +117,19 @@ export interface CardPreview { * decides emptiness too. See `buildPresenceQuery`. */ sparse: boolean; + /** + * The rate window that actually charted this data, when the standard one + * could not — the widened-retry rung that succeeded (e.g. "4m"), else `null`. + * + * Carried for the drill-in: the editor normally receives `$__rate_interval` + * and re-derives the window from the org's scrape interval — the very value + * whose overstatement forced the widening — so it would resolve straight back + * to the window that just returned nothing. A card that needed widening hands + * the editor this concrete window instead. Survives the persisted cache for + * the same reason `nanGuardApplied` does: a restore fires no query, so it has + * no way to re-learn it. + */ + widenedRateWindow: string | null; /** * When the data was actually fetched (ms). Survives the persisted cache, so a * card restored from it reports the true age of what it shows — the same @@ -1218,8 +1234,17 @@ export function useMetricsExplorerGrid() { * override and the rate window, so they are the whole identity — there is no * separate variables/schema object to compare, as there is for a dashboard * panel. + * + * `v` is the SHAPE version of the cached value, bumped when the preview + * grows a field a restore cannot reconstruct. v2: `widenedRateWindow` — a + * pre-v2 entry can hold results a widened retry fetched with no memory of + * the window that fetched them, and restoring it hands the drill-in + * `$__rate_interval`, which resolves back to the window that returned + * nothing: the card charts, the editor opens blank. Missing the cache once + * and re-learning the window live is the cheap way out. */ const cacheIdentity = (queries: any[], step: number) => ({ + v: 2, queries: queries.map((q: any) => q.expr), step, org: org.value, @@ -1302,6 +1327,7 @@ export function useMetricsExplorerGrid() { // paint fine on the live path and then vanish from the grid on the next // visit, when the cache answered instead. sparse: !!cached.value.sparse, + widenedRateWindow: cached.value.widenedRateWindow ?? null, lastTriggeredAt: cached.value.lastTriggeredAt ?? cached.timestamp ?? null, cachedDataDiffersFromTimeRange: differs, footerLabel: resolved.footerLabel, @@ -1322,6 +1348,7 @@ export function useMetricsExplorerGrid() { results: preview.results, nanGuardApplied: preview.nanGuardApplied, sparse: preview.sparse, + widenedRateWindow: preview.widenedRateWindow, lastTriggeredAt: preview.lastTriggeredAt, }, { @@ -1371,6 +1398,7 @@ export function useMetricsExplorerGrid() { stale: false, nanGuardApplied: false, sparse: false, + widenedRateWindow: null, lastTriggeredAt: null, cachedDataDiffersFromTimeRange: false, footerLabel: resolved.footerLabel, @@ -1428,6 +1456,10 @@ export function useMetricsExplorerGrid() { stale: false, nanGuardApplied: false, sparse: false, + // Carried like `results`: the old (possibly widened) chart stays on + // screen while the re-query runs, so a drill-in during that window must + // still hand the editor the window that chart is true of. + widenedRateWindow: existing?.widenedRateWindow ?? null, lastTriggeredAt: existing?.lastTriggeredAt ?? null, cachedDataDiffersFromTimeRange: false, // Carried with `results` above: a re-query keeps the OLD chart on screen @@ -1475,12 +1507,55 @@ export function useMetricsExplorerGrid() { // calling it empty and hiding it. Gated on `card.hasData` so the long tail // of registered-but-never-written metrics — the ones the "With data" filter // is FOR — is settled from the stream list without a second query. - const sparse = + let sparse = !results.some(hasSamples) && isRateBasedKind(defaults.cardKind) && card.hasData && (await hasSamplesInWindow(card, step, opts)); + // The samples are THERE — the probe just said so — the window is merely + // too narrow to catch two of them, which means the org's configured + // scrape interval overstates how often this metric actually arrives. + // Before settling for the "too few samples" card, re-ask with wider + // windows and chart the data if any of them can. The step stays the + // same on purpose: the retry changes how far back each point may look, + // not which points the chart is made of. + let widenedRateWindow: string | null = null; + if (sparse) { + for (const rateWindow of computeWidenedRateWindows( + rangeSeconds.value, + points, + scrapeIntervalSeconds.value, + )) { + const widened = effectiveVariant(card, points, { rateWindow }); + if (!widened.resolved?.queries.length) break; + let retried: any[]; + try { + retried = await runQueries( + widened.resolved.queries, + step, + card.name, + opts?.priority ?? PRIORITY.VISIBLE, + !!opts?.skipCache, + ); + } catch (error) { + // A cancel means the user left this state — abort like any other + // request. Any OTHER failure must not escape to the error handler: + // the card's own query already succeeded and its honest answer — + // sparse — is in hand, so a retry that merely tried to improve on + // it settles for it instead of repainting the card as an error. + if (isCancelled(error)) throw error; + break; + } + if (retried.some(hasSamples)) { + results = retried; + sparse = false; + widenedRateWindow = rateWindow; + break; + } + } + } + // The response outlived the state that asked for it: a bulk clear empties // the map and cancels what is in flight, but a request that had ALREADY // resolved is past the point of cancelling and would write the old org's @@ -1497,6 +1572,7 @@ export function useMetricsExplorerGrid() { stale: false, nanGuardApplied, sparse, + widenedRateWindow, // Stamped on the real fetch, and persisted — this is what lets a // cache-restored card tell the user how old its data actually is. lastTriggeredAt: Date.now(), @@ -1550,6 +1626,7 @@ export function useMetricsExplorerGrid() { stale: previous.length > 0, nanGuardApplied: existing?.nanGuardApplied ?? false, sparse: existing?.sparse ?? false, + widenedRateWindow: existing?.widenedRateWindow ?? null, lastTriggeredAt: existing?.lastTriggeredAt ?? null, cachedDataDiffersFromTimeRange: existing?.cachedDataDiffersFromTimeRange ?? false, // Carried with `results`: this path KEEPS the previous chart up, so it @@ -1568,7 +1645,11 @@ export function useMetricsExplorerGrid() { * Includes the NaN-guarded rewrite: when the first query comes back all-NaN we * re-run a *different* query string, so its result lives under a different * key. Omitting it here would let a refresh re-serve the stale guarded - * response from cache and appear to do nothing. + * response from cache and appear to do nothing. The widened-window retries + * (see `computeWidenedRateWindows`) are extra query strings for the same + * reason, and omitting THEM would be worse than a no-op refresh: the standard + * window re-runs live, comes back empty as ever, and the retry then replays + * the stale widened response from cache — a refresh that lies. */ const previewKeysOf = (card: MetricCard): string[] => { const points = pointsFor(card); @@ -1583,6 +1664,16 @@ export function useMetricsExplorerGrid() { keys.add(previewCacheKey(q.expr, step)); } } + for (const rateWindow of computeWidenedRateWindows( + rangeSeconds.value, + points, + scrapeIntervalSeconds.value, + )) { + const { resolved } = effectiveVariant(card, points, { rateWindow }); + for (const q of (resolved?.queries ?? []) as any[]) { + keys.add(previewCacheKey(q.expr, step)); + } + } return [...keys]; }; diff --git a/web/src/plugins/metrics/explorer/MetricsExplorer.spec.ts b/web/src/plugins/metrics/explorer/MetricsExplorer.spec.ts index 63aed7ccf4..8595aaee69 100644 --- a/web/src/plugins/metrics/explorer/MetricsExplorer.spec.ts +++ b/web/src/plugins/metrics/explorer/MetricsExplorer.spec.ts @@ -524,9 +524,36 @@ describe("MetricsExplorer wiring", () => { (wrapper.vm as any).onSelect(CARD); expect(grid.effectiveVariant).toHaveBeenCalled(); + // The default handoff: `$__rate_interval`, for the editor to re-derive + // against its own range and width. + expect(grid.effectiveVariant).toHaveBeenCalledWith( + CARD, + undefined, + expect.objectContaining({ rateWindow: "$__rate_interval" }), + ); expect((wrapper.vm as any).mode).toBe("visualize"); expect((wrapper.vm as any).visualizeSeed).toBeTruthy(); }); + + it("a card that charted through a WIDENED window hands the editor that window", () => { + // The editor resolves `$__rate_interval` from the org's scrape interval — + // the very value whose overstatement forced the card to widen — so the + // variable would resolve straight back to the window that returned + // nothing, and the drill-in would open on an empty chart of a metric the + // card is visibly charting. + grid.previews.value[CARD.name] = { widenedRateWindow: "4m" }; + const wrapper = mountExplorer(); + + (wrapper.vm as any).onSelect(CARD); + + expect(grid.effectiveVariant).toHaveBeenCalledWith( + CARD, + undefined, + expect.objectContaining({ rateWindow: "4m" }), + ); + expect((wrapper.vm as any).mode).toBe("visualize"); + delete grid.previews.value[CARD.name]; + }); }); describe("the type facet uses OCheckboxGroup over a Set<->array boundary", () => { diff --git a/web/src/plugins/metrics/explorer/MetricsExplorer.vue b/web/src/plugins/metrics/explorer/MetricsExplorer.vue index 0d2a80aee5..f0b20f7dd5 100644 --- a/web/src/plugins/metrics/explorer/MetricsExplorer.vue +++ b/web/src/plugins/metrics/explorer/MetricsExplorer.vue @@ -1180,8 +1180,19 @@ export default defineComponent({ // panel arrives frozen at whatever the range happened to be on the card — // and a 4-minute rate window sampled every 30 minutes on a 7-day view is // not a chart of anything. + // + // EXCEPT when the card only charted because it widened its window: the + // editor resolves `$__rate_interval` from the org's scrape interval — the + // very value whose overstatement forced the widening — so handing it the + // variable would resolve straight back to the window that returned + // nothing, and the drill-in would open on an empty chart of a metric the + // card is visibly charting. The concrete widened window goes over + // instead. It IS frozen to the card's range, but frozen-and-charting + // beats adaptive-and-blank, and it keeps the drill-in contract: the + // editor opens on what the card actually shows. + const widenedRateWindow = grid.previews.value[card.name]?.widenedRateWindow; const { defaults, resolved } = grid.effectiveVariant(card, undefined, { - rateWindow: PANEL_RATE_WINDOW, + rateWindow: widenedRateWindow ?? PANEL_RATE_WINDOW, percentileWindow: PANEL_PERCENTILE_WINDOW, }); if (!resolved) return; @@ -1288,8 +1299,12 @@ export default defineComponent({ for (const name of grid.favorites.value) { const card = byName.get(name); if (!card) continue; + // Same rule as the drill-in: a card that only charted through a widened + // window must hand the panel that concrete window, or the dashboard is + // born with a permanently blank panel for a metric the card charts. + const widenedRateWindow = grid.previews.value[name]?.widenedRateWindow; const { defaults, resolved } = grid.effectiveVariant(card, undefined, { - rateWindow: PANEL_RATE_WINDOW, + rateWindow: widenedRateWindow ?? PANEL_RATE_WINDOW, percentileWindow: PANEL_PERCENTILE_WINDOW, }); if (!resolved) continue; diff --git a/web/src/utils/metrics/metricDefaults.spec.ts b/web/src/utils/metrics/metricDefaults.spec.ts index 0666c30100..cc060397e3 100644 --- a/web/src/utils/metrics/metricDefaults.spec.ts +++ b/web/src/utils/metrics/metricDefaults.spec.ts @@ -21,6 +21,7 @@ import { computePercentileWindow, computeRateWindow, computeStepSeconds, + computeWidenedRateWindows, formatPromDuration, getMetricDefaults, MIN_PERCENTILE_SAMPLES, @@ -135,6 +136,48 @@ describe("rate window (PRD 6.4)", () => { }); }); +/** + * The retry ladder for a rate query whose standard window found nothing while + * the metric's samples sit right there in the range: the org's configured + * scrape interval overstated how often the data actually arrives, so the + * window gets widened geometrically before the card concedes "too few samples". + */ +describe("widened rate windows (sparse retry)", () => { + it("escalates 4x then 16x from the standard window", () => { + // The demo-app case: 15m range at a declared 15s scrape gives a [1m] + // standard window, but the app exports every 60s — [1m] rarely holds two + // samples. The first rung, [4m], is exactly the window a correctly + // configured 60s org would have computed; the second absorbs the + // range/2 cap. + expect(computeRateWindow(15 * 60, undefined, 15)).toBe("1m"); + expect(computeWidenedRateWindows(15 * 60, undefined, 15)).toEqual(["4m", "7m30s"]); + }); + + it("widens multiplicatively even where the step dominates the scrape floor", () => { + // At long ranges the standard window is already step-sized, and adding a + // few scrape intervals to it would barely move it — a metric arriving at a + // 10m cadence under a 5m-ish window would still find nothing. Scaling the + // whole window is what actually reaches slower cadences. + const range = 24 * 3600; // step 288s, standard window 303s + expect(computeWidenedRateWindows(range, undefined, 15)).toEqual(["20m12s", "1h20m48s"]); + }); + + it("caps every rung at half the range, and dedupes rungs the cap flattens", () => { + // A window wider than half the view is one global average pretending to be + // a trend; and if two samples don't fit in half the range, no window was + // ever going to chart this as a rate. + const rungs = computeWidenedRateWindows(15 * 60, undefined, 60); // standard [4m], cap 7m30s + expect(rungs).toEqual(["7m30s"]); + }); + + it("returns nothing when the cap leaves no room to widen", () => { + // A 5m view at a 60s scrape already has a [4m] standard window — wider than + // the 2m30s cap. There is no honest wider window to offer; the card keeps + // its "too few samples" message. + expect(computeWidenedRateWindows(5 * 60, undefined, 60)).toEqual([]); + }); +}); + /** * The rate window is sized for `rate()`, which needs two samples. Reusing it for * `quantile_over_time` gave a 15m view a 1m window — FOUR samples at a 15s diff --git a/web/src/utils/metrics/metricDefaults.ts b/web/src/utils/metrics/metricDefaults.ts index 968cb01e34..a3336f28d6 100644 --- a/web/src/utils/metrics/metricDefaults.ts +++ b/web/src/utils/metrics/metricDefaults.ts @@ -739,6 +739,48 @@ export function computeRateWindow( return formatPromDuration(rateWindowSeconds(rangeSeconds, maxDataPoints, scrapeIntervalSeconds)); } +/** + * Widened `[W]`s to RETRY a rate query with, after the standard window came + * back empty on a metric that demonstrably has samples in the range. + * + * The standard window is sized from the org's CONFIGURED scrape interval, and + * that setting is a claim, not a measurement. An app exporting OTLP metrics + * every 60s under an org that says `scrape_interval: 15` gets a `[1m]` window + * that almost never holds the two samples `rate()` needs — so every rate-based + * card reports "too few samples" while the data sits fully ingested in the + * selected range. + * + * The true cadence is unknown at this point (all that is known is that it is + * wider than roughly half the window that just failed), so the retries + * escalate geometrically: 4x, then 16x. Two rounds cover two orders of + * magnitude of mismatch for at most two extra queries, and only on cards + * already known to be in this state. + * + * Capped at half the range: rate over a window wider than that is one global + * average masquerading as a trend, and a cadence so slow that two samples do + * not fit in half the range could never chart as a rate anyway — at that point + * the "too few samples" card is the honest answer. + * + * @returns {string[]} PromQL durations, each strictly wider than the last and + * than the standard window; empty when the cap leaves no room to widen. + */ +export function computeWidenedRateWindows( + rangeSeconds: number, + maxDataPoints: number = MAX_DATA_POINTS, + scrapeIntervalSeconds: number = DEFAULT_SCRAPE_INTERVAL_SECONDS, +): string[] { + const base = rateWindowSeconds(rangeSeconds, maxDataPoints, scrapeIntervalSeconds); + const cap = Math.floor(Math.max(0, Number(rangeSeconds) || 0) / 2); + const widths: number[] = []; + for (const factor of [4, 16]) { + const w = Math.min(base * factor, cap); + if (w > base && w > (widths[widths.length - 1] ?? 0)) widths.push(w); + // The cap absorbed this rung; the next factor would only repeat it. + if (base * factor >= cap) break; + } + return widths.map(formatPromDuration); +} + /** * The rate window to hand to a PANEL, as opposed to one we execute ourselves. *