Merge branch 'main' into feat/dashboard-tab-reorder-rename

The rename input kept the tab wide in edit mode, leaving a trailing gap
after the name. Auto-size it with an invisible grid sizer (matching
OInlineEdit): an explicit max-content column plus size="1" on the input
(so its default ~20ch intrinsic width no longer drives the column) make
the field exactly as wide as the name.

.claude/skills/ui-architect/references/calm-signal.md

@@ -58,8 +58,30 @@ All token-backed and dark-mode-safe. Reuse these before inventing anything.
   Status chip beside them. Pass `size` at the call site, matching its siblings.
 - **Row state signal** — an extreme-left colour **rail** via `OTable`'s
   per-row `getRowStyle` (inset box-shadow, rem width, token colour) + a **light
-  exception highlight** via `row-class` (tint only the rows that need action —
-  never the normal ones).
+  exception highlight** via `row-class`. The rail and the wash are two different
+  strengths of the same signal, and the rail is the default:
+
+  | | Rail (`getRowStyle`) | Wash (`row-class`) |
+  | --- | --- | --- |
+  | Cost | a few px at the row edge | ~the whole row |
+  | Use for | **every** state, always | only the two cases below |
+
+  A full-row wash is the loudest thing on a list, so it earns its place only when
+  **all three** hold: the state means **act now** (not merely "not green"), it is
+  **rare in a healthy system**, and **the row is the unit of action**. In practice
+  that leaves exactly two washes:
+  - `!bg-status-error-bg` — failing/errored/offline. *Alerts* failed, *Pipelines*
+    errored, *Nodes* offline, *Synthetics* failed.
+  - `!bg-surface-panel` — paused/disabled. This one is **de-emphasis, not alarm**:
+    the row is deliberately inert, so it recedes rather than shouts.
+
+  Everything else keeps a clean row and reads from the rail — including states
+  that are "bad but not urgent": **degraded/warning** (worth noticing, not worth
+  acting on this second), **stale**, **unknown**, **never-ran**. And whatever the
+  state means, if it is **common** the wash is wrong regardless: *Streams* drops it
+  entirely, because "never ingested" and "quiet for a day" describe a large share
+  of rows in a normal org and a table where most rows are tinted signals nothing.
+  Pages with no failure state at all — Users, Roles, Dashboards — never wash.
 - **Recency** — `OTimeCell` `mode="relative"` (`"3 min ago"`) with a hot/warm/
   cold dot, instead of a raw timestamp column.
 - **People** — `OUserCell` for owner/author columns.
@@ -156,6 +178,24 @@ people to ignore the colour. When in doubt, grey.
 
 Colour only earns attention if most of the screen stays quiet:
 
+- **Earn every tile — a strip is not a page decoration.** Before adding one, put
+  each tile to three questions: does the number **vary**, does someone **act** on
+  it, and is it **not already on screen**? Tiles that fail are noise dressed as
+  signal:
+  - *structurally constant* — "System accounts" is 1 in almost every org, so the
+    tile is a label with a number stuck to it;
+  - *almost always zero* — "New this week" on a list that gains an item a quarter;
+  - *derivable from its neighbours* — "In use" beside "Unused" and "Total";
+  - *already in the footer* — `footerTitle` renders "N Dashboards" under every
+    table, so a Total tile alone is not a reason to have a strip.
+  A page whose only candidates fail these gets **no strip** — keep the per-row
+  signals (relative recency, a state rail, a count column) and stop. Dashboards,
+  Service Accounts and Roles all ended up here: pages where a strip added pixels
+  and no information. Roles is the clearest case — "Unused" is just the member
+  column sorted ascending, so two tiles restated what the rows already said. **A
+  count column plus sorting usually beats a strip**; reach for a strip only when
+  the page has a real distribution to summarise (Users across roles, monitors
+  across health) *and* the tiles double as the facet.
 - **Highlight exceptions, not the norm.** Tint the failed/paused rows; leave the
   healthy majority clean. A table where every row is coloured signals nothing.
   And if a page has no true failure state (a catalog list), the calm answer is a

Cargo.lock

@@ -2480,12 +2480,14 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "o2_enterprise",
+ "object_store",
  "parking_lot 0.12.5",
  "parquet",
  "rand 0.10.1",
  "schema",
  "search",
  "search_service",
+ "tantivy",
  "tantivy_utils",
  "tokio",
 ]
@@ -2551,6 +2553,7 @@ dependencies = [
  "dotenv_config",
  "dotenvy",
  "expect-test",
+ "fastdivide",
  "faststr",
  "float-cmp",
  "futures",
@@ -10530,6 +10533,7 @@ dependencies = [
  "futures",
  "futures-util",
  "hashbrown 0.16.1",
+ "hashlink 0.11.0",
  "infra",
  "itertools 0.14.0",
  "log",

Cargo.toml

@@ -456,6 +456,7 @@ sqlparser = { version = "0.62", features = ["serde", "visitor"] }
 dotenv_config = "0.2"
 dotenvy = "0.15"
 env_logger = "0.11"
+fastdivide = "0.4"
 faststr = { version = "0.2", features = ["serde"] }
 flate2 = { version = "1.0", features = ["zlib"] }
 futures = "0.3"

src/compaction/Cargo.toml

@@ -28,11 +28,13 @@ ingester.workspace = true
 itertools.workspace = true
 log.workspace = true
 o2_enterprise = { workspace = true, optional = true, default-features = false }
+object_store.workspace = true
 parking_lot.workspace = true
 parquet.workspace = true
 rand.workspace = true
 schema.workspace = true
 search.workspace = true
 search_service.workspace = true
+tantivy.workspace = true
 tantivy_utils.workspace = true
 tokio.workspace = true

src/compaction/src/bloom/builder.rs

@@ -0,0 +1,127 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Build SBBF blooms by iterating tantivy term dictionaries.
+//!
+//! For an indexed field, tantivy already stores a deduplicated term
+//! dictionary. Iterating it is much cheaper than re-scanning the parquet
+//! column — terms come back already unique, sorted, and as raw bytes.
+//!
+//! Used by the compactor merge hook to build per-(file, field)
+//! blooms after the .ttv file is written.
+
+use std::collections::HashSet;
+
+use anyhow::Context;
+use hashbrown::HashMap;
+use infra::bloom::{BloomBuilder, FieldBloom};
+use tantivy::Index;
+use tantivy_utils::puffin_directory::reader::warm_up_terms;
+
+/// Build per-field SBBFs for one file using a group-uniform `num_blocks`.
+///
+/// Every file in a (stream, hour, bloom_ver) group must pass the same
+/// `num_blocks` so the transposed `.bf` layout can read one block-row per
+/// group (see `infra::bloom` module docs). The caller derives it once from
+/// the configured expected cardinality.
+///
+/// Behavior:
+/// - Fields not present in the schema are silently skipped — the compactor passes the union of
+///   `index_fields ∩ bloom_filter_fields` over potentially many streams, and not every field exists
+///   everywhere.
+/// - Terms across all segments of the index are merged into one bloom per field. Today
+///   `create_tantivy_index` produces a single segment, but this is robust to that changing.
+pub(super) async fn build_blooms_from_index(
+    index: &Index,
+    file_id: u64,
+    fields: &[String],
+    num_blocks: u32,
+) -> Result<Vec<FieldBloom>, anyhow::Error> {
+    if fields.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let schema = index.schema();
+    let reader = index
+        .reader_builder()
+        .reload_policy(tantivy::ReloadPolicy::Manual)
+        .num_warming_threads(0)
+        .try_into()
+        .context("open tantivy reader")?;
+    let searcher = reader.searcher();
+
+    let warm_terms: HashMap<tantivy::schema::Field, HashMap<tantivy::Term, bool>> = HashMap::new();
+    let mut need_all_term_fields = HashSet::new();
+    for field in fields {
+        let Ok(field) = schema.get_field(field) else {
+            continue;
+        };
+        need_all_term_fields.insert(field);
+    }
+
+    // warm_up_terms operates on one SegmentReader at a time; warm each segment.
+    // need_all_term_fields / need_fast_field are consumed per call, so clone them.
+    for seg in searcher.segment_readers() {
+        warm_up_terms(
+            seg,
+            &warm_terms,
+            need_all_term_fields.clone(),
+            HashSet::new(),
+        )
+        .await?;
+    }
+
+    let mut builder = BloomBuilder::new();
+
+    for field_name in fields {
+        let Ok(field) = schema.get_field(field_name) else {
+            continue;
+        };
+
+        // Skip fields with no terms in this file so they don't become an
+        // empty column in the transposed matrix.
+        let mut has_terms = false;
+        for seg in searcher.segment_readers() {
+            if let Ok(inv) = seg.inverted_index(field)
+                && inv.terms().num_terms() > 0
+            {
+                has_terms = true;
+                break;
+            }
+        }
+        if !has_terms {
+            continue;
+        }
+
+        // Uniform block count across the whole group (caller-provided).
+        let idx = builder.begin_with_blocks(file_id, field_name, num_blocks);
+
+        for seg in searcher.segment_readers() {
+            let inv = match seg.inverted_index(field) {
+                Ok(i) => i,
+                Err(_) => continue,
+            };
+            let mut stream = inv
+                .terms()
+                .stream()
+                .with_context(|| format!("stream terms for {field_name}"))?;
+            while let Some((term_bytes, _info)) = stream.next() {
+                builder.insert(idx, term_bytes);
+            }
+        }
+    }
+
+    Ok(builder.finish())
+}

src/compaction/src/bloom/compact.rs

@@ -0,0 +1,349 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bloom filter build + orphan cleanup for the compactor.
+//!
+//! One public entry point, [`build_for_stream`], called once per hour bucket
+//! at the end of a compaction merge round. It does two things in order:
+//!
+//! 1. **Orphan cleanup.** Given the `bloom_ver` values of the files this merge round just deleted,
+//!    [`cleanup_orphan_blooms`] retires any `.bf` no longer referenced by a live row in the bucket.
+//! 2. **Build.** It queries the bucket itself (`query_for_bloom` → rows with `index_size > 0` and
+//!    `bloom_ver = 0`) and builds `.bf` coverage for those files. The caller does not pass a file
+//!    list; this module owns both the "which files" and the "how to bloom" decisions.
+//!
+//! `bloom_ver` is not append-only: a merge produces new output files with a
+//! fresh `bloom_ver` and deletes the small inputs, so an old `.bf` whose every
+//! file got merged away becomes an orphan — step 1 catches it.
+//!
+//! Failure is **always non-fatal**. A failed build leaves `bloom_ver = 0`
+//! on the affected files and the search side falls back to opening every
+//! tantivy file in the bucket. A failed cleanup leaves an orphan `.bf` for
+//! data-retention to reap. Neither affects correctness, so neither
+//! propagates to the caller as a hard error.
+
+use std::{collections::HashSet, sync::Arc};
+
+use anyhow::Context;
+use bytes::Bytes;
+use config::{
+    get_config,
+    meta::stream::{FileKey, FileListDeleted, StreamType},
+    utils::{inverted_index::to_tantivy_name, time::now_micros},
+};
+use infra::{
+    bloom::{BloomWriter, FieldBloom, path::bloom_path},
+    errors::Result,
+    file_list as infra_file_list, storage,
+};
+use tantivy::Directory;
+use tantivy_utils::puffin_directory::{
+    caching_directory::CachingDirectory, footer_cache::FooterCache, reader::PuffinDirReader,
+};
+
+use super::builder::build_blooms_from_index;
+
+/// Clean orphan `.bf`s for `orphan_blooms`, then build `.bf` coverage for the
+/// bucket's `bloom_ver = 0` files (queried internally via `query_for_bloom`).
+///
+/// `date_key` is the `YYYY/MM/DD/HH` string used in `file_list.date`.
+/// `orphan_blooms` is the `bloom_ver` values of the files the merge round just
+/// deleted — used only for cleanup. Files already carrying a non-zero
+/// `bloom_ver` are never re-stamped, so no live file migrates off a `bloom_ver`
+/// that a dumped row may still point to.
+///
+/// Returns `Ok(false)` when bloom is disabled or there is nothing to build (no
+/// target fields, nothing at `bloom_ver = 0`); `Ok(true)` when a build ran.
+/// Recoverable per-file/per-chunk errors are logged, not propagated — never
+/// surfaces a build failure to user requests.
+pub(crate) async fn build_for_stream(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    date_key: &str,
+    is_incremental: bool,
+    orphan_blooms: Vec<i64>,
+) -> Result<bool> {
+    let cfg = get_config();
+    if !cfg.common.bloom_filter_enabled {
+        return Ok(false);
+    }
+
+    // clean up orphan blooms
+    if let Err(e) =
+        cleanup_orphan_blooms(org_id, stream_type, stream_name, date_key, orphan_blooms).await
+    {
+        log::warn!("[BLOOM_BUILD] cleanup orphan blooms failed: {e}");
+    }
+
+    // don't build bloom for incremental round
+    if is_incremental {
+        return Ok(false);
+    }
+
+    // Resolve target fields from current stream settings: a field is bloomed
+    // only when the operator put it in BOTH `bloom_filter_fields` and
+    // `index_fields` (bloom is built on top of the tantivy index).
+    let stream_settings = infra::schema::get_settings(org_id, stream_name, stream_type).await;
+    let bloom_filter_fields =
+        infra::schema::get_stream_setting_bloom_filter_fields(&stream_settings);
+    if bloom_filter_fields.is_empty() {
+        return Ok(false);
+    }
+    let index_fields: HashSet<String> =
+        infra::schema::get_stream_setting_index_fields(&stream_settings)
+            .into_iter()
+            .collect();
+    let target_fields: Vec<String> = bloom_filter_fields
+        .into_iter()
+        .filter(|f| index_fields.contains(f))
+        .collect();
+    if target_fields.is_empty() {
+        return Ok(false);
+    }
+
+    // Pull every file currently in this hour bucket.
+    let mut files =
+        infra_file_list::query_for_bloom(org_id, stream_type, stream_name, date_key).await?;
+    if files.is_empty() {
+        return Ok(false);
+    }
+
+    // Build blooms only for files without a `.bf` yet (bloom_ver = 0).
+    //
+    // **Sub-grouping**: a single hour can hold thousands of files. Since the
+    // transposed `.bf` holds every file's SBBF in memory before serialize
+    // (M × B × 32 bytes), one giant `.bf` would OOM the compactor at high
+    // volume. So we split the new files into chunks of at most
+    // `max_files_per_bf` and write one `.bf` per chunk, each with its own
+    // `bloom_ver`. The search side groups by `(date, bloom_ver)`, so it
+    // naturally reads one block-row per chunk. Total read bytes stay
+    // `M × 32`; only the request count grows to `ceil(M / max_files_per_bf)`.
+    //
+    // Files are sorted by record count first so similar-sized files share a
+    // chunk — minimizes the uniform-`B` padding waste (B is sized to each
+    // chunk's max cardinality).
+    let fpp = cfg.common.bloom_filter_fpp;
+    let max_files_per_bf = cfg.common.bloom_filter_max_files_per_bf.max(1);
+    files.sort_by_key(|k| std::cmp::Reverse(k.meta.records));
+
+    let base_ver = now_micros();
+    let chunk_total = files.len().div_ceil(max_files_per_bf);
+    for (chunk_idx, chunk) in files.chunks(max_files_per_bf).enumerate() {
+        let bloom_ver = base_ver + chunk_idx as i64;
+        let bloom_path = bloom_path(org_id, stream_type, stream_name, date_key, bloom_ver);
+        match build_for_chunk(org_id, bloom_ver, &bloom_path, chunk, &target_fields, fpp).await {
+            Ok((took, num_blocks, contributing_ids)) => {
+                log::info!(
+                    "[BLOOM_BUILD] {bloom_path}: wrote chunk {}/{chunk_total}, num_blocks={num_blocks} covering {contributing_ids} files in {took} ms",
+                    chunk_idx + 1,
+                );
+            }
+            Err(e) => {
+                log::warn!("[BLOOM_BUILD] {bloom_path}: build chunk failed: {e}");
+            }
+        }
+    }
+
+    Ok(true)
+}
+
+// B for this chunk is sized from the chunk's MAX record count, used as
+// a safe NDV upper bound: a file can't hold more distinct values than
+// rows. This never under-sizes (no saturation) for the target
+// high-cardinality fields where distinct ≈ rows. It over-sizes for
+// fields that repeat heavily (distinct ≪ rows) — acceptable; exact
+// sizing would read each file's tantivy term count in a pre-pass.
+async fn build_for_chunk(
+    org_id: &str,
+    bloom_ver: i64,
+    bf_path: &str,
+    files: &[FileKey],
+    target_fields: &[String],
+    fpp: f64,
+) -> Result<(u64, u32, usize)> {
+    let start = std::time::Instant::now();
+
+    let max_records = files
+        .iter()
+        .map(|f| f.meta.records.max(0) as u64)
+        .max()
+        .unwrap_or(0)
+        .max(1);
+    let num_blocks = infra::bloom::num_blocks_for(max_records, fpp);
+
+    let mut all_blooms: Vec<FieldBloom> = Vec::new();
+    let mut contributing_ids: Vec<i64> = Vec::new();
+    for f in files {
+        match build_for_file(f, target_fields, num_blocks).await {
+            Ok(mut blooms) if !blooms.is_empty() => {
+                all_blooms.append(&mut blooms);
+                contributing_ids.push(f.id);
+            }
+            Ok(_) => {} // no blooms (no index / field absent) — leave at 0
+            Err(e) => {
+                log::warn!(
+                    "[BLOOM_BUILD] {bf_path}: skipping {} (bloom build failed): {e}",
+                    f.key
+                );
+            }
+        }
+    }
+    if all_blooms.is_empty() {
+        return Ok((0, 0, 0));
+    }
+
+    // Distinct bloom_ver per chunk (base + idx) → distinct `.bf` path.
+    let blob: Vec<u8> = BloomWriter::serialize(all_blooms).context("serialize blooms")?;
+    let bf_account = storage::get_account(org_id, bf_path).unwrap_or_default();
+    storage::put(&bf_account, bf_path, Bytes::from(blob))
+        .await
+        .context("upload blooms")?;
+
+    debug_assert!(
+        contributing_ids.iter().all(|id| *id > 0),
+        "bloom builder must only stamp file_list rows with assigned ids"
+    );
+    infra_file_list::update_bloom_ver(&contributing_ids, bloom_ver)
+        .await
+        .context("update bloom ver")?;
+
+    let took = start.elapsed().as_millis() as u64;
+
+    Ok((took, num_blocks, contributing_ids.len()))
+}
+
+async fn build_for_file(
+    file: &FileKey,
+    target_fields: &[String],
+    num_blocks: u32,
+) -> anyhow::Result<Vec<FieldBloom>> {
+    let Some(ttv_file_name) = to_tantivy_name(&file.key) else {
+        return Ok(Vec::new()); // not an indexable file
+    };
+    if file.meta.index_size == 0 {
+        return Ok(Vec::new()); // no .ttv was emitted
+    }
+    let file_account = file.account.clone();
+    let puffin_dir =
+        Arc::new(get_tantivy_directory(&file_account, &ttv_file_name, file.meta.index_size).await?);
+    let footer_cache = FooterCache::from_directory(puffin_dir.clone(), &ttv_file_name).await?;
+    let cache_dir = CachingDirectory::new_with_cacher(puffin_dir, Arc::new(footer_cache));
+    let reader_directory: Box<dyn Directory> = Box::new(cache_dir);
+    let index = tantivy::Index::open(reader_directory).context("open index")?;
+
+    // file.id is the file_list row id, assigned by the INSERT that
+    // happened in `write_file_list` before this build runs. Always > 0
+    // by the time we get here.
+    let file_id = file.id as u64;
+    build_blooms_from_index(&index, file_id, target_fields, num_blocks).await
+}
+
+/// Open a tantivy puffin directory for `file_name` in `file_account`.
+///
+/// Mirrors the search-side `get_tantivy_directory` helper: builds a synthetic
+/// `ObjectMeta` (the `.ttv` is immutable, so a fixed `last_modified` is fine)
+/// and opens a streaming puffin reader over it.
+async fn get_tantivy_directory(
+    file_account: &str,
+    file_name: &str,
+    file_size: i64,
+) -> anyhow::Result<PuffinDirReader> {
+    let file_account = file_account.to_string();
+    let source = object_store::ObjectMeta {
+        location: file_name.into(),
+        last_modified: *config::utils::time::BASE_TIME,
+        size: file_size as u64,
+        e_tag: None,
+        version: None,
+    };
+    Ok(PuffinDirReader::from_path(file_account, source).await?)
+}
+
+/// Retire any `.bf` whose `bloom_ver` is no longer referenced by a live file
+/// in this (stream, `date_key`) bucket.
+///
+/// `deleted_bloom_vers` is the set of `bloom_ver` values carried by files a
+/// merge round just deleted (the caller dedups / drops the 0 sentinel, but we
+/// guard again here). For each, we probe `file_list` for any surviving row at
+/// that version; if none, the `.bf` is enqueued for deletion via the existing
+/// `file_list_deleted` queue.
+///
+/// Orphan detection is a **cleanup optimization, not correctness**: if the
+/// EXISTS probe or the enqueue fails, we log and move on — the stale `.bf`
+/// stays until data-retention reaps the day's subtree. Errors never
+/// propagate.
+async fn cleanup_orphan_blooms(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    date_key: &str,
+    orphan_blooms: Vec<i64>,
+) -> Result<()> {
+    if orphan_blooms.is_empty() {
+        return Ok(());
+    }
+    let candidates: HashSet<i64> = orphan_blooms.into_iter().collect();
+
+    let mut orphans: Vec<FileListDeleted> = Vec::new();
+    for v_old in candidates {
+        match infra_file_list::bloom_ver_referenced(
+            org_id,
+            stream_type,
+            stream_name,
+            date_key,
+            v_old,
+        )
+        .await
+        {
+            Ok(true) => {} // still referenced by a live file — keep the `.bf`
+            Ok(false) => {
+                let path = bloom_path(org_id, stream_type, stream_name, date_key, v_old);
+                let account = storage::get_account(org_id, &path).unwrap_or_default();
+                orphans.push(FileListDeleted {
+                    id: 0,
+                    account,
+                    file: path,
+                    index_file: false, // a `.bf` has no companion `.ttv`
+                    flattened: false,
+                });
+            }
+            Err(e) => {
+                log::warn!(
+                    "[BLOOM_CLEANUP] {org_id}/{stream_type}/{stream_name}/{date_key}: \
+                     bloom_ver_referenced({v_old}) failed: {e}; leaving `.bf` for retention"
+                );
+            }
+        }
+    }
+    if orphans.is_empty() {
+        return Ok(());
+    }
+
+    let created_at = now_micros();
+    let n = orphans.len();
+    if let Err(e) = infra_file_list::batch_add_deleted(org_id, created_at, &orphans).await {
+        log::warn!(
+            "[BLOOM_CLEANUP] {org_id}/{stream_type}/{stream_name}/{date_key}: \
+             enqueue {n} orphan `.bf` for deletion failed: {e}"
+        );
+    } else {
+        log::info!(
+            "[BLOOM_CLEANUP] {org_id}/{stream_type}/{stream_name}/{date_key}: \
+             enqueued {n} orphan `.bf` for deletion"
+        );
+    }
+    Ok(())
+}

src/compaction/src/bloom/mod.rs

@@ -0,0 +1,26 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bloom-filter build side for compaction.
+//!
+//! - [`builder`] extracts per-(file, field) SBBFs from a tantivy term dictionary.
+//! - [`compact`] is the compactor entry point that owns "which files to bloom" and writes the
+//!   transposed `.bf` for each hour bucket.
+//!
+//! The search-side bloom pruner lives in the search crate, while the underlying
+//! SBBF format and reader/writer live in `infra::bloom`.
+
+mod builder;
+pub(crate) mod compact;

src/compaction/src/lib.rs

@@ -32,6 +32,7 @@ use infra::{
 use o2_enterprise::enterprise::common::downsampling::get_matching_downsampling_rules;
 use tokio::sync::mpsc;
 
+mod bloom;
 pub mod deleted;
 pub mod dump;
 pub mod flatten;

src/compaction/src/merge.rs

@@ -620,7 +620,32 @@ pub async fn merge_by_stream(
         orphan_blooms.extend(task.await??);
     }
 
-    let _ = (is_incremental, orphan_blooms);
+    // Build bloom filters for the current hour. Failures are non-fatal: files
+    // left at bloom_ver = 0 fall back to the regular Tantivy search path.
+    let build_start = std::time::Instant::now();
+    match crate::bloom::compact::build_for_stream(
+        org_id,
+        stream_type,
+        stream_name,
+        &date_start,
+        is_incremental,
+        orphan_blooms,
+    )
+    .await
+    {
+        Ok(false) => {}
+        Ok(true) => {
+            let build_time = build_start.elapsed().as_millis();
+            log::info!(
+                "[COMPACTOR] bloom build for {org_id}/{stream_type}/{stream_name}/{date_start} took: {build_time} ms"
+            );
+        }
+        Err(e) => {
+            log::warn!(
+                "[COMPACTOR] bloom build for {org_id}/{stream_type}/{stream_name}/{date_start} failed: {e}"
+            );
+        }
+    }
 
     // update job status
     if let Err(e) = infra_file_list::set_job_done(&[job_id]).await {

src/config/Cargo.toml

@@ -35,6 +35,7 @@ datafusion.workspace = true
 dashmap.workspace = true
 dotenv_config.workspace = true
 dotenvy.workspace = true
+fastdivide.workspace = true
 faststr.workspace = true
 futures.workspace = true
 local-ip-address.workspace = true

src/config/src/config.rs

@@ -1577,6 +1577,18 @@ pub struct Common {
     pub dashboard_placeholder: String,
     #[env_config(name = "ZO_AGGREGATION_TOPK_ENABLED", default = true)]
     pub aggregation_topk_enabled: bool,
+    #[env_config(
+        name = "ZO_DF_USE_AGG_TOPK_HEAP",
+        default = true,
+        help = "Use the heap implementation for eligible aggregate TopK plans"
+    )]
+    pub use_agg_topk_heap: bool,
+    #[env_config(
+        name = "ZO_DF_TOPK_HEAP_MAX_LIMIT",
+        default = 500,
+        help = "Maximum aggregate TopK limit that uses the heap implementation"
+    )]
+    pub agg_topk_heap_max_limit: u64,
     #[env_config(name = "ZO_SEARCH_INSPECTOR_ENABLED", default = false)]
     pub search_inspector_enabled: bool,
     #[env_config(name = "ZO_UTF8_VIEW_ENABLED", default = true)]

src/config/src/meta/slo/mod.rs

@@ -1796,13 +1796,6 @@ mod tests {
         ));
     }
 
-    // ---- time-slice threshold finiteness ------------------------------------
-
-    /// The threshold decides whether every bucket is good or bad. `NaN`
-    /// compares false against everything, so every slice classifies bad;
-    /// `±inf` classifies every slice the same way in the other direction.
-    /// Either way the SLO reports a confident, uniform, wrong answer.
-    #[test]
     /// A create request must not have to invent server-assigned fields.
     #[test]
     fn an_slo_deserializes_without_server_assigned_fields() {
@@ -1849,6 +1842,12 @@ mod tests {
         assert_eq!(SliType::from_storage_id(4), None);
     }
 
+    // ---- time-slice threshold finiteness ------------------------------------
+
+    /// The threshold decides whether every bucket is good or bad. `NaN`
+    /// compares false against everything, so every slice classifies bad;
+    /// `±inf` classifies every slice the same way in the other direction.
+    /// Either way the SLO reports a confident, uniform, wrong answer.
     #[test]
     fn a_non_finite_time_slice_threshold_is_rejected() {
         for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {

src/infra/src/file_list/mod.rs

@@ -88,7 +88,7 @@ pub trait FileList: Sync + Send + 'static {
     async fn update_flattened(&self, file: &str, flattened: bool) -> Result<()>;
     async fn update_compressed_size(&self, file: &str, size: i64) -> Result<()>;
     /// Bulk-set `bloom_ver` for the given file_list ids. Used by the
-    /// post-merge bloom builder (enterprise `bloom::compact`).
+    /// post-merge bloom builder (`compaction::bloom::compact`).
     /// Empty `ids` is a no-op.
     async fn update_bloom_ver(&self, ids: &[i64], bloom_ver: i64) -> Result<()>;
     /// Is `bloom_ver` still referenced by at least one live file_list row in

src/infra/src/table/migration/mod.rs

@@ -173,38 +173,6 @@ pub(crate) async fn create_slo_tables_for_test(
         .await
 }
 
-/// Apply the `alert_states` chain, for targeted integration tests (§7.6).
-///
-/// Three migrations because the level columns and the group-lifecycle columns
-/// arrived after the tables. Applied in order, as production would.
-#[cfg(test)]
-pub(crate) async fn create_alert_state_tables_for_test(
-    db: &sea_orm::DatabaseConnection,
-) -> Result<(), DbErr> {
-    use sea_orm::ConnectionTrait;
-    use sea_orm_migration::MigrationTrait;
-    let manager = SchemaManager::new(db);
-
-    // m20260725_000002 ALTERs `alerts` as well as `alert_states`, so the
-    // table has to exist. A minimal stand-in is enough and is honest about
-    // what the fixture provides: these tests are about alert *state*, not
-    // about the alerts table, and building the real one would mean replaying
-    // years of unrelated migrations.
-    db.execute_unprepared("CREATE TABLE IF NOT EXISTS alerts (id VARCHAR(27) PRIMARY KEY)")
-        .await?;
-
-    m20260725_000001_create_alert_states_tables::Migration
-        .up(&manager)
-        .await?;
-    m20260725_000002_add_threshold_and_level_columns::Migration
-        .up(&manager)
-        .await?;
-    m20260726_000003_add_group_lifecycle_columns::Migration
-        .up(&manager)
-        .await?;
-    Ok(())
-}
-
 pub struct Migrator;
 
 #[async_trait::async_trait]

src/search/Cargo.toml

@@ -29,6 +29,7 @@ flight.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 hashbrown.workspace = true
+hashlink.workspace = true
 infra.workspace = true
 itertools.workspace = true
 log.workspace = true

src/search/src/cache/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+pub mod streaming_agg;

src/search/src/cache/streaming_agg/discovery.rs

@@ -0,0 +1,484 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use config::meta::search::Interval;
+
+use super::files::STREAMING_AGGS_CACHE_DIR;
+
+/// Time conversion constants
+pub const MICROS_PER_SECOND: i64 = 1_000_000;
+pub const MICROS_PER_MINUTE: i64 = 60 * MICROS_PER_SECOND;
+
+/// Represents a single cache file entry with its metadata
+#[derive(Debug, Clone, PartialEq)]
+pub struct CacheEntry {
+    pub file_path: String,
+    pub start_time: i64,
+    pub end_time: i64,
+    pub interval: Interval,
+}
+
+impl CacheEntry {
+    /// Checks if this cache entry overlaps with the given time range
+    pub fn overlaps_with(&self, start: i64, end: i64, interval: Interval) -> bool {
+        self.start_time < end && self.end_time > start && self.interval <= interval
+    }
+}
+
+/// Represents a time range that is not covered by cache
+#[derive(Debug, Clone, PartialEq)]
+pub struct TimeRange {
+    pub start_time: i64,
+    pub end_time: i64,
+}
+
+impl TimeRange {
+    pub fn new(start_time: i64, end_time: i64) -> Self {
+        Self {
+            start_time,
+            end_time,
+        }
+    }
+}
+
+/// Result of cache discovery operation
+#[derive(Debug, Clone)]
+pub struct CacheDiscoveryResult {
+    pub cached_ranges: Vec<CacheEntry>,
+    pub uncached_ranges: Vec<TimeRange>,
+    pub cache_coverage_ratio: f64,
+}
+
+impl CacheDiscoveryResult {
+    pub fn new(
+        cached_ranges: Vec<CacheEntry>,
+        uncached_ranges: Vec<TimeRange>,
+        cache_coverage_ratio: f64,
+    ) -> Self {
+        Self {
+            cached_ranges,
+            uncached_ranges,
+            cache_coverage_ratio,
+        }
+    }
+
+    /// Creates an empty discovery result (no cache available) for the given time range
+    pub fn empty(start_time: i64, end_time: i64) -> Self {
+        Self {
+            cached_ranges: vec![],
+            uncached_ranges: vec![TimeRange::new(start_time, end_time)],
+            cache_coverage_ratio: 0.0,
+        }
+    }
+
+    /// Returns true if the entire query range is cached
+    pub fn is_fully_cached(&self) -> bool {
+        self.uncached_ranges.is_empty() && self.cache_coverage_ratio >= 1.0
+    }
+
+    /// Returns true if no cache is available
+    pub fn has_no_cache(&self) -> bool {
+        self.cache_coverage_ratio == 0.0
+    }
+}
+
+/// Discovers existing cache files for a given query
+///
+/// # Arguments
+/// * `cache_file_path` - The base cache file path (e.g., "org_id/stream_type/stream_name/hash")
+/// * `query_start` - Query start time in microseconds
+/// * `query_end` - Query end time in microseconds
+/// * `query_interval` - The interval of the query
+///
+/// # Returns
+/// * `Ok(CacheDiscoveryResult)` - Discovery result with cached and uncached ranges
+/// * `Err(std::io::Error)` - If cache directory cannot be accessed
+pub async fn discover_cache_for_query(
+    cache_file_path: &str,
+    query_start: i64,
+    query_end: i64,
+    query_interval: Interval,
+) -> std::io::Result<CacheDiscoveryResult> {
+    let cache_path = format!(
+        "{}{}/{}",
+        config::get_config().common.data_cache_dir,
+        STREAMING_AGGS_CACHE_DIR,
+        cache_file_path
+    );
+
+    // If cache directory doesn't exist, return empty result
+    if !tokio::fs::try_exists(&cache_path).await.unwrap_or(false) {
+        return Ok(CacheDiscoveryResult::new(
+            vec![],
+            vec![TimeRange::new(query_start, query_end)],
+            0.0,
+        ));
+    }
+
+    // Read all cache files in the directory asynchronously
+    let mut read_dir = tokio::fs::read_dir(&cache_path).await?;
+    let mut files = Vec::new();
+
+    // Collect all directory entries
+    while let Some(entry) = read_dir.next_entry().await? {
+        files.push(entry);
+    }
+
+    let mut cache_entries = Vec::new();
+
+    // Parse each cache file to extract metadata
+    for file in files {
+        let file_name = file.file_name();
+        let file_name_str = match file_name.to_str() {
+            Some(name) => name,
+            None => continue,
+        };
+
+        // Skip temporary files
+        if file_name_str.contains("_tmp") {
+            continue;
+        }
+
+        // Parse the cache entry
+        if let Some(entry) = parse_cache_file_name(cache_file_path, file_name_str) {
+            // Only include entries that overlap with the query range
+            if entry.overlaps_with(query_start, query_end, query_interval) {
+                cache_entries.push(entry);
+            }
+        }
+    }
+
+    // Sort cache entries by start time
+    cache_entries.sort_by_key(|e| e.start_time);
+
+    // Calculate uncached ranges (gaps in coverage)
+    let uncached_ranges = calculate_uncached_ranges(&cache_entries, query_start, query_end);
+
+    // Calculate cache coverage ratio
+    let total_duration = query_end - query_start;
+    let cached_duration = calculate_cached_duration(&cache_entries, query_start, query_end);
+    let cache_coverage_ratio = if total_duration > 0 {
+        cached_duration as f64 / total_duration as f64
+    } else {
+        0.0
+    };
+
+    Ok(CacheDiscoveryResult::new(
+        cache_entries,
+        uncached_ranges,
+        cache_coverage_ratio,
+    ))
+}
+
+/// Parses a cache file name to extract cache entry metadata
+fn parse_cache_file_name(cache_file_path: &str, file_name: &str) -> Option<CacheEntry> {
+    // Remove file extension before parsing timestamps
+    let name_without_ext = file_name
+        .rsplit_once('.')
+        .map(|(name, _)| name)
+        .unwrap_or(file_name);
+
+    let parts: Vec<&str> = name_without_ext.split('_').collect();
+    if parts.len() < 2 {
+        return None;
+    }
+
+    let start_time = parts[0].parse::<i64>().ok()?;
+    let end_time = parts[1].parse::<i64>().ok()?;
+
+    // Calculate interval from the time range duration in the filename
+    // The interval is inferred from the duration (end_time - start_time)
+    let duration_micros = end_time - start_time;
+    let interval = interval_from_duration_micros(duration_micros);
+
+    // Build the full cache file path
+    let full_cache_path = format!("{STREAMING_AGGS_CACHE_DIR}/{cache_file_path}/{file_name}",);
+
+    Some(CacheEntry {
+        file_path: full_cache_path,
+        start_time,
+        end_time,
+        interval,
+    })
+}
+
+/// Infers the interval enum from duration in microseconds
+///
+/// # Arguments
+/// * `duration_micros` - Duration in microseconds (expected range: 5 min to 1 day)
+///
+/// # Returns
+/// The corresponding `Interval` enum variant based on the duration in minutes
+fn interval_from_duration_micros(duration_micros: i64) -> Interval {
+    // Convert microseconds to minutes
+    let duration_minutes = duration_micros / MICROS_PER_MINUTE;
+    Interval::from(duration_minutes)
+}
+
+/// Calculates the uncached ranges (gaps) in the query time range
+fn calculate_uncached_ranges(
+    cache_entries: &[CacheEntry],
+    query_start: i64,
+    query_end: i64,
+) -> Vec<TimeRange> {
+    if cache_entries.is_empty() {
+        return vec![TimeRange::new(query_start, query_end)];
+    }
+
+    let mut uncached_ranges = Vec::new();
+    let mut current_time = query_start;
+
+    for entry in cache_entries {
+        // If there's a gap before this entry, add it as uncached
+        if current_time < entry.start_time {
+            uncached_ranges.push(TimeRange::new(
+                current_time,
+                entry.start_time.min(query_end),
+            ));
+        }
+
+        // Move current time to the end of this cached entry
+        current_time = current_time.max(entry.end_time);
+
+        // If we've covered the entire query range, stop
+        if current_time >= query_end {
+            break;
+        }
+    }
+
+    // If there's still time left after the last cache entry, add it as uncached
+    if current_time < query_end {
+        uncached_ranges.push(TimeRange::new(current_time, query_end));
+    }
+
+    uncached_ranges
+}
+
+/// Calculates the total cached duration within the query range
+fn calculate_cached_duration(
+    cache_entries: &[CacheEntry],
+    query_start: i64,
+    query_end: i64,
+) -> i64 {
+    let mut total_cached = 0i64;
+    let mut last_covered_time = query_start;
+
+    for entry in cache_entries {
+        // Calculate the overlap between this cache entry and the query range
+        let overlap_start = entry.start_time.max(last_covered_time);
+        let overlap_end = entry.end_time.min(query_end);
+
+        if overlap_start < overlap_end {
+            total_cached += overlap_end - overlap_start;
+            last_covered_time = overlap_end;
+        }
+
+        if last_covered_time >= query_end {
+            break;
+        }
+    }
+
+    total_cached
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_cache_entry_overlaps_with() {
+        let interval = Interval::OneHour;
+        let entry = CacheEntry {
+            file_path: "test.arrow".to_string(),
+            start_time: 1000,
+            end_time: 2000,
+            interval,
+        };
+
+        // Test overlapping ranges
+        assert!(entry.overlaps_with(500, 1500, interval)); // Starts before, ends during
+        assert!(entry.overlaps_with(1500, 2500, interval)); // Starts during, ends after
+        assert!(entry.overlaps_with(1200, 1800, interval)); // Completely within
+        assert!(entry.overlaps_with(500, 2500, interval)); // Completely encompasses
+
+        // Test non-overlapping ranges
+        assert!(!entry.overlaps_with(0, 1000, interval)); // Ends exactly at start
+        assert!(!entry.overlaps_with(2000, 3000, interval)); // Starts exactly at end
+        assert!(!entry.overlaps_with(0, 500, interval)); // Completely before
+        assert!(!entry.overlaps_with(2500, 3000, interval)); // Completely after
+    }
+
+    #[test]
+    fn test_interval_from_minutes() {
+        assert_eq!(Interval::from(0), Interval::Zero);
+        assert_eq!(Interval::from(5), Interval::FiveMinutes);
+        assert_eq!(Interval::from(10), Interval::TenMinutes);
+        assert_eq!(Interval::from(30), Interval::ThirtyMinutes);
+        assert_eq!(Interval::from(60), Interval::OneHour);
+        assert_eq!(Interval::from(120), Interval::TwoHours);
+        assert_eq!(Interval::from(360), Interval::SixHours);
+        assert_eq!(Interval::from(720), Interval::TwelveHours);
+        assert_eq!(Interval::from(1440), Interval::OneDay);
+        assert_eq!(Interval::from(999), Interval::Zero); // Unknown interval
+    }
+
+    #[test]
+    fn test_calculate_uncached_ranges_no_cache() {
+        let cache_entries = vec![];
+        let ranges = calculate_uncached_ranges(&cache_entries, 0, 10000);
+
+        assert_eq!(ranges.len(), 1);
+        assert_eq!(ranges[0].start_time, 0);
+        assert_eq!(ranges[0].end_time, 10000);
+    }
+
+    #[test]
+    fn test_calculate_uncached_ranges_fully_cached() {
+        let cache_entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 0,
+                end_time: 5000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 5000,
+                end_time: 10000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let ranges = calculate_uncached_ranges(&cache_entries, 0, 10000);
+        assert_eq!(ranges.len(), 0); // No gaps
+    }
+
+    #[test]
+    fn test_calculate_uncached_ranges_with_gaps() {
+        let cache_entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 1000,
+                end_time: 2000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 5000,
+                end_time: 7000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let ranges = calculate_uncached_ranges(&cache_entries, 0, 10000);
+
+        assert_eq!(ranges.len(), 3);
+        assert_eq!(ranges[0].start_time, 0);
+        assert_eq!(ranges[0].end_time, 1000);
+        assert_eq!(ranges[1].start_time, 2000);
+        assert_eq!(ranges[1].end_time, 5000);
+        assert_eq!(ranges[2].start_time, 7000);
+        assert_eq!(ranges[2].end_time, 10000);
+    }
+
+    #[test]
+    fn test_calculate_cached_duration_no_cache() {
+        let cache_entries = vec![];
+        let duration = calculate_cached_duration(&cache_entries, 0, 10000);
+        assert_eq!(duration, 0);
+    }
+
+    #[test]
+    fn test_calculate_cached_duration_fully_cached() {
+        let cache_entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 0,
+                end_time: 5000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 5000,
+                end_time: 10000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let duration = calculate_cached_duration(&cache_entries, 0, 10000);
+        assert_eq!(duration, 10000);
+    }
+
+    #[test]
+    fn test_calculate_cached_duration_partial() {
+        let cache_entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 1000,
+                end_time: 4000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 6000,
+                end_time: 9000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let duration = calculate_cached_duration(&cache_entries, 0, 10000);
+        assert_eq!(duration, 6000); // 3000 + 3000
+    }
+
+    #[test]
+    fn test_calculate_cached_duration_overlapping() {
+        // This tests the case where cache entries might overlap (shouldn't double count)
+        let cache_entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 0,
+                end_time: 6000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 4000,
+                end_time: 10000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let duration = calculate_cached_duration(&cache_entries, 0, 10000);
+        assert_eq!(duration, 10000); // Should cover 0-10000 without double counting
+    }
+
+    #[test]
+    fn test_cache_discovery_result_states() {
+        // Fully cached
+        let result = CacheDiscoveryResult::new(vec![], vec![], 1.0);
+        assert!(result.is_fully_cached());
+        assert!(!result.has_no_cache());
+
+        // Partial cache
+        let result = CacheDiscoveryResult::new(vec![], vec![], 0.5);
+        assert!(!result.is_fully_cached());
+        assert!(!result.has_no_cache());
+
+        // No cache
+        let result = CacheDiscoveryResult::new(vec![], vec![], 0.0);
+        assert!(!result.is_fully_cached());
+        assert!(result.has_no_cache());
+    }
+}

src/search/src/cache/streaming_agg/files.rs

@@ -0,0 +1,657 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    io::Cursor,
+    path::Path,
+    sync::{Arc, LazyLock},
+};
+
+use arrow::{
+    array::{ArrayRef, RecordBatch, RecordBatchOptions},
+    compute::cast,
+    datatypes::{DataType, Field, SchemaRef},
+    ipc::{reader::FileReader as ArrowFileReader, writer::FileWriter as ArrowFileWriter},
+};
+use config::{
+    meta::search::SearchPartitionRequest,
+    utils::{
+        record_batch_ext::RecordBatchExt,
+        time::{now_micros, second_micros},
+    },
+};
+use hashbrown::HashMap;
+use infra::cache::file_data::disk;
+use tokio::sync::mpsc;
+
+use crate::datafusion::aggregates::gc_string_view_batch;
+
+pub const STREAMING_AGGS_CACHE_DIR: &str = "aggregations";
+
+#[derive(Debug)]
+pub struct RecordBatchCacheRequest {
+    pub streaming_id: String,
+    pub file_path: String,
+    pub schema: SchemaRef,
+    pub records: Vec<Arc<RecordBatch>>,
+    pub overwrite_cache: bool,
+}
+
+// Global queue for cache requests
+static CACHE_QUEUE: LazyLock<mpsc::UnboundedSender<(String, String)>> = LazyLock::new(|| {
+    let (sender, mut receiver) = mpsc::unbounded_channel::<(String, String)>();
+
+    // Spawn background task to process cache requests
+    tokio::spawn(async move {
+        while let Some((streaming_id, file_key)) = receiver.recv().await {
+            log::debug!("[streaming_id: {streaming_id}] Received cache request");
+            if let Err(e) = load_record_batches_file_to_disk_cache(&streaming_id, &file_key).await {
+                log::error!(
+                    "[streaming_id: {streaming_id}] Failed to load record batches file to disk cache: {e:?}"
+                );
+            }
+        }
+    });
+
+    sender
+});
+
+async fn load_record_batches_file_to_disk_cache(
+    streaming_id: &str,
+    file_key: &str,
+) -> Result<(), std::io::Error> {
+    // Skip caching if record batch for the time range is already cached
+    let Some(file_path) = disk::get_file_path(file_key) else {
+        return Ok(()); // no need to cache, it's not a valid file path
+    };
+    let Some(file_meta) = config::utils::file::get_file_meta(&file_path).ok() else {
+        return Ok(()); // no need to cache, it's not a valid file path
+    };
+    let file_size = file_meta.len();
+    if file_size == 0 {
+        return Ok(()); // no need to cache, it's not a valid file
+    }
+
+    log::debug!(
+        "load_record_batches_file_to_disk_cache: streaming_id: {streaming_id}, file_key: {file_key}, file_size: {file_size}"
+    );
+
+    // set to disk cache
+    disk::set_size(file_key, file_size as usize)
+        .await
+        .map_err(|e| std::io::Error::other(format!("Failed to set size to disk cache: {e}")))?;
+
+    Ok(())
+}
+
+// Main handler to write record batches to disk
+pub fn cache_record_batches_to_disk(
+    request: RecordBatchCacheRequest,
+) -> Result<(), std::io::Error> {
+    let start = std::time::Instant::now();
+    let RecordBatchCacheRequest {
+        streaming_id,
+        file_path,
+        schema,
+        records,
+        overwrite_cache,
+    } = request;
+
+    // Skip caching if record batch for the time range is already cached
+    let file_key = file_path.clone();
+    let Some(file_path) = disk::get_file_path(&file_path) else {
+        return Err(std::io::Error::other(
+            "ZO_DISK_CACHE_ENABLED is not enabled",
+        ));
+    };
+    let file_meta = config::utils::file::get_file_meta(&file_path).ok();
+    let file_exists = file_meta.is_some() && file_meta.unwrap().is_file();
+    if file_exists && !overwrite_cache {
+        log::warn!(
+            "[streaming_id: {streaming_id}] file_exists: {file_exists}, Skipping cache to disk because the data for the time range is already cached",
+        );
+        return Ok(());
+    }
+
+    if file_exists && overwrite_cache {
+        log::info!(
+            "[streaming_id: {streaming_id}] file_exists: {file_exists}, overwrite_cache: {overwrite_cache}, Overwriting existing cache file",
+        );
+    }
+
+    let batches_num = records.len();
+    let rows_num = records.iter().map(|r| r.num_rows()).sum::<usize>();
+    let batches = records
+        .iter()
+        .map(|r| Arc::new(gc_string_view_batch(r)))
+        .collect::<Vec<Arc<RecordBatch>>>();
+
+    // Serialize the record batches into bytes
+    let data = match serialize_record_batches(schema, batches) {
+        Ok(data) => data,
+        Err(e) => {
+            log::error!("[streaming_id: {streaming_id}] Failed to serialize record batches: {e:?}",);
+            return Err(std::io::Error::other("Serialization failed"));
+        }
+    };
+
+    // create the directory if it doesn't exist
+    std::fs::create_dir_all(Path::new(&file_path).parent().unwrap())?;
+    // write the data to the file
+    match config::utils::file::put_file_contents(&file_path, &data) {
+        Ok(_) => {
+            log::info!(
+                "cache_record_batches_to_disk: streaming_id: {streaming_id}, file_path: {file_path}, batches: {batches_num}, rows: {rows_num}, write to file took: {} ms",
+                start.elapsed().as_millis()
+            );
+
+            // add to cache list
+            // Send to background queue (non-blocking)
+            if let Err(e) = CACHE_QUEUE.send((streaming_id.clone(), file_key)) {
+                log::error!(
+                    "[streaming_id: {streaming_id}] Failed to queue cache file to disk: {file_path}, error: {e:?}",
+                );
+            }
+
+            Ok(())
+        }
+        Err(e) => {
+            log::error!("Error caching results to disk: {e:?}");
+            Err(std::io::Error::other(format!(
+                "[streaming_id: {streaming_id}] Error caching results to disk: file_path={file_path}"
+            )))
+        }
+    }
+}
+
+// write to arrow ipc format
+fn serialize_record_batches(
+    schema: SchemaRef,
+    batches: Vec<Arc<RecordBatch>>,
+) -> arrow::error::Result<Vec<u8>> {
+    let mut buffer = Cursor::new(Vec::new());
+    let mut writer = ArrowFileWriter::try_new(&mut buffer, &schema)?;
+
+    for batch in batches {
+        writer.write(&batch)?;
+    }
+
+    writer.finish()?;
+    Ok(buffer.into_inner())
+}
+
+pub fn get_record_batches(
+    streaming_id: &str,
+    file_path: &str,
+    schema: SchemaRef,
+) -> std::io::Result<Vec<RecordBatch>> {
+    let start = std::time::Instant::now();
+    let file_path = disk::get_file_path(file_path).ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::NotFound,
+            format!("File not found: {file_path}"),
+        )
+    })?;
+    let data = config::utils::file::get_file_contents(&file_path, None).map_err(|e| {
+        log::error!("Error getting file contents: {e}, file_path: {file_path}");
+        e
+    })?;
+    let reader = unsafe {
+        ArrowFileReader::try_new(Cursor::new(data), None)
+            .map_err(|e| {
+                log::error!("Error creating arrow reader: {e}, file_path: {file_path}");
+                std::io::Error::other(format!("Arrow error: {e}"))
+            })?
+            .with_skip_validation(true)
+    };
+    let schema_field_map = schema
+        .fields()
+        .iter()
+        .map(|f| (f.name(), f.data_type()))
+        .collect::<HashMap<&String, &DataType>>();
+    let mut batches = Vec::new();
+    for batch in reader {
+        let batch = batch.map_err(|e| std::io::Error::other(format!("Arrow error: {e}")))?;
+        let new_columns: Vec<ArrayRef> = batch
+            .columns()
+            .iter()
+            .zip(batch.schema().fields().iter())
+            .map(|(c, f)| {
+                let file_datatype = f.data_type();
+                let need_datatype = *schema_field_map.get(f.name()).unwrap_or(&&DataType::Null);
+
+                // Use the recursive cast function
+                if let Some(casted) = cast_array_recursive(c, file_datatype, need_datatype) {
+                    casted
+                } else {
+                    Arc::clone(c)
+                }
+            })
+            .collect();
+        let mut options = RecordBatchOptions::new();
+        options = options.with_row_count(Some(batch.num_rows()));
+        let batch = RecordBatch::try_new_with_options(schema.clone(), new_columns, &options)
+            .expect("Failed to re-create the record batch");
+
+        batches.push(batch);
+    }
+
+    log::debug!(
+        "get_record_batches: streaming_id: {streaming_id}, file_path: {file_path}, batches: {}, rows: {}, arrow_size: {}, took: {} ms",
+        batches.len(),
+        batches.iter().map(|r| r.num_rows()).sum::<usize>(),
+        batches.iter().map(|r| r.size()).sum::<usize>(),
+        start.elapsed().as_millis()
+    );
+
+    Ok(batches)
+}
+
+/// Recursively checks if two data types need casting and performs the cast if needed.
+/// This function handles:
+/// - String type conversions (Utf8, LargeUtf8, Utf8View)
+/// - Nested List types (recursively checks inner types)
+/// - Nested Struct types (recursively checks field types)
+/// - Other types (returns None if no cast needed)
+///
+/// # Arguments
+/// * `array` - The array to potentially cast
+/// * `from_type` - The source data type
+/// * `to_type` - The target data type
+///
+/// # Returns
+/// * `Some(ArrayRef)` - If casting was needed and successful
+/// * `None` - If no casting is needed (types are compatible)
+fn cast_array_recursive(
+    array: &ArrayRef,
+    from_type: &DataType,
+    to_type: &DataType,
+) -> Option<ArrayRef> {
+    // If types are identical, no cast needed
+    if from_type == to_type {
+        return None;
+    }
+
+    match (from_type, to_type) {
+        // Handle string type conversions
+        (
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View,
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View,
+        ) => {
+            // Cast between string types
+            cast(array, to_type).ok()
+        }
+
+        // Handle List types recursively
+        (DataType::List(from_field), DataType::List(to_field)) => {
+            let from_inner = from_field.data_type();
+            let to_inner = to_field.data_type();
+
+            // If inner types need casting, create a new List type with the target inner type
+            if needs_recursive_cast(from_inner, to_inner) {
+                // Create a new field with the target data type
+                let new_field = Arc::new(Field::new(
+                    to_field.name(),
+                    to_inner.clone(),
+                    to_field.is_nullable(),
+                ));
+                let new_list_type = DataType::List(new_field);
+                cast(array, &new_list_type).ok()
+            } else {
+                None
+            }
+        }
+
+        // Handle LargeList types recursively
+        (DataType::LargeList(from_field), DataType::LargeList(to_field)) => {
+            let from_inner = from_field.data_type();
+            let to_inner = to_field.data_type();
+
+            if needs_recursive_cast(from_inner, to_inner) {
+                let new_field = Arc::new(Field::new(
+                    to_field.name(),
+                    to_inner.clone(),
+                    to_field.is_nullable(),
+                ));
+                let new_list_type = DataType::LargeList(new_field);
+                cast(array, &new_list_type).ok()
+            } else {
+                None
+            }
+        }
+
+        // Handle Struct types recursively
+        (DataType::Struct(from_fields), DataType::Struct(to_fields)) => {
+            // Check if any field needs casting
+            let needs_cast =
+                from_fields
+                    .iter()
+                    .zip(to_fields.iter())
+                    .any(|(from_field, to_field)| {
+                        needs_recursive_cast(from_field.data_type(), to_field.data_type())
+                    });
+
+            if needs_cast {
+                cast(array, to_type).ok()
+            } else {
+                None
+            }
+        }
+
+        // For all other type combinations, no cast is performed
+        _ => None,
+    }
+}
+
+/// Helper function to check if two types need recursive casting
+fn needs_recursive_cast(from_type: &DataType, to_type: &DataType) -> bool {
+    if from_type == to_type {
+        return false;
+    }
+
+    match (from_type, to_type) {
+        // String types can be cast between each other
+        (
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View,
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View,
+        ) => true,
+
+        // Recursively check List types
+        (DataType::List(from_field), DataType::List(to_field)) => {
+            needs_recursive_cast(from_field.data_type(), to_field.data_type())
+        }
+
+        // Recursively check LargeList types
+        (DataType::LargeList(from_field), DataType::LargeList(to_field)) => {
+            needs_recursive_cast(from_field.data_type(), to_field.data_type())
+        }
+
+        // Recursively check Struct types
+        (DataType::Struct(from_fields), DataType::Struct(to_fields)) => {
+            from_fields.len() == to_fields.len()
+                && from_fields
+                    .iter()
+                    .zip(to_fields.iter())
+                    .any(|(from_field, to_field)| {
+                        needs_recursive_cast(from_field.data_type(), to_field.data_type())
+                    })
+        }
+
+        _ => false,
+    }
+}
+
+pub fn create_aggregation_cache_file_path(
+    org_id: &str,
+    stream_type: &str,
+    stream_name: &str,
+    hashed_query: u64,
+) -> String {
+    if org_id.is_empty() || stream_type.is_empty() || stream_name.is_empty() {
+        return "".to_string();
+    }
+    // eg: /org_id/stream_type/stream_name/12345678
+    // Note: interval is NOT included in the path anymore - all intervals share the same directory
+    format!("{org_id}/{stream_type}/{stream_name}/{hashed_query}")
+}
+
+pub fn generate_aggregation_cache_file_name(
+    id: &str,
+    start_time: i64,
+    end_time: i64,
+    is_complete_partition_window: bool,
+) -> String {
+    // set cache as tmp if the time range is within the delay window
+    let delay_window_micros = second_micros(config::get_config().limit.cache_delay_secs);
+    let skip_cache = now_micros() - delay_window_micros;
+    let can_be_cached = end_time < skip_cache;
+
+    let is_tmp_file = if is_complete_partition_window && can_be_cached {
+        "".to_string()
+    } else {
+        format!("_{id}_tmp")
+    };
+    format!("{start_time}_{end_time}{is_tmp_file}.arrow")
+}
+
+pub fn get_cache_file_path(file_path: &str, file_name: &str) -> String {
+    format!("{STREAMING_AGGS_CACHE_DIR}/{file_path}/{file_name}")
+}
+
+pub fn get_aggregation_cache_key_from_request(req: &SearchPartitionRequest) -> u64 {
+    let origin_sql = req.sql.clone();
+
+    let mut hash_body = vec![origin_sql];
+    if let Some(vrl_function) = &req.query_fn {
+        hash_body.push(vrl_function.to_string());
+    }
+    if !req.regions.is_empty() {
+        hash_body.extend(req.regions.clone());
+    }
+    if !req.clusters.is_empty() {
+        hash_body.extend(req.clusters.clone());
+    }
+    config::utils::hash::sum64(&hash_body.join(","))
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow::array::StringArray;
+
+    use super::*;
+
+    #[test]
+    fn test_cast_array_recursive_string_types() {
+        use arrow::array::StringArray;
+
+        // Test Utf8 to Utf8View
+        let array: ArrayRef = Arc::new(StringArray::from(vec!["hello", "world"]));
+        let result = cast_array_recursive(&array, &DataType::Utf8, &DataType::Utf8View);
+        assert!(result.is_some());
+        let casted = result.unwrap();
+        assert_eq!(casted.data_type(), &DataType::Utf8View);
+
+        // Test Utf8View to LargeUtf8
+        let array: ArrayRef = Arc::new(StringArray::from(vec!["foo", "bar"]));
+        let result = cast_array_recursive(&array, &DataType::Utf8View, &DataType::LargeUtf8);
+        assert!(result.is_some());
+
+        // Test LargeUtf8 to Utf8
+        let array: ArrayRef = Arc::new(StringArray::from(vec!["test"]));
+        let result = cast_array_recursive(&array, &DataType::LargeUtf8, &DataType::Utf8);
+        assert!(result.is_some());
+    }
+
+    #[test]
+    fn test_cast_array_recursive_identical_types() {
+        use arrow::array::Int32Array;
+
+        // Test that identical types return None (no cast needed)
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        let result = cast_array_recursive(&array, &DataType::Int32, &DataType::Int32);
+        assert!(result.is_none());
+
+        // Test string types
+        let array: ArrayRef = Arc::new(StringArray::from(vec!["test"]));
+        let result = cast_array_recursive(&array, &DataType::Utf8, &DataType::Utf8);
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_cast_array_recursive_list_with_string() {
+        use arrow::array::{ListArray, StringArray};
+
+        // Create a List<Utf8> array
+        let values = StringArray::from(vec!["a", "b", "c", "d"]);
+        let offsets = arrow::buffer::OffsetBuffer::new(vec![0, 2, 4].into());
+        let field = Arc::new(Field::new("item", DataType::Utf8, true));
+        let list_array = ListArray::new(field.clone(), offsets, Arc::new(values), None);
+        let array: ArrayRef = Arc::new(list_array);
+
+        // Cast from List<Utf8> to List<Utf8View>
+        let from_type = DataType::List(Arc::new(Field::new("item", DataType::Utf8, true)));
+        let to_type = DataType::List(Arc::new(Field::new("item", DataType::Utf8View, true)));
+
+        let result = cast_array_recursive(&array, &from_type, &to_type);
+        assert!(result.is_some());
+
+        let casted = result.unwrap();
+        if let DataType::List(inner_field) = casted.data_type() {
+            assert_eq!(inner_field.data_type(), &DataType::Utf8View);
+        } else {
+            panic!("Expected List type");
+        }
+    }
+
+    #[test]
+    fn test_cast_array_recursive_nested_list() {
+        use arrow::array::{ListArray, StringArray};
+
+        // Create a List<List<Utf8>> structure
+        let inner_values = StringArray::from(vec!["x", "y"]);
+        let inner_offsets = arrow::buffer::OffsetBuffer::new(vec![0, 1, 2].into());
+        let inner_field = Arc::new(Field::new("item", DataType::Utf8, true));
+        let inner_list = ListArray::new(
+            inner_field.clone(),
+            inner_offsets,
+            Arc::new(inner_values),
+            None,
+        );
+
+        let outer_offsets = arrow::buffer::OffsetBuffer::new(vec![0, 2].into());
+        let outer_field = Arc::new(Field::new(
+            "item",
+            DataType::List(inner_field.clone()),
+            true,
+        ));
+        let outer_list = ListArray::new(outer_field, outer_offsets, Arc::new(inner_list), None);
+        let array: ArrayRef = Arc::new(outer_list);
+
+        // Cast from List<List<Utf8>> to List<List<LargeUtf8>>
+        let from_type = DataType::List(Arc::new(Field::new(
+            "item",
+            DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
+            true,
+        )));
+        let to_type = DataType::List(Arc::new(Field::new(
+            "item",
+            DataType::List(Arc::new(Field::new("item", DataType::LargeUtf8, true))),
+            true,
+        )));
+
+        let result = cast_array_recursive(&array, &from_type, &to_type);
+        assert!(result.is_some());
+    }
+
+    #[test]
+    fn test_cast_array_recursive_struct_with_string() {
+        use arrow::{
+            array::{Int32Array, StringArray, StructArray},
+            datatypes::Fields,
+        };
+
+        // Create a Struct with (name: Utf8, age: Int32)
+        let name_array = Arc::new(StringArray::from(vec!["Alice", "Bob"]));
+        let age_array = Arc::new(Int32Array::from(vec![30, 25]));
+
+        let from_fields = Fields::from(vec![
+            Field::new("name", DataType::Utf8, false),
+            Field::new("age", DataType::Int32, false),
+        ]);
+        let struct_array = StructArray::new(
+            from_fields.clone(),
+            vec![name_array as ArrayRef, age_array as ArrayRef],
+            None,
+        );
+        let array: ArrayRef = Arc::new(struct_array);
+
+        // Cast to Struct with (name: Utf8View, age: Int32)
+        let from_type = DataType::Struct(from_fields);
+        let to_fields = Fields::from(vec![
+            Field::new("name", DataType::Utf8View, false),
+            Field::new("age", DataType::Int32, false),
+        ]);
+        let to_type = DataType::Struct(to_fields.clone());
+
+        let result = cast_array_recursive(&array, &from_type, &to_type);
+        assert!(result.is_some());
+
+        let casted = result.unwrap();
+        if let DataType::Struct(fields) = casted.data_type() {
+            assert_eq!(fields[0].data_type(), &DataType::Utf8View);
+            assert_eq!(fields[1].data_type(), &DataType::Int32);
+        } else {
+            panic!("Expected Struct type");
+        }
+    }
+
+    #[test]
+    fn test_needs_recursive_cast_string_types() {
+        // String type variations should return true
+        assert!(needs_recursive_cast(&DataType::Utf8, &DataType::Utf8View));
+        assert!(needs_recursive_cast(&DataType::LargeUtf8, &DataType::Utf8));
+        assert!(needs_recursive_cast(
+            &DataType::Utf8View,
+            &DataType::LargeUtf8
+        ));
+
+        // Same string type should return false
+        assert!(!needs_recursive_cast(&DataType::Utf8, &DataType::Utf8));
+    }
+
+    #[test]
+    fn test_needs_recursive_cast_list_types() {
+        // List with different inner types
+        let from_list = DataType::List(Arc::new(Field::new("item", DataType::Utf8, true)));
+        let to_list = DataType::List(Arc::new(Field::new("item", DataType::Utf8View, true)));
+        assert!(needs_recursive_cast(&from_list, &to_list));
+
+        // List with same inner types
+        let same_list = DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
+        assert!(!needs_recursive_cast(&same_list, &same_list));
+    }
+
+    #[test]
+    fn test_needs_recursive_cast_struct_types() {
+        use arrow::datatypes::Fields;
+
+        // Struct with different field types
+        let from_fields = Fields::from(vec![Field::new("name", DataType::Utf8, false)]);
+        let to_fields = Fields::from(vec![Field::new("name", DataType::Utf8View, false)]);
+        let from_struct = DataType::Struct(from_fields);
+        let to_struct = DataType::Struct(to_fields);
+        assert!(needs_recursive_cast(&from_struct, &to_struct));
+
+        // Struct with same field types
+        let same_fields = Fields::from(vec![Field::new("id", DataType::Int32, false)]);
+        let same_struct = DataType::Struct(same_fields);
+        assert!(!needs_recursive_cast(&same_struct, &same_struct));
+    }
+
+    #[test]
+    fn test_cast_array_recursive_incompatible_types() {
+        use arrow::array::{Int32Array, StringArray};
+
+        // Test that incompatible types return None
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        let result = cast_array_recursive(&array, &DataType::Int32, &DataType::Float64);
+        assert!(result.is_none());
+
+        // Test string to int (should return None as we don't handle this)
+        let array: ArrayRef = Arc::new(StringArray::from(vec!["test"]));
+        let result = cast_array_recursive(&array, &DataType::Utf8, &DataType::Int32);
+        assert!(result.is_none());
+    }
+}

src/search/src/cache/streaming_agg/mod.rs

@@ -0,0 +1,27 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Streaming aggregation cache implementation
+//!
+//! This module provides cache-aware partition generation for streaming aggregations,
+//! including cache discovery, loading, and partition optimization.
+mod discovery;
+mod files;
+mod partition_optimizer;
+
+// Re-export public APIs
+pub use discovery::*;
+pub use files::*;
+pub use partition_optimizer::*;

src/search/src/cache/streaming_agg/partition_optimizer.rs

@@ -0,0 +1,615 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use config::meta::{
+    search::{CardinalityLevel, Interval, generate_aggregation_search_interval},
+    sql::OrderBy,
+};
+
+use super::discovery::{CacheDiscoveryResult, CacheEntry, TimeRange};
+
+/// Strategy for handling query partitions based on cache availability
+#[derive(Debug, Clone)]
+pub enum StreamingAggsPartitionStrategy {
+    /// All data is available in cache - no query execution needed
+    FullyCached { cache_files: Vec<CacheEntry> },
+    /// Mix of cached and uncached data
+    Hybrid {
+        cached_partitions: Vec<CachedPartition>,
+        uncached_partitions: Vec<UncachedPartition>,
+    },
+    /// No cache available - execute query normally
+    NoCacheAvailable { partitions: Vec<UncachedPartition> },
+}
+
+impl StreamingAggsPartitionStrategy {
+    /// Strategy Name
+    pub fn strategy_name(&self) -> &str {
+        match self {
+            StreamingAggsPartitionStrategy::FullyCached { .. } => "FullyCached",
+            StreamingAggsPartitionStrategy::Hybrid { .. } => "Hybrid",
+            StreamingAggsPartitionStrategy::NoCacheAvailable { .. } => "NoCacheAvailable",
+        }
+    }
+
+    /// Returns true if this strategy requires query execution
+    pub fn requires_execution(&self) -> bool {
+        match self {
+            StreamingAggsPartitionStrategy::FullyCached { .. } => false,
+            StreamingAggsPartitionStrategy::Hybrid { .. }
+            | StreamingAggsPartitionStrategy::NoCacheAvailable { .. } => true,
+        }
+    }
+
+    /// Returns the number of partitions that need to be executed
+    pub fn execution_partition_count(&self) -> usize {
+        match self {
+            StreamingAggsPartitionStrategy::FullyCached { .. } => 0,
+            StreamingAggsPartitionStrategy::Hybrid {
+                uncached_partitions,
+                ..
+            } => uncached_partitions.len(),
+            StreamingAggsPartitionStrategy::NoCacheAvailable { partitions } => partitions.len(),
+        }
+    }
+
+    /// Converts the partition strategy into time range partitions [start, end]
+    /// This is the format expected by the rest of the search system
+    ///
+    /// # Arguments
+    /// * `order_by` - The sort order (Asc or Desc) for partitions
+    ///
+    /// # Returns
+    /// Vector of [start_time, end_time] pairs in the requested order
+    pub fn to_time_partitions(&self, order_by: OrderBy) -> Vec<[i64; 2]> {
+        let mut partitions = Vec::new();
+
+        match self {
+            StreamingAggsPartitionStrategy::FullyCached { cache_files } => {
+                // For fully cached queries, return a SINGLE partition covering the entire range
+                // This avoids unnecessary iteration over individual cache files
+                if !cache_files.is_empty() {
+                    let min_start = cache_files.iter().map(|f| f.start_time).min().unwrap();
+                    let max_end = cache_files.iter().map(|f| f.end_time).max().unwrap();
+                    partitions.push([min_start, max_end]);
+                }
+            }
+            StreamingAggsPartitionStrategy::Hybrid {
+                cached_partitions,
+                uncached_partitions,
+            } => {
+                // Add cached partitions
+                for cached in cached_partitions {
+                    partitions.push([cached.start_time, cached.end_time]);
+                }
+
+                // Add uncached partitions
+                for uncached in uncached_partitions {
+                    partitions.push([uncached.start_time, uncached.end_time]);
+                }
+
+                // Sort by start time to ensure chronological order
+                partitions.sort_by_key(|p| p[0]);
+            }
+            StreamingAggsPartitionStrategy::NoCacheAvailable { partitions: p } => {
+                for partition in p {
+                    partitions.push([partition.start_time, partition.end_time]);
+                }
+            }
+        }
+
+        // Apply ordering
+        if order_by == OrderBy::Desc {
+            partitions.reverse();
+        }
+
+        partitions
+    }
+}
+
+/// Represents a partition that is fully covered by cache
+#[derive(Debug, Clone)]
+pub struct CachedPartition {
+    pub cache_files: Vec<CacheEntry>,
+    pub start_time: i64,
+    pub end_time: i64,
+    pub interval: Interval,
+}
+
+impl CachedPartition {
+    pub fn new(
+        cache_files: Vec<CacheEntry>,
+        start_time: i64,
+        end_time: i64,
+        interval: Interval,
+    ) -> Self {
+        Self {
+            cache_files,
+            start_time,
+            end_time,
+            interval,
+        }
+    }
+}
+
+/// Represents a partition that needs to be executed (not in cache)
+#[derive(Debug, Clone)]
+pub struct UncachedPartition {
+    pub start_time: i64,
+    pub end_time: i64,
+}
+
+impl UncachedPartition {
+    pub fn new(start_time: i64, end_time: i64) -> Self {
+        Self {
+            start_time,
+            end_time,
+        }
+    }
+}
+
+/// Generates optimal partition strategy based on cache discovery results
+///
+/// # Arguments
+/// * `discovery_result` - Result from cache discovery
+/// * `query_start` - Query start time in microseconds
+/// * `query_end` - Query end time in microseconds
+/// * `cardinality_level` - Cardinality level for determining cache intervals
+///
+/// # Returns
+/// * `PartitionStrategy` - Optimal strategy for executing the query
+pub fn generate_optimal_partitions(
+    discovery_result: CacheDiscoveryResult,
+    query_start: i64,
+    query_end: i64,
+    cardinality_level: CardinalityLevel,
+) -> StreamingAggsPartitionStrategy {
+    // Case 1: Fully cached - no execution needed
+    if discovery_result.is_fully_cached() {
+        return StreamingAggsPartitionStrategy::FullyCached {
+            cache_files: discovery_result.cached_ranges,
+        };
+    }
+
+    // Calculate the target interval for the entire query based on query duration
+    let query_target_interval =
+        generate_aggregation_search_interval(query_start, query_end, cardinality_level);
+
+    // Case 2: No cache available - use standard ladder logic
+    if discovery_result.has_no_cache() {
+        let partitions =
+            generate_uncached_partitions_from_range(query_start, query_end, query_target_interval);
+        return StreamingAggsPartitionStrategy::NoCacheAvailable { partitions };
+    }
+
+    // Case 3: Hybrid - mix of cached and uncached
+    let cached_partitions = group_cache_entries_into_partitions(discovery_result.cached_ranges);
+    let uncached_partitions =
+        generate_uncached_partitions(discovery_result.uncached_ranges, query_target_interval);
+
+    StreamingAggsPartitionStrategy::Hybrid {
+        cached_partitions,
+        uncached_partitions,
+    }
+}
+
+/// Groups consecutive cache entries with the same interval into cached partitions
+///
+/// # Preconditions
+/// - `cache_entries` must be sorted by `start_time` (ascending)
+/// - This is guaranteed by `discover_cache_for_query()` which sorts entries before returning
+///
+/// # Behavior
+/// - Creates separate partitions when interval changes
+/// - Creates separate partitions when there's a time gap between entries
+/// - Assumes entries are non-overlapping (guaranteed by cache write logic)
+fn group_cache_entries_into_partitions(cache_entries: Vec<CacheEntry>) -> Vec<CachedPartition> {
+    if cache_entries.is_empty() {
+        return vec![];
+    }
+
+    let mut partitions = Vec::new();
+    let mut current_group: Vec<CacheEntry> = vec![];
+    let mut current_interval = cache_entries[0].interval;
+
+    for entry in cache_entries {
+        // If the interval changes or there's a gap, create a new partition
+        if entry.interval != current_interval
+            || (!current_group.is_empty()
+                && entry.start_time > current_group.last().unwrap().end_time)
+        {
+            if !current_group.is_empty() {
+                let start_time = current_group.first().unwrap().start_time;
+                let end_time = current_group.last().unwrap().end_time;
+                partitions.push(CachedPartition::new(
+                    current_group,
+                    start_time,
+                    end_time,
+                    current_interval,
+                ));
+            }
+            current_group = vec![entry.clone()];
+            current_interval = entry.interval;
+        } else {
+            current_group.push(entry);
+        }
+    }
+
+    // Don't forget the last group
+    if !current_group.is_empty() {
+        let start_time = current_group.first().unwrap().start_time;
+        let end_time = current_group.last().unwrap().end_time;
+        partitions.push(CachedPartition::new(
+            current_group,
+            start_time,
+            end_time,
+            current_interval,
+        ));
+    }
+
+    partitions
+}
+
+/// Generates uncached partitions from time ranges using the query's target interval
+fn generate_uncached_partitions(
+    uncached_ranges: Vec<TimeRange>,
+    target_interval: Interval,
+) -> Vec<UncachedPartition> {
+    let mut partitions = Vec::new();
+
+    for range in uncached_ranges {
+        let range_partitions = generate_uncached_partitions_from_range(
+            range.start_time,
+            range.end_time,
+            target_interval,
+        );
+        partitions.extend(range_partitions);
+    }
+
+    partitions
+}
+
+/// Generates uncached partitions for a single time range using the specified target interval
+fn generate_uncached_partitions_from_range(
+    start_time: i64,
+    end_time: i64,
+    target_interval: Interval,
+) -> Vec<UncachedPartition> {
+    // Use the query's target interval (already calculated based on total query duration)
+    // This ensures all uncached partitions use the same interval regardless of gap size
+
+    // If interval is Zero, we don't cache (e.g., for Huge cardinality)
+    if target_interval == Interval::Zero {
+        return vec![UncachedPartition::new(start_time, end_time)];
+    }
+
+    let interval_micros = target_interval.get_interval_microseconds();
+    let mut partitions = Vec::new();
+
+    // Align start time to UTC boundary
+    let aligned_start = align_time_to_interval(start_time, interval_micros, true);
+
+    // If query starts before the first aligned boundary, create a non-aligned partition
+    if start_time < aligned_start {
+        partitions.push(UncachedPartition::new(
+            start_time,
+            aligned_start.min(end_time),
+        ));
+    }
+
+    // Generate UTC-aligned partitions
+    let mut current_time = aligned_start;
+    while current_time < end_time {
+        let partition_end = (current_time + interval_micros).min(end_time);
+
+        partitions.push(UncachedPartition::new(current_time, partition_end));
+
+        current_time = partition_end;
+    }
+
+    partitions
+}
+
+/// Aligns a timestamp to the nearest interval boundary
+///
+/// # Arguments
+/// * `timestamp` - Timestamp in microseconds (assumed to be within reasonable bounds: year
+///   1970-2200)
+/// * `interval_micros` - Interval duration in microseconds (max: 1 day = 86,400,000,000)
+/// * `round_up` - If true, rounds up to next boundary; if false, rounds down
+///
+/// # Safety
+/// This function uses unchecked arithmetic. Integer overflow is not possible with realistic
+/// timestamp values (years 1970-2200) and supported intervals (5min to 1day). The maximum
+/// result is bounded by `timestamp + interval_micros`, which is well within i64::MAX for
+/// any reasonable query timestamp.
+fn align_time_to_interval(timestamp: i64, interval_micros: i64, round_up: bool) -> i64 {
+    let remainder = timestamp % interval_micros;
+    if remainder == 0 {
+        timestamp
+    } else if round_up {
+        timestamp + (interval_micros - remainder)
+    } else {
+        timestamp - remainder
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_partition_strategy_fully_cached() {
+        let cache_files = vec![CacheEntry {
+            file_path: "test.arrow".to_string(),
+            start_time: 0,
+            end_time: 1000,
+            interval: Interval::OneHour,
+        }];
+
+        let strategy = StreamingAggsPartitionStrategy::FullyCached {
+            cache_files: cache_files.clone(),
+        };
+
+        assert!(!strategy.requires_execution());
+        assert_eq!(strategy.execution_partition_count(), 0);
+    }
+
+    #[test]
+    fn test_partition_strategy_no_cache() {
+        let partitions = vec![UncachedPartition::new(0, 1000)];
+
+        let strategy = StreamingAggsPartitionStrategy::NoCacheAvailable {
+            partitions: partitions.clone(),
+        };
+
+        assert!(strategy.requires_execution());
+        assert_eq!(strategy.execution_partition_count(), 1);
+    }
+
+    #[test]
+    fn test_partition_strategy_hybrid() {
+        let cached = vec![CachedPartition::new(vec![], 0, 500, Interval::OneHour)];
+        let uncached = vec![UncachedPartition::new(500, 1000)];
+
+        let strategy = StreamingAggsPartitionStrategy::Hybrid {
+            cached_partitions: cached.clone(),
+            uncached_partitions: uncached.clone(),
+        };
+
+        assert!(strategy.requires_execution());
+        assert_eq!(strategy.execution_partition_count(), 1);
+    }
+
+    #[test]
+    fn test_align_time_to_interval() {
+        let one_hour_micros = 3_600_000_000i64;
+
+        // Already aligned
+        assert_eq!(align_time_to_interval(0, one_hour_micros, true), 0);
+        assert_eq!(align_time_to_interval(0, one_hour_micros, false), 0);
+        assert_eq!(
+            align_time_to_interval(one_hour_micros, one_hour_micros, true),
+            one_hour_micros
+        );
+
+        // Not aligned - round up
+        assert_eq!(
+            align_time_to_interval(100, one_hour_micros, true),
+            one_hour_micros
+        );
+        assert_eq!(
+            align_time_to_interval(one_hour_micros + 100, one_hour_micros, true),
+            one_hour_micros * 2
+        );
+
+        // Not aligned - round down
+        assert_eq!(align_time_to_interval(100, one_hour_micros, false), 0);
+        assert_eq!(
+            align_time_to_interval(one_hour_micros + 100, one_hour_micros, false),
+            one_hour_micros
+        );
+    }
+
+    #[test]
+    fn test_generate_uncached_partitions_from_range_zero_interval() {
+        // For Zero interval, we don't partition
+        let partitions = generate_uncached_partitions_from_range(0, 10000, Interval::Zero);
+
+        assert_eq!(partitions.len(), 1);
+        assert_eq!(partitions[0].start_time, 0);
+        assert_eq!(partitions[0].end_time, 10000);
+    }
+
+    #[test]
+    fn test_generate_uncached_partitions_from_range_aligned() {
+        let five_min_micros = 300_000_000i64;
+
+        // Query range that's perfectly aligned: 0 to 15 minutes (3 x 5-minute intervals)
+        let partitions =
+            generate_uncached_partitions_from_range(0, five_min_micros * 3, Interval::FiveMinutes);
+
+        assert_eq!(partitions.len(), 3);
+
+        for (i, partition) in partitions.iter().enumerate() {
+            assert_eq!(partition.start_time, five_min_micros * i as i64);
+            assert_eq!(partition.end_time, five_min_micros * (i as i64 + 1));
+        }
+    }
+
+    #[test]
+    fn test_generate_uncached_partitions_from_range_unaligned() {
+        let five_min_micros = 300_000_000i64;
+
+        // Query range that starts at a non-aligned time
+        let start = 100_000; // 100ms offset
+        let end = five_min_micros * 2 + 100_000; // 10min + 100ms
+
+        let partitions = generate_uncached_partitions_from_range(start, end, Interval::FiveMinutes);
+
+        // Should create: [100ms -> 5min], [5min -> 10min], [10min -> 10min+100ms]
+        assert_eq!(partitions.len(), 3);
+
+        // First partition: non-aligned start
+        assert_eq!(partitions[0].start_time, start);
+        assert_eq!(partitions[0].end_time, five_min_micros);
+
+        // Middle partition: fully aligned
+        assert_eq!(partitions[1].start_time, five_min_micros);
+        assert_eq!(partitions[1].end_time, five_min_micros * 2);
+
+        // Last partition: non-aligned end
+        assert_eq!(partitions[2].start_time, five_min_micros * 2);
+        assert_eq!(partitions[2].end_time, end);
+    }
+
+    #[test]
+    fn test_group_cache_entries_same_interval() {
+        let entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 0,
+                end_time: 1000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 1000,
+                end_time: 2000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let partitions = group_cache_entries_into_partitions(entries);
+
+        assert_eq!(partitions.len(), 1);
+        assert_eq!(partitions[0].start_time, 0);
+        assert_eq!(partitions[0].end_time, 2000);
+        assert_eq!(partitions[0].cache_files.len(), 2);
+        assert_eq!(partitions[0].interval, Interval::OneHour);
+    }
+
+    #[test]
+    fn test_group_cache_entries_different_intervals() {
+        let entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 0,
+                end_time: 1000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 1000,
+                end_time: 2000,
+                interval: Interval::FiveMinutes,
+            },
+        ];
+
+        let partitions = group_cache_entries_into_partitions(entries);
+
+        assert_eq!(partitions.len(), 2);
+        assert_eq!(partitions[0].interval, Interval::OneHour);
+        assert_eq!(partitions[1].interval, Interval::FiveMinutes);
+    }
+
+    #[test]
+    fn test_group_cache_entries_with_gap() {
+        let entries = vec![
+            CacheEntry {
+                file_path: "test1.arrow".to_string(),
+                start_time: 0,
+                end_time: 1000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "test2.arrow".to_string(),
+                start_time: 2000, // Gap between 1000 and 2000
+                end_time: 3000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        let partitions = group_cache_entries_into_partitions(entries);
+
+        assert_eq!(partitions.len(), 2);
+        assert_eq!(partitions[0].cache_files.len(), 1);
+        assert_eq!(partitions[1].cache_files.len(), 1);
+    }
+
+    #[test]
+    fn test_generate_optimal_partitions_fully_cached() {
+        let discovery = CacheDiscoveryResult::new(
+            vec![CacheEntry {
+                file_path: "test.arrow".to_string(),
+                start_time: 0,
+                end_time: 10000,
+                interval: Interval::OneHour,
+            }],
+            vec![],
+            1.0,
+        );
+
+        let strategy = generate_optimal_partitions(discovery, 0, 10000, CardinalityLevel::Low);
+
+        match strategy {
+            StreamingAggsPartitionStrategy::FullyCached { cache_files } => {
+                assert_eq!(cache_files.len(), 1);
+            }
+            _ => panic!("Expected FullyCached strategy"),
+        }
+    }
+
+    #[test]
+    fn test_generate_optimal_partitions_no_cache() {
+        let discovery = CacheDiscoveryResult::new(vec![], vec![TimeRange::new(0, 10000)], 0.0);
+
+        let strategy = generate_optimal_partitions(discovery, 0, 10000, CardinalityLevel::Low);
+
+        match strategy {
+            StreamingAggsPartitionStrategy::NoCacheAvailable { partitions } => {
+                assert!(!partitions.is_empty());
+            }
+            _ => panic!("Expected NoCacheAvailable strategy"),
+        }
+    }
+
+    #[test]
+    fn test_generate_optimal_partitions_hybrid() {
+        let discovery = CacheDiscoveryResult::new(
+            vec![CacheEntry {
+                file_path: "test.arrow".to_string(),
+                start_time: 0,
+                end_time: 5000,
+                interval: Interval::OneHour,
+            }],
+            vec![TimeRange::new(5000, 10000)],
+            0.5,
+        );
+
+        let strategy = generate_optimal_partitions(discovery, 0, 10000, CardinalityLevel::Low);
+
+        match strategy {
+            StreamingAggsPartitionStrategy::Hybrid {
+                cached_partitions,
+                uncached_partitions,
+            } => {
+                assert_eq!(cached_partitions.len(), 1);
+                assert!(!uncached_partitions.is_empty());
+            }
+            _ => panic!("Expected Hybrid strategy"),
+        }
+    }
+}

src/search/src/datafusion/aggregates/merge_phase.rs

@@ -0,0 +1,272 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hash aggregation
+
+use std::sync::Arc;
+
+use arrow::{array::*, datatypes::SchemaRef};
+use datafusion::{
+    common::Result,
+    logical_expr::{EmitTo, GroupsAccumulator},
+    physical_expr::{GroupsAccumulatorAdapter, aggregate::AggregateFunctionExpr},
+    physical_plan::{
+        PhysicalExpr,
+        aggregates::{
+            AggregateExec, AggregateMode, PhysicalGroupBy, aggregate_expressions,
+            evaluate_group_by, evaluate_many,
+            group_values::{GroupValues, new_group_values},
+            order::GroupOrdering,
+        },
+    },
+};
+use log::debug;
+
+#[derive(Debug, Clone)]
+/// This object tracks the aggregation phase (input/output)
+pub(crate) enum ExecutionState {
+    ReadingInput,
+    /// When producing output, the remaining rows to output are stored
+    /// here and are sliced off as needed in batch_size chunks
+    ProducingOutput(RecordBatch),
+    /// All input has been consumed and all groups have been emitted
+    Done,
+}
+
+pub struct GroupedHashAggregateStream {
+    // ========================================================================
+    // PROPERTIES:
+    // These fields are initialized at the start and remain constant throughout
+    // the execution.
+    // ========================================================================
+    schema: SchemaRef,
+    mode: AggregateMode,
+
+    /// Arguments to pass to each accumulator.
+    ///
+    /// The arguments in `accumulator[i]` is passed `aggregate_arguments[i]`
+    ///
+    /// The argument to each accumulator is itself a `Vec` because
+    /// some aggregates such as `CORR` can accept more than one
+    /// argument.
+    aggregate_arguments: Vec<Vec<Arc<dyn PhysicalExpr>>>,
+
+    /// GROUP BY expressions
+    group_by: PhysicalGroupBy,
+
+    // ========================================================================
+    // STATE FLAGS:
+    // These fields will be updated during the execution. And control the flow of
+    // the execution.
+    // ========================================================================
+    /// Tracks if this stream is generating input or output
+    exec_state: ExecutionState,
+
+    /// Have we seen the end of the input
+    input_done: bool,
+
+    // ========================================================================
+    // STATE BUFFERS:
+    // These fields will accumulate intermediate results during the execution.
+    // ========================================================================
+    /// An interning store of group keys
+    group_values: Box<dyn GroupValues>,
+
+    /// scratch space for the current input [`RecordBatch`] being
+    /// processed. Reused across batches here to avoid reallocations
+    current_group_indices: Vec<usize>,
+
+    /// Accumulators, one for each `AggregateFunctionExpr` in the query
+    ///
+    /// For example, if the query has aggregates, `SUM(x)`,
+    /// `COUNT(y)`, there will be two accumulators, each one
+    /// specialized for that particular aggregate and its input types
+    accumulators: Vec<Box<dyn GroupsAccumulator>>,
+}
+
+impl GroupedHashAggregateStream {
+    /// Create a new GroupedHashAggregateStream
+    pub fn new(agg: &AggregateExec) -> Result<Self> {
+        debug!("Creating GroupedHashAggregateStream");
+        let agg_schema = agg.input().schema();
+        let agg_group_by = agg.group_expr().clone();
+        let aggregate_exprs = agg.aggr_expr();
+        let aggregate_arguments =
+            aggregate_expressions(agg.aggr_expr(), agg.mode(), agg_group_by.num_group_exprs())?;
+
+        // Instantiate the accumulators
+        let accumulators: Vec<_> = aggregate_exprs
+            .iter()
+            .map(create_group_accumulator)
+            .collect::<Result<_>>()?;
+
+        let group_schema = agg_group_by.group_schema(&agg.input().schema())?;
+
+        let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
+
+        let exec_state = ExecutionState::ReadingInput;
+
+        Ok(GroupedHashAggregateStream {
+            schema: agg_schema,
+            mode: *agg.mode(),
+            accumulators,
+            aggregate_arguments,
+            group_by: agg_group_by,
+            group_values,
+            current_group_indices: Default::default(),
+            exec_state,
+            input_done: false,
+        })
+    }
+
+    pub fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+}
+
+/// Create an accumulator for `agg_expr` -- a [`GroupsAccumulator`] if
+/// that is supported by the aggregate, or a
+/// [`GroupsAccumulatorAdapter`] if not.
+pub(crate) fn create_group_accumulator(
+    agg_expr: &Arc<AggregateFunctionExpr>,
+) -> Result<Box<dyn GroupsAccumulator>> {
+    if agg_expr.groups_accumulator_supported() {
+        agg_expr.create_groups_accumulator()
+    } else {
+        // Note in the log when the slow path is used
+        debug!(
+            "Creating GroupsAccumulatorAdapter for {}: {agg_expr:?}",
+            agg_expr.name()
+        );
+        let agg_expr_captured = Arc::clone(agg_expr);
+        let factory = move || agg_expr_captured.create_accumulator();
+        Ok(Box::new(GroupsAccumulatorAdapter::new(factory)))
+    }
+}
+
+impl GroupedHashAggregateStream {
+    /// Perform group-by aggregation for the given [`RecordBatch`].
+    pub fn group_aggregate_batch(&mut self, batch: RecordBatch) -> Result<()> {
+        // Evaluate the grouping expressions
+        let group_by_values = evaluate_group_by(&self.group_by, &batch)?;
+
+        // Evaluate the aggregation expressions.
+        let input_values = evaluate_many(&self.aggregate_arguments, &batch)?;
+
+        for group_values in &group_by_values {
+            // calculate the group indices for each input row
+            self.group_values
+                .intern(group_values, &mut self.current_group_indices)?;
+            let group_indices = &self.current_group_indices;
+
+            // Update ordering information if necessary
+            let total_num_groups = self.group_values.len();
+
+            // Gather the inputs to call the actual accumulator
+            let t = self.accumulators.iter_mut().zip(input_values.iter());
+
+            for (acc, values) in t {
+                // Call the appropriate method on each aggregator with
+                // the entire input row and the relevant group indexes
+                match self.mode {
+                    AggregateMode::Partial
+                    | AggregateMode::Single
+                    | AggregateMode::SinglePartitioned => {
+                        acc.update_batch(values, group_indices, None, total_num_groups)?;
+                    }
+                    _ => {
+                        // if aggregation is over intermediate states,
+                        // use merge
+                        acc.merge_batch(values, group_indices, None, total_num_groups)?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Create an output RecordBatch with the group keys and
+    /// accumulator states/values specified in emit_to
+    fn emit(&mut self, emit_to: EmitTo) -> Result<Option<RecordBatch>> {
+        let schema = self.schema();
+        if self.group_values.is_empty() {
+            return Ok(None);
+        }
+
+        let mut output = self.group_values.emit(emit_to)?;
+
+        // Next output each aggregate value
+        for acc in self.accumulators.iter_mut() {
+            output.extend(acc.state(emit_to)?);
+        }
+
+        let batch = RecordBatch::try_new(schema, output)?;
+        debug_assert!(batch.num_rows() > 0);
+        Ok(Some(batch))
+    }
+
+    /// Clear memory and shirk capacities to the size of the batch.
+    fn clear_shrink(&mut self, num_rows: usize) {
+        self.group_values.clear_shrink(num_rows);
+        self.current_group_indices.clear();
+        self.current_group_indices.shrink_to(num_rows);
+    }
+
+    /// Clear memory and shirk capacities to zero.
+    fn clear_all(&mut self) {
+        self.clear_shrink(0);
+    }
+
+    /// common function for signalling end of processing of the input stream
+    fn set_input_done_and_produce_output(&mut self) -> Result<()> {
+        self.input_done = true;
+        let batch = self.emit(EmitTo::All)?;
+        self.exec_state = batch.map_or(ExecutionState::Done, ExecutionState::ProducingOutput);
+        Ok(())
+    }
+
+    pub fn get_final_result(&mut self) -> Result<Vec<RecordBatch>> {
+        self.set_input_done_and_produce_output()?;
+        let batch = match &self.exec_state {
+            ExecutionState::ProducingOutput(batch) => batch.clone(),
+            _ => RecordBatch::new_empty(self.schema()),
+        };
+        self.exec_state = ExecutionState::Done;
+        self.clear_all();
+
+        // split the batch into multiple batches
+        let num_rows = batch.num_rows();
+        // early return for empty batches
+        if num_rows == 0 {
+            return Ok(vec![]);
+        }
+        // calculate optimal batch size and pre-allocate vector
+        let batch_size = 8192;
+        let full_batches = num_rows / batch_size;
+        let has_remaining = !num_rows.is_multiple_of(batch_size);
+        let total_batches = full_batches + if has_remaining { 1 } else { 0 };
+        let mut result_vec = Vec::with_capacity(total_batches);
+        for i in 0..full_batches {
+            result_vec.push(batch.slice(i * batch_size, batch_size));
+        }
+        if has_remaining {
+            let start_idx = full_batches * batch_size;
+            let remaining_rows = num_rows - start_idx;
+            result_vec.push(batch.slice(start_idx, remaining_rows));
+        }
+
+        Ok(result_vec)
+    }
+}

src/search/src/datafusion/aggregates/mod.rs

@@ -0,0 +1,101 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+pub mod merge_phase;
+pub mod no_grouping_merge_phase;
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, RecordBatch, RecordBatchOptions, StringViewBuilder};
+
+/// refer to: https://github.com/apache/datafusion/pull/11587
+/// Heuristically compact `StringViewArray`s to reduce memory usage, if needed
+///
+/// Decides when to consolidate the StringView into a new buffer to reduce
+/// memory usage and improve string locality for better performance.
+///
+/// This differs from `StringViewArray::gc` because:
+/// 1. It may not compact the array depending on a heuristic.
+/// 2. It uses a precise block size to reduce the number of buffers to track.
+///
+/// # Heuristic
+///
+/// If the average size of each view is larger than 32 bytes, we compact the array.
+///
+/// `StringViewArray` include pointers to buffer that hold the underlying data.
+/// One of the great benefits of `StringViewArray` is that many operations
+/// (e.g., `filter`) can be done without copying the underlying data.
+///
+/// However, after a while (e.g., after `FilterExec` or `HashJoinExec`) the
+/// `StringViewArray` may only refer to a small portion of the buffer,
+/// significantly increasing memory usage.
+pub(crate) fn gc_string_view_batch(batch: &RecordBatch) -> RecordBatch {
+    let new_columns: Vec<ArrayRef> = batch
+        .columns()
+        .iter()
+        .map(|c| {
+            // Try to re-create the `StringViewArray` to prevent holding the underlying buffer too
+            // long.
+            let Some(s) = c.as_string_view_opt() else {
+                return Arc::clone(c);
+            };
+
+            // Fast path: if the data buffers are empty, we can return the original array
+            if s.data_buffers().is_empty() {
+                return Arc::clone(c);
+            }
+
+            let ideal_buffer_size: usize = s
+                .views()
+                .iter()
+                .map(|v| {
+                    let len = (*v as u32) as usize;
+                    if len > 12 { len } else { 0 }
+                })
+                .sum();
+
+            // We don't use get_buffer_memory_size here, because gc is for the contents of the
+            // data buffers, not views and nulls.
+            let actual_buffer_size = s.data_buffers().iter().map(|b| b.capacity()).sum::<usize>();
+
+            // Re-creating the array copies data and can be time consuming.
+            // We only do it if the array is sparse
+            if actual_buffer_size > (ideal_buffer_size * 2) {
+                // We set the block size to `ideal_buffer_size` so that the new StringViewArray only
+                // has one buffer, which accelerate later concat_batches. See https://github.com/apache/arrow-rs/issues/6094 for more details.
+                let mut builder = StringViewBuilder::with_capacity(s.len());
+                if ideal_buffer_size > 0 {
+                    builder = builder.with_fixed_block_size(ideal_buffer_size as u32);
+                }
+
+                for v in s.iter() {
+                    builder.append_option(v);
+                }
+
+                let gc_string = builder.finish();
+
+                debug_assert!(gc_string.data_buffers().len() <= 1); // buffer count can be 0 if the `ideal_buffer_size` is 0
+
+                Arc::new(gc_string)
+            } else {
+                Arc::clone(c)
+            }
+        })
+        .collect();
+    let mut options = RecordBatchOptions::new();
+    options = options.with_row_count(Some(batch.num_rows()));
+    RecordBatch::try_new_with_options(batch.schema(), new_columns, &options)
+        .expect("Failed to re-create the gc'ed record batch")
+}

src/search/src/datafusion/aggregates/no_grouping_merge_phase.rs

@@ -0,0 +1,178 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Aggregate without grouping columns
+
+use std::{borrow::Cow, sync::Arc};
+
+use arrow::{array::ArrayRef, datatypes::SchemaRef, record_batch::RecordBatch};
+use datafusion::{
+    common::Result,
+    physical_plan::{
+        PhysicalExpr,
+        aggregates::{
+            AccumulatorItem, AggregateExec, AggregateMode, aggregate_expressions,
+            create_accumulators,
+        },
+        filter::batch_filter,
+    },
+};
+use itertools::Itertools;
+
+/// stream struct for aggregation without grouping columns
+pub struct AggregateStream {
+    schema: SchemaRef,
+    mode: AggregateMode,
+    aggregate_expressions: Vec<Vec<Arc<dyn PhysicalExpr>>>,
+    filter_expressions: Vec<Option<Arc<dyn PhysicalExpr>>>,
+    accumulators: Vec<AccumulatorItem>,
+}
+
+impl AggregateStream {
+    /// Create a new AggregateStream
+    pub fn new(agg: &AggregateExec) -> Result<Self> {
+        let agg_schema = agg.input().schema();
+        let agg_filter_expr = agg.filter_expr().to_vec();
+
+        let aggregate_expressions = aggregate_expressions(agg.aggr_expr(), agg.mode(), 0)?;
+        let filter_expressions = match *agg.mode() {
+            AggregateMode::Partial | AggregateMode::Single | AggregateMode::SinglePartitioned => {
+                agg_filter_expr
+            }
+            AggregateMode::Final
+            | AggregateMode::FinalPartitioned
+            | AggregateMode::PartialReduce => {
+                vec![None; agg.aggr_expr().len()]
+            }
+        };
+        let accumulators = create_accumulators(agg.aggr_expr())?;
+
+        Ok(AggregateStream {
+            schema: agg_schema,
+            mode: *agg.mode(),
+            aggregate_expressions,
+            filter_expressions,
+            accumulators,
+        })
+    }
+
+    pub fn aggregate_batch(&mut self, batch: RecordBatch) -> Result<()> {
+        let _ = aggregate_batch(
+            &self.mode,
+            batch,
+            &mut self.accumulators,
+            &self.aggregate_expressions,
+            &self.filter_expressions,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn finalize_aggregation(&mut self) -> Result<Vec<RecordBatch>> {
+        let result = finalize_aggregation(&mut self.accumulators)?;
+        let batch = RecordBatch::try_new(Arc::clone(&self.schema), result)?;
+
+        // split the batch into multiple batches
+        let num_rows = batch.num_rows();
+        // early return for empty batches
+        if num_rows == 0 {
+            return Ok(vec![]);
+        }
+        // calculate optimal batch size and pre-allocate vector
+        let batch_size = 8192;
+        let full_batches = num_rows / batch_size;
+        let has_remaining = !num_rows.is_multiple_of(batch_size);
+        let total_batches = full_batches + if has_remaining { 1 } else { 0 };
+        let mut result_vec = Vec::with_capacity(total_batches);
+        for i in 0..full_batches {
+            result_vec.push(batch.slice(i * batch_size, batch_size));
+        }
+        if has_remaining {
+            let start_idx = full_batches * batch_size;
+            let remaining_rows = num_rows - start_idx;
+            result_vec.push(batch.slice(start_idx, remaining_rows));
+        }
+
+        Ok(result_vec)
+    }
+}
+
+fn aggregate_batch(
+    mode: &AggregateMode,
+    batch: RecordBatch,
+    accumulators: &mut [AccumulatorItem],
+    expressions: &[Vec<Arc<dyn PhysicalExpr>>],
+    filters: &[Option<Arc<dyn PhysicalExpr>>],
+) -> Result<usize> {
+    let mut allocated = 0usize;
+
+    // 1.1 iterate accumulators and respective expressions together
+    // 1.2 filter the batch if necessary
+    // 1.3 evaluate expressions
+    // 1.4 update / merge accumulators with the expressions' values
+
+    // 1.1
+    accumulators
+        .iter_mut()
+        .zip(expressions)
+        .zip(filters)
+        .try_for_each(|((accum, expr), filter)| {
+            // 1.2
+            let batch = match filter {
+                Some(filter) => Cow::Owned(batch_filter(&batch, filter)?),
+                None => Cow::Borrowed(&batch),
+            };
+
+            // 1.3
+            let values = &expr
+                .iter()
+                .map(|e| {
+                    e.evaluate(&batch)
+                        .and_then(|v| v.into_array(batch.num_rows()))
+                })
+                .collect::<Result<Vec<_>>>()?;
+
+            // 1.4
+            let size_pre = accum.size();
+            let res = match mode {
+                AggregateMode::Partial
+                | AggregateMode::Single
+                | AggregateMode::SinglePartitioned => accum.update_batch(values),
+                AggregateMode::Final
+                | AggregateMode::FinalPartitioned
+                | AggregateMode::PartialReduce => accum.merge_batch(values),
+            };
+            let size_post = accum.size();
+            allocated += size_post.saturating_sub(size_pre);
+            res
+        })?;
+
+    Ok(allocated)
+}
+
+fn finalize_aggregation(accumulators: &mut [AccumulatorItem]) -> Result<Vec<ArrayRef>> {
+    // Build the vector of states
+    accumulators
+        .iter_mut()
+        .map(|accumulator| {
+            accumulator.state().and_then(|e| {
+                e.iter()
+                    .map(|v| v.to_array())
+                    .collect::<Result<Vec<ArrayRef>>>()
+            })
+        })
+        .flatten_ok()
+        .collect()
+}

src/search/src/datafusion/distributed_plan/aggregate_topk_exec/heap.rs

@@ -0,0 +1,794 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+use std::{
+    cmp::Ordering,
+    collections::{BinaryHeap, HashMap},
+    fmt::Debug,
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use arrow::{
+    array::{ArrayRef, RecordBatch},
+    compute::{SortOptions, interleave_record_batch},
+    datatypes::SchemaRef,
+    row::{RowConverter, SortField},
+};
+use datafusion::{
+    common::Result,
+    execution::{RecordBatchStream, SendableRecordBatchStream},
+};
+use futures::{Stream, StreamExt};
+
+pub struct TopKHeapStream {
+    schema: SchemaRef,
+    stream: SendableRecordBatchStream,
+    limit: usize,
+    topk_heap: BinaryHeap<HeapRow>,
+    record_batch_registry: RecordBatchRegistry,
+    sort_column_index: usize,
+    row_converter: RowConverter,
+}
+
+struct RecordBatchRegistry {
+    store: HashMap<u32, RecordBatchEntry>,
+    next_id: u32,
+}
+
+impl RecordBatchRegistry {
+    pub fn new() -> Self {
+        Self {
+            store: HashMap::new(),
+            next_id: 0,
+        }
+    }
+
+    pub fn register_entry(&mut self, rb: RecordBatch) -> RecordBatchEntry {
+        let id = self.next_id;
+        let record_batch_entry = RecordBatchEntry::new(id, rb);
+        self.next_id += 1;
+        record_batch_entry
+    }
+
+    pub fn submit_entry(&mut self, entry: RecordBatchEntry) {
+        if entry.uses > 0 {
+            self.store.insert(entry.id, entry);
+        }
+    }
+
+    pub fn remove_use_from_entry(&mut self, id: u32) {
+        if let Some(entry) = self.store.get_mut(&id) {
+            let Some(uses) = entry.uses.checked_sub(1) else {
+                panic!("underflow of uses for batch {id}");
+            };
+
+            if uses == 0 {
+                // remove the record batch from the registry
+                self.store.remove(&id).expect("cannot remove batch {id}");
+            }
+        } else {
+            panic!("entry does not exists batch {id}");
+        }
+    }
+}
+
+struct RecordBatchEntry {
+    id: u32,
+    record_batch: RecordBatch,
+    uses: usize,
+}
+
+impl RecordBatchEntry {
+    pub fn new(id: u32, record_batch: RecordBatch) -> Self {
+        Self {
+            id,
+            record_batch,
+            uses: 0,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct HeapRow {
+    sort_value: Vec<u8>,
+    row_id: usize,
+    batch_id: u32,
+}
+
+impl HeapRow {
+    fn with_new_row(mut self, new_row_bytes: &[u8], row_id: usize, batch_id: u32) -> Self {
+        self.sort_value.clear();
+        self.sort_value.extend_from_slice(new_row_bytes);
+        self.row_id = row_id;
+        self.batch_id = batch_id;
+        self
+    }
+}
+
+impl PartialEq for HeapRow {
+    fn eq(&self, other: &Self) -> bool {
+        self.sort_value == other.sort_value
+    }
+}
+
+impl Eq for HeapRow {}
+
+impl PartialOrd for HeapRow {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for HeapRow {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // For min-heap behavior to get top-K largest values
+        // We want smallest values at the top of the heap so we can pop them
+        // RowConverter produces lexicographically sortable byte arrays
+        self.sort_value.cmp(&other.sort_value)
+    }
+}
+
+impl TopKHeapStream {
+    pub fn new(
+        schema: SchemaRef,
+        stream: SendableRecordBatchStream,
+        sort_field: String,
+        descending: bool,
+        limit: usize,
+    ) -> Self {
+        // Find the index of the sort column
+        // also handle cases where the sort fields are not alias and can be names as count(*)[count]
+        let sort_column_index = schema
+            .fields()
+            .iter()
+            .position(|f| {
+                f.name() == &sort_field || f.name().split('[').next().unwrap_or("") == sort_field
+            })
+            .expect("Sort field not found in schema");
+
+        // Create RowConverter for the sort column with proper sort options
+        let sort_field_ref = &schema.fields()[sort_column_index];
+        let sort_options = if descending {
+            SortOptions::default().desc()
+        } else {
+            SortOptions::default().asc()
+        };
+        let sort_field =
+            SortField::new_with_options(sort_field_ref.data_type().clone(), sort_options);
+        let row_converter =
+            RowConverter::new(vec![sort_field]).expect("Failed to create RowConverter");
+
+        Self {
+            schema,
+            stream,
+            limit,
+            record_batch_registry: RecordBatchRegistry::new(),
+            topk_heap: BinaryHeap::new(),
+            sort_column_index,
+            row_converter,
+        }
+    }
+
+    fn convert_sort_column(&mut self, array: &ArrayRef) -> arrow::row::Rows {
+        // Direct conversion - simpler and avoids persistent memory
+        self.row_converter
+            .convert_columns(std::slice::from_ref(array))
+            .expect("Failed to convert column")
+    }
+
+    fn process_batch(&mut self, batch: RecordBatch) {
+        if batch.num_rows() == 0 {
+            return;
+        }
+
+        let sort_column = batch.column(self.sort_column_index);
+        let mut entry = self.record_batch_registry.register_entry(batch.clone());
+
+        // Convert all sort values at once - gets cleaned up automatically
+        let converted_rows = self.convert_sort_column(sort_column);
+        for row_index in 0..batch.num_rows() {
+            // Get row from converted batch
+            let row_ref = converted_rows.row(row_index);
+
+            if self.topk_heap.len() < self.limit {
+                // Heap not full - create new row
+                let new_row = HeapRow {
+                    sort_value: row_ref.as_ref().to_vec(),
+                    row_id: row_index,
+                    batch_id: entry.id,
+                };
+                entry.uses += 1;
+                self.topk_heap.push(new_row);
+            } else if let Some(heap_top) = self.topk_heap.peek() {
+                let should_replace =
+                    row_ref.as_ref().cmp(heap_top.sort_value.as_slice()) == Ordering::Less;
+                if should_replace {
+                    let popped_row = self.topk_heap.pop().unwrap();
+                    // Update batch tracking
+                    if popped_row.batch_id.ne(&entry.id) {
+                        entry.uses += 1;
+                        self.record_batch_registry
+                            .remove_use_from_entry(popped_row.batch_id);
+                    }
+                    // Reuse the Vec<u8> memory - this is the key optimization
+                    let reused_row = popped_row.with_new_row(row_ref.as_ref(), row_index, entry.id);
+                    self.topk_heap.push(reused_row);
+                }
+            }
+        }
+
+        self.record_batch_registry.submit_entry(entry);
+    }
+
+    fn heap_to_record_batch(&mut self) -> Option<RecordBatch> {
+        if self.topk_heap.is_empty() {
+            return None;
+        }
+
+        // Convert heap to sorted vec
+        // Since the heap is already having elements which are bit flipped in row converter
+        // we do not need to resort the final results outside.
+        let sorted_rows = std::mem::take(&mut self.topk_heap).into_sorted_vec();
+
+        let mut record_batches = Vec::new();
+        let mut batch_id_array_pos = HashMap::new();
+        for (batch_pos, (batch_id, batch)) in self.record_batch_registry.store.iter().enumerate() {
+            record_batches.push(&batch.record_batch);
+            batch_id_array_pos.insert(*batch_id, batch_pos);
+        }
+
+        let indices: Vec<_> = sorted_rows
+            .iter()
+            .map(|row| (batch_id_array_pos[&row.batch_id], row.row_id))
+            .collect();
+
+        let final_batch = interleave_record_batch(&record_batches, &indices)
+            .map_err(|_| log::error!("Failed to interleave_record_batch"))
+            .ok()?;
+        Some(final_batch)
+    }
+}
+
+impl Stream for TopKHeapStream {
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        match self.stream.poll_next_unpin(cx) {
+            Poll::Ready(Some(Ok(batch))) => {
+                // Process the batch incrementally with heap
+                self.process_batch(batch);
+
+                // Return empty batch to indicate progress
+                let schema = self.schema.clone();
+                let empty_batch = RecordBatch::new_empty(schema);
+                Poll::Ready(Some(Ok(empty_batch)))
+            }
+            Poll::Ready(None) => {
+                // Stream is finished, return final top-K result
+                let topk_batch = self.heap_to_record_batch();
+                Poll::Ready(topk_batch.map(Ok))
+            }
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(Some(Err(e))) => {
+                log::error!("Error in CacheTopkStream: {e}");
+                Poll::Ready(None)
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.stream.size_hint()
+    }
+}
+
+impl RecordBatchStream for TopKHeapStream {
+    /// Get the schema
+    fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow::{
+        array::{Array, Int64Array, StringArray},
+        datatypes::{Field, Schema},
+        util::pretty::pretty_format_batches,
+    };
+
+    use super::*;
+
+    struct TestRecordBatchStream {
+        schema: SchemaRef,
+        batches: Vec<RecordBatch>,
+        index: usize,
+    }
+
+    impl TestRecordBatchStream {
+        fn new(schema: SchemaRef, batches: Vec<RecordBatch>) -> Self {
+            Self {
+                schema,
+                batches,
+                index: 0,
+            }
+        }
+    }
+
+    impl Stream for TestRecordBatchStream {
+        type Item = Result<RecordBatch>;
+
+        fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+            if self.index < self.batches.len() {
+                let batch = self.batches[self.index].clone();
+                self.index += 1;
+                Poll::Ready(Some(Ok(batch)))
+            } else {
+                Poll::Ready(None)
+            }
+        }
+    }
+
+    impl RecordBatchStream for TestRecordBatchStream {
+        fn schema(&self) -> SchemaRef {
+            self.schema.clone()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_cache_topk_stream_descending() {
+        // Create schema with name and count columns
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("name", arrow::datatypes::DataType::Utf8, false),
+            Field::new("count", arrow::datatypes::DataType::Int64, false),
+        ]));
+
+        // Create test data with multiple batches
+        let batch1 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![
+                    "item1", "item7", "item8", "item6", "item2", "item3",
+                ])),
+                Arc::new(Int64Array::from(vec![10, 12, 13, 24, 25, 15])),
+            ],
+        )
+        .unwrap();
+
+        let batch2 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["item4", "item5"])),
+                Arc::new(Int64Array::from(vec![5, 30])),
+            ],
+        )
+        .unwrap();
+
+        let test_stream = TestRecordBatchStream::new(schema.clone(), vec![batch1, batch2]);
+        let stream: SendableRecordBatchStream = Box::pin(test_stream);
+
+        // Create CacheTopkStream for top-3 descending by count
+        let mut topk_stream = TopKHeapStream::new(
+            schema.clone(),
+            stream,
+            "count".to_string(),
+            true, // descending
+            5,    // limit
+        );
+
+        let mut results = Vec::new();
+        while let Some(result) = topk_stream.next().await {
+            match result {
+                Ok(batch) => {
+                    if batch.num_rows() > 0 {
+                        results.push(batch);
+                    }
+                }
+                Err(e) => panic!("Stream error: {e}"),
+            }
+        }
+
+        println!("{}", pretty_format_batches(&results).unwrap());
+
+        // Should have one final result batch with top-3 items
+        assert_eq!(results.len(), 1);
+        let final_batch = &results[0];
+        assert_eq!(final_batch.num_rows(), 5);
+        // println!("{}", pretty_format_batches(&[final_batch.clone()]).unwrap());
+        // Verify the results are in descending order: item5(30), item2(25), item3(15)
+        let names = final_batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let counts = final_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+
+        assert_eq!(names.value(0), "item5");
+        assert_eq!(counts.value(0), 30);
+        assert_eq!(names.value(1), "item2");
+        assert_eq!(counts.value(1), 25);
+        assert_eq!(names.value(2), "item6");
+        assert_eq!(counts.value(2), 24);
+    }
+
+    #[tokio::test]
+    async fn test_cache_topk_stream_ascending() {
+        // Create schema
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("name", arrow::datatypes::DataType::Utf8, false),
+            Field::new("value", arrow::datatypes::DataType::Int64, false),
+        ]));
+
+        // Create test data
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])),
+                Arc::new(Int64Array::from(vec![50, 20, 80, 10, 30])),
+            ],
+        )
+        .unwrap();
+
+        let test_stream = TestRecordBatchStream::new(schema.clone(), vec![batch]);
+        let stream: SendableRecordBatchStream = Box::pin(test_stream);
+
+        // Create CacheTopkStream for top-3 ascending by value
+        let mut topk_stream = TopKHeapStream::new(
+            schema.clone(),
+            stream,
+            "value".to_string(),
+            false, // ascending
+            5,     // limit
+        );
+
+        let mut results = Vec::new();
+        while let Some(result) = topk_stream.next().await {
+            match result {
+                Ok(batch) => {
+                    if batch.num_rows() > 0 {
+                        results.push(batch);
+                    }
+                }
+                Err(e) => panic!("Stream error: {e}"),
+            }
+        }
+
+        println!("{}", pretty_format_batches(&results).unwrap());
+        // Should have one final result batch with top-3 smallest items
+        assert_eq!(results.len(), 1);
+        let final_batch = &results[0];
+        assert_eq!(final_batch.num_rows(), 5);
+        // println!("{}", pretty_format_batches(&[final_batch.clone()]).unwrap());
+
+        // Verify the results are in ascending order: d(10), b(20), e(30)
+        let names = final_batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let values = final_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+
+        assert_eq!(names.value(0), "d");
+        assert_eq!(values.value(0), 10);
+        assert_eq!(names.value(1), "b");
+        assert_eq!(values.value(1), 20);
+        assert_eq!(names.value(2), "e");
+        assert_eq!(values.value(2), 30);
+    }
+
+    #[tokio::test]
+    async fn test_cache_topk_stream_limit() {
+        // Create schema
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", arrow::datatypes::DataType::Utf8, false),
+            Field::new("score", arrow::datatypes::DataType::Int64, false),
+        ]));
+
+        // Create test data with more items than limit
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![
+                    "id1", "id2", "id3", "id4", "id5", "id6",
+                ])),
+                Arc::new(Int64Array::from(vec![100, 200, 50, 300, 150, 75])),
+            ],
+        )
+        .unwrap();
+
+        let test_stream = TestRecordBatchStream::new(schema.clone(), vec![batch]);
+        let stream: SendableRecordBatchStream = Box::pin(test_stream);
+
+        // Create CacheTopkStream for top-2 descending by score
+        let mut topk_stream = TopKHeapStream::new(
+            schema.clone(),
+            stream,
+            "score".to_string(),
+            true, // descending
+            2,    // limit to 2
+        );
+
+        let mut results = Vec::new();
+        while let Some(result) = topk_stream.next().await {
+            match result {
+                Ok(batch) => {
+                    if batch.num_rows() > 0 {
+                        results.push(batch);
+                    }
+                }
+                Err(e) => panic!("Stream error: {e}"),
+            }
+        }
+
+        println!("{}", pretty_format_batches(&results).unwrap());
+        // println!("{}", pretty_format_batches(&results).unwrap());
+        // Should have one final result batch with top-2 items only
+        assert_eq!(results.len(), 1);
+        let final_batch = &results[0];
+        // the final batch rows can never be less than the limit when
+        // enough data is present
+        assert!(final_batch.num_rows() >= 2);
+
+        // Verify the results are the top-2: id4(300), id2(200)
+        let ids = final_batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let scores = final_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+
+        assert_eq!(ids.value(0), "id4");
+        assert_eq!(scores.value(0), 300);
+        assert_eq!(ids.value(1), "id2");
+        assert_eq!(scores.value(1), 200);
+    }
+
+    #[tokio::test]
+    async fn test_complex_schema_final_row_construction() {
+        use arrow::array::{BooleanArray, Float32Array, Int32Array, TimestampMillisecondArray};
+
+        // Test with a very complex schema including different data types
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("user_name", arrow::datatypes::DataType::Utf8, false),
+            Field::new("is_premium", arrow::datatypes::DataType::Boolean, false),
+            Field::new("score", arrow::datatypes::DataType::Float32, false),
+            Field::new("rank", arrow::datatypes::DataType::Int64, false),
+            Field::new("session_id", arrow::datatypes::DataType::Int32, false),
+            Field::new(
+                "timestamp",
+                arrow::datatypes::DataType::Timestamp(
+                    arrow::datatypes::TimeUnit::Millisecond,
+                    None,
+                ),
+                false,
+            ),
+            Field::new("region", arrow::datatypes::DataType::Utf8, true),
+        ]));
+
+        // Create batches with mixed data types
+        let batch1 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["alice", "bob", "carol"])),
+                Arc::new(BooleanArray::from(vec![true, false, true])),
+                Arc::new(Float32Array::from(vec![75.5, 67.2, 82.1])),
+                Arc::new(Int64Array::from(vec![10, 20, 15])),
+                Arc::new(Int32Array::from(vec![1001, 1002, 1003])),
+                Arc::new(TimestampMillisecondArray::from(vec![
+                    1000000, 2000000, 1500000,
+                ])),
+                Arc::new(StringArray::from(vec![Some("US"), Some("EU"), None])),
+            ],
+        )
+        .unwrap();
+
+        let batch2 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["diana", "eve"])),
+                Arc::new(BooleanArray::from(vec![true, false])),
+                Arc::new(Float32Array::from(vec![95.7, 88.3])),
+                Arc::new(Int64Array::from(vec![5, 8])),
+                Arc::new(Int32Array::from(vec![1004, 1005])),
+                Arc::new(TimestampMillisecondArray::from(vec![3000000, 2500000])),
+                Arc::new(StringArray::from(vec![Some("APAC"), Some("US")])),
+            ],
+        )
+        .unwrap();
+
+        let test_stream = TestRecordBatchStream::new(schema.clone(), vec![batch1, batch2]);
+        let stream: SendableRecordBatchStream = Box::pin(test_stream);
+
+        let mut topk_stream = TopKHeapStream::new(
+            schema.clone(),
+            stream,
+            "score".to_string(),
+            true, // descending
+            5,    // all 5 rows
+        );
+
+        let mut results = Vec::new();
+        while let Some(result) = topk_stream.next().await {
+            match result {
+                Ok(batch) => {
+                    if batch.num_rows() > 0 {
+                        results.push(batch);
+                    }
+                }
+                Err(e) => panic!("Stream error: {e}"),
+            }
+        }
+
+        assert_eq!(results.len(), 1);
+        let final_batch = &results[0];
+        assert_eq!(final_batch.num_rows(), 5);
+        assert_eq!(final_batch.num_columns(), 7);
+
+        // Verify all data types are preserved correctly
+        // Expected order: diana(95.7), eve(88.3), carol(82.1), alice(75.5), bob(67.2)
+        let user_names = final_batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let is_premium = final_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .unwrap();
+        let scores = final_batch
+            .column(2)
+            .as_any()
+            .downcast_ref::<Float32Array>()
+            .unwrap();
+        let ranks = final_batch
+            .column(3)
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        let session_ids = final_batch
+            .column(4)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        let timestamps = final_batch
+            .column(5)
+            .as_any()
+            .downcast_ref::<TimestampMillisecondArray>()
+            .unwrap();
+        let regions = final_batch
+            .column(6)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+
+        // First row: diana (highest score)
+        assert_eq!(user_names.value(0), "diana");
+        assert!(is_premium.value(0));
+        assert!((scores.value(0) - 95.7).abs() < f32::EPSILON);
+        assert_eq!(ranks.value(0), 5);
+        assert_eq!(session_ids.value(0), 1004);
+        assert_eq!(timestamps.value(0), 3000000);
+        assert_eq!(regions.value(0), "APAC");
+
+        // Second row: eve
+        assert_eq!(user_names.value(1), "eve");
+        assert!(!is_premium.value(1));
+        assert!((scores.value(1) - 88.3).abs() < f32::EPSILON);
+        assert_eq!(ranks.value(1), 8);
+        assert_eq!(session_ids.value(1), 1005);
+        assert_eq!(timestamps.value(1), 2500000);
+        assert_eq!(regions.value(1), "US");
+
+        // Third row: carol
+        assert_eq!(user_names.value(2), "carol");
+        assert!(is_premium.value(2));
+        assert!((scores.value(2) - 82.1).abs() < f32::EPSILON);
+        assert_eq!(ranks.value(2), 15);
+        assert_eq!(session_ids.value(2), 1003);
+        assert_eq!(timestamps.value(2), 1500000);
+        assert!(regions.is_null(2));
+
+        println!("{}", pretty_format_batches(&results).unwrap());
+    }
+
+    #[tokio::test]
+    async fn test_many_batches_force_eviction() {
+        // Test with many batches where we have more batches than the limit
+        // This forces the registry to manage entries across different batches
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("batch_name", arrow::datatypes::DataType::Utf8, false),
+            Field::new("value", arrow::datatypes::DataType::Int64, false),
+        ]));
+
+        // Create 6 batches, each with 1 row, but limit to only 3 results
+        let mut batches = Vec::new();
+        let values = [10, 50, 20, 80, 30, 90]; // 90, 80, 50 should be top 3
+        //
+        for (i, &value) in values.iter().enumerate() {
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(StringArray::from(vec![format!("batch_{}", i)])),
+                    Arc::new(Int64Array::from(vec![value])),
+                ],
+            )
+            .unwrap();
+            batches.push(batch);
+        }
+
+        let test_stream = TestRecordBatchStream::new(schema.clone(), batches);
+        let stream: SendableRecordBatchStream = Box::pin(test_stream);
+
+        let mut topk_stream = TopKHeapStream::new(
+            schema.clone(),
+            stream,
+            "value".to_string(),
+            true, // descending
+            3,    // limit to 3, but we have 6 batches
+        );
+
+        let mut results = Vec::new();
+        while let Some(result) = topk_stream.next().await {
+            match result {
+                Ok(batch) => {
+                    if batch.num_rows() > 0 {
+                        results.push(batch);
+                    }
+                }
+                Err(e) => panic!("Stream error: {e}"),
+            }
+        }
+
+        assert_eq!(results.len(), 1);
+        let final_batch = &results[0];
+        assert_eq!(final_batch.num_rows(), 3);
+
+        // Should be: batch_5(90), batch_3(80), batch_1(50)
+        let names = final_batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let values = final_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+
+        assert_eq!(names.value(0), "batch_5");
+        assert_eq!(values.value(0), 90);
+        assert_eq!(names.value(1), "batch_3");
+        assert_eq!(values.value(1), 80);
+        assert_eq!(names.value(2), "batch_1");
+        assert_eq!(values.value(2), 50);
+
+        println!("{}", pretty_format_batches(&results).unwrap());
+    }
+}

src/search/src/datafusion/distributed_plan/aggregate_topk_exec/mod.rs

@@ -0,0 +1,203 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt::Debug, sync::Arc};
+
+use arrow::datatypes::SchemaRef;
+use datafusion::{
+    common::Result,
+    execution::{SendableRecordBatchStream, TaskContext},
+    physical_expr::EquivalenceProperties,
+    physical_plan::{
+        DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
+        PlanProperties,
+        execution_plan::{Boundedness, EmissionType},
+    },
+};
+
+pub mod heap;
+pub mod sort;
+
+#[derive(Debug)]
+pub struct AggregateTopkExec {
+    input: Arc<dyn ExecutionPlan>,
+    /// Cache holding plan properties like equivalences, output partitioning etc.
+    cache: Arc<PlanProperties>,
+    target_partitions: usize,
+    sort_field: String,
+    descending: bool,
+    limit: u64,
+}
+
+impl AggregateTopkExec {
+    /// Create a new AggregateMergeExec with explicit cache strategy
+    pub fn new(
+        input: Arc<dyn ExecutionPlan>,
+        sort_field: &str,
+        descending: bool,
+        limit: u64,
+    ) -> Self {
+        // Partial or no cache: cached partitions + input partitions
+        let target_partitions = input.output_partitioning().partition_count();
+        let cache = Self::compute_properties(Arc::clone(&input.schema()), target_partitions);
+        let sort_field = input
+            .schema()
+            .fields()
+            .iter()
+            .find(|f| {
+                // field name like count(*)[count]
+                f.name() == sort_field
+                    || f.name().split('[').next().is_some_and(|v| v == sort_field)
+            })
+            .unwrap()
+            .name()
+            .to_string();
+
+        Self {
+            input,
+            cache,
+            target_partitions,
+            sort_field,
+            descending,
+            limit,
+        }
+    }
+
+    fn output_partitioning_helper(n_partitions: usize) -> Partitioning {
+        Partitioning::UnknownPartitioning(n_partitions)
+    }
+
+    /// This function creates the cache object that stores the plan properties such as schema,
+    /// equivalence properties, ordering, partitioning, etc.
+    fn compute_properties(schema: SchemaRef, n_partitions: usize) -> Arc<PlanProperties> {
+        let eq_properties = EquivalenceProperties::new(schema);
+        let output_partitioning = Self::output_partitioning_helper(n_partitions);
+        Arc::new(PlanProperties::new(
+            eq_properties,
+            // Output Partitioning
+            output_partitioning,
+            // Execution Mode
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ))
+    }
+
+    pub fn sort_field(&self) -> &str {
+        &self.sort_field
+    }
+
+    pub fn descending(&self) -> bool {
+        self.descending
+    }
+
+    pub fn limit(&self) -> u64 {
+        self.limit
+    }
+}
+
+impl DisplayAs for AggregateTopkExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                write!(
+                    f,
+                    "AggregateTopkExec: target_partitions={}, limit={}, descending={}",
+                    self.target_partitions, self.limit, self.descending
+                )
+            }
+            DisplayFormatType::TreeRender => {
+                _ = writeln!(f, "target_partitions={}", self.target_partitions);
+                _ = writeln!(f, "limit={}", self.limit);
+                _ = writeln!(f, "descending={}", self.descending);
+                Ok(())
+            }
+        }
+    }
+}
+
+impl ExecutionPlan for AggregateTopkExec {
+    fn name(&self) -> &'static str {
+        "AggregateTopkExec"
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.cache
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if children.is_empty() {
+            return Ok(self);
+        }
+        Ok(Arc::new(Self::new(
+            children[0].clone(),
+            &self.sort_field,
+            self.descending,
+            self.limit,
+        )))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        let cfg = config::get_config();
+
+        // We need to dynamically choose operator to use based on K (limit) because
+        // heap is more memory effecient and performant when the K <= 200 range, but as the range
+        // increases the performance takes a hit. In such cases, giving up on memory and
+        // prioritizing performance make more sense.
+        let can_use_top_k_heap =
+            cfg.common.use_agg_topk_heap && self.limit <= cfg.common.agg_topk_heap_max_limit;
+
+        let pinned_stream: SendableRecordBatchStream = if can_use_top_k_heap {
+            // we use inflated limit here to calculate topK values on partial aggregation results
+            // such that we mimize the risk of losing counts. Having a large limit ensures we take
+            // more keys into consideration when sending out the final record batch to leader.
+            let inflated_limit = (self.limit * 4).max(1000) as usize;
+            Box::pin(heap::TopKHeapStream::new(
+                self.input.schema(),
+                self.input.execute(partition, Arc::clone(&context))?,
+                self.sort_field.clone(),
+                self.descending,
+                inflated_limit,
+            ))
+        } else {
+            Box::pin(sort::TopKSortStream::new(
+                self.input.schema(),
+                self.input.execute(partition, context)?,
+                self.sort_field.clone(),
+                self.descending,
+                self.limit,
+            ))
+        };
+        Ok(pinned_stream)
+    }
+
+    fn benefits_from_input_partitioning(&self) -> Vec<bool> {
+        vec![false; self.children().len()]
+    }
+
+    fn supports_limit_pushdown(&self) -> bool {
+        true
+    }
+}

src/search/src/datafusion/distributed_plan/aggregate_topk_exec/sort.rs

@@ -0,0 +1,148 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use arrow::{array::RecordBatch, compute::concat_batches, datatypes::SchemaRef};
+use config::utils::record_batch_ext::sort_record_batch_by_column;
+use datafusion::{
+    common::Result,
+    execution::{RecordBatchStream, SendableRecordBatchStream},
+};
+use futures::{Stream, StreamExt};
+
+pub struct TopKSortStream {
+    schema: SchemaRef,
+    stream: SendableRecordBatchStream,
+    sort_field: String,
+    descending: bool,
+    limit: u64,
+    cache_buf: Vec<RecordBatch>,
+}
+
+impl TopKSortStream {
+    pub fn new(
+        schema: SchemaRef,
+        stream: SendableRecordBatchStream,
+        sort_field: String,
+        descending: bool,
+        limit: u64,
+    ) -> Self {
+        Self {
+            schema,
+            stream,
+            sort_field,
+            descending,
+            limit,
+            cache_buf: Vec::new(),
+        }
+    }
+
+    fn topk_batch(&self, mut batches: Vec<RecordBatch>) -> Option<RecordBatch> {
+        if batches.is_empty() {
+            return None;
+        }
+        let mut topk_batch = batches.remove(0);
+        let schema = topk_batch.schema();
+        while !batches.is_empty() {
+            let next_batch = batches.remove(0);
+            if next_batch.num_rows() == 0 {
+                continue;
+            }
+            let new_batch = match concat_batches(&schema, vec![&topk_batch, &next_batch]) {
+                Ok(batch) => batch,
+                Err(e) => {
+                    log::error!("CacheTopkStream: concat_batches failed: {e}");
+                    continue;
+                }
+            };
+            match sort_record_batch_by_column(
+                new_batch,
+                &self.sort_field,
+                self.descending,
+                Some((self.limit as usize * 4).max(1000)),
+            ) {
+                Ok(batch) => {
+                    topk_batch = batch;
+                }
+                Err(e) => {
+                    log::error!("CacheTopkStream: sort_record_batch_by_column failed: {e}");
+                    continue;
+                }
+            };
+        }
+        Some(topk_batch)
+    }
+}
+
+impl Stream for TopKSortStream {
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        match self.stream.poll_next_unpin(cx) {
+            Poll::Ready(Some(Ok(batch))) => {
+                let schema = batch.schema();
+                let empty_batch = RecordBatch::new_empty(schema);
+                match sort_record_batch_by_column(
+                    batch,
+                    &self.sort_field,
+                    self.descending,
+                    Some((self.limit as usize * 4).max(1000)),
+                ) {
+                    Ok(batch) => {
+                        self.cache_buf.push(batch);
+                    }
+                    Err(e) => {
+                        log::error!("CacheTopkStream: sort_record_batch_by_column failed: {e}");
+                    }
+                };
+                Poll::Ready(Some(Ok(empty_batch)))
+            }
+            Poll::Ready(None) => {
+                if self.cache_buf.is_empty() {
+                    return Poll::Ready(None);
+                }
+
+                // sort the cache_buf by the group_expr and return topK
+                let batches = std::mem::take(&mut self.cache_buf);
+                let topk_batch = self.topk_batch(batches);
+                // if let Some(batch) = topk_batch.as_ref() {
+                //     _ = arrow::util::pretty::print_batches(&[batch.clone()]);
+                // }
+                Poll::Ready(topk_batch.map(Ok))
+            }
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(Some(Err(e))) => {
+                log::error!("Error in CacheTopkStream: {e}");
+                Poll::Ready(None)
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.stream.size_hint()
+    }
+}
+
+impl RecordBatchStream for TopKSortStream {
+    /// Get the schema
+    fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+}

src/search/src/datafusion/distributed_plan/broadcast_join_exec.rs

@@ -0,0 +1,378 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{io::Cursor, pin::Pin, sync::Arc, task::Poll};
+
+use arrow::{array::RecordBatch, ipc::writer::FileWriter};
+use config::get_config;
+use datafusion::{
+    arrow::datatypes::SchemaRef,
+    common::{
+        Result, internal_err,
+        tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter},
+    },
+    execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext},
+    physical_expr::{EquivalenceProperties, Partitioning},
+    physical_plan::{
+        DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
+        execute_stream,
+        execution_plan::{Boundedness, EmissionType},
+        metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet},
+    },
+};
+use futures::{Stream, StreamExt};
+use futures_util::ready;
+
+use crate::datafusion::distributed_plan::{
+    once_async::{OnceAsync, OnceFut},
+    tmp_exec::TmpExec,
+};
+
+#[derive(Debug)]
+pub struct BroadcastJoinExec {
+    trace_id: String,
+    left: Arc<dyn ExecutionPlan>,
+    hash_join: Arc<dyn ExecutionPlan>,
+    cache: Arc<PlanProperties>,
+    metrics: ExecutionPlanMetricsSet,
+    // left table result store path in s3
+    cluster: String,
+    path: String,
+    // if left table is not large, directly send to follower node
+    left_data: OnceAsync<Option<Vec<u8>>>,
+}
+
+impl BroadcastJoinExec {
+    pub fn new(
+        trace_id: String,
+        left: Arc<dyn ExecutionPlan>,
+        hash_join: Arc<dyn ExecutionPlan>,
+        cluster: String,
+        path: String,
+    ) -> Self {
+        let schema = hash_join.schema();
+        let partition = hash_join.output_partitioning().partition_count();
+        let cache = Self::compute_properties(Arc::clone(&schema), partition);
+        BroadcastJoinExec {
+            trace_id,
+            left,
+            hash_join,
+            cache,
+            metrics: ExecutionPlanMetricsSet::new(),
+            cluster,
+            path,
+            left_data: OnceAsync::default(),
+        }
+    }
+
+    fn compute_properties(schema: SchemaRef, n_partitions: usize) -> Arc<PlanProperties> {
+        let eq_properties = EquivalenceProperties::new(schema);
+        let output_partitioning = Partitioning::UnknownPartitioning(n_partitions);
+        Arc::new(PlanProperties::new(
+            eq_properties,
+            output_partitioning,
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ))
+    }
+}
+
+impl DisplayAs for BroadcastJoinExec {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "BroadcastJoinExec: cluster={}, path={}",
+            self.cluster, self.path
+        )
+    }
+}
+
+impl ExecutionPlan for BroadcastJoinExec {
+    fn name(&self) -> &'static str {
+        "BroadcastJoinExec"
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.cache
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![&self.left, &self.hash_join]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if children.len() != 2 {
+            return internal_err!("BroadcastJoinExec should have 2 children");
+        }
+        let left = children[0].clone();
+        let hash_join = children[1].clone();
+        Ok(Arc::new(BroadcastJoinExec::new(
+            self.trace_id.clone(),
+            left,
+            hash_join,
+            self.cluster.clone(),
+            self.path.clone(),
+        )))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        let trace_id = self.trace_id.clone();
+        let left_schema = self.left.schema().clone();
+        let path = self.path.clone();
+        let metrics = self.metrics.clone();
+        let left_data = self.left_data.try_once(|| {
+            let left_stream = execute_stream(self.left.clone(), context.clone())?;
+            Ok(collect_left_data(
+                trace_id,
+                left_stream,
+                left_schema,
+                path,
+                metrics,
+            ))
+        })?;
+
+        let metrics = BaselineMetrics::new(&self.metrics, partition);
+        Ok(Box::pin(BroadcastJoinStream::new(
+            self.hash_join.schema().clone(),
+            left_data,
+            self.hash_join.clone(),
+            partition,
+            context,
+            metrics,
+        )))
+    }
+
+    fn metrics(&self) -> Option<MetricsSet> {
+        Some(self.metrics.clone_inner())
+    }
+}
+
+async fn collect_left_data(
+    trace_id: String,
+    mut stream: SendableRecordBatchStream,
+    schema: SchemaRef,
+    path: String,
+    metrics: ExecutionPlanMetricsSet,
+) -> Result<Option<Vec<u8>>> {
+    // 1. collect all left data
+    let collect_left_time = MetricBuilder::new(&metrics).subset_time("collect_left_time", 0);
+    let timer = collect_left_time.timer();
+    let mut batches = Vec::new();
+    while let Some(batch) = stream.next().await.transpose()? {
+        batches.push(batch);
+    }
+    timer.done();
+    log::info!(
+        "[trace_id {trace_id}] BroadcastJoinExec: collect left data took: {} ms",
+        std::time::Duration::from_nanos(collect_left_time.value() as u64).as_millis()
+    );
+
+    // 2. convert record batch to bytes
+    let convert_time = MetricBuilder::new(&metrics).subset_time("convert_time", 0);
+    let timer = convert_time.timer();
+    let mut buffer = Cursor::new(Vec::new());
+    let mut writer = FileWriter::try_new(&mut buffer, &schema)?;
+    for batch in batches {
+        writer.write(&batch)?;
+    }
+    writer.finish()?;
+    let buf = buffer.into_inner();
+    timer.done();
+    log::info!(
+        "[trace_id {trace_id}] BroadcastJoinExec: convert record batch to bytes took: {} ms",
+        std::time::Duration::from_nanos(convert_time.value() as u64).as_millis()
+    );
+
+    // 3. if left data is too large, save to s3, otherwise return bytes
+    if buf.len()
+        > get_config()
+            .common
+            .feature_broadcast_join_left_side_max_size
+            * 1024
+            * 1024
+    {
+        log::info!(
+            "[trace_id {trace_id}] BroadcastJoinExec: left data is too large, save to s3, size: {} MB",
+            buf.len() as f64 / 1024.0 / 1024.0
+        );
+        infra::storage::put("", &path, buf.into()).await?;
+        Ok(None)
+    } else {
+        log::info!(
+            "[trace_id {trace_id}] BroadcastJoinExec: left data is not large, save to memory, size: {} MB",
+            buf.len() as f64 / 1024.0 / 1024.0
+        );
+        Ok(Some(buf))
+    }
+}
+
+impl Drop for BroadcastJoinExec {
+    fn drop(&mut self) {
+        let path = self.path.clone();
+        tokio::task::spawn(async move {
+            if let Err(e) = infra::storage::del(vec![("", &path)]).await {
+                log::error!("[BroadcastJoinExec] Failed to delete left data, path: {path}: {e}");
+            }
+        });
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(super) enum BroadcastJoinStreamState {
+    WaitBuildSide,
+    ProcessProbeBatch,
+    Completed,
+}
+
+struct BroadcastJoinStream {
+    schema: SchemaRef,
+    left_data: OnceFut<Option<Vec<u8>>>,
+    hash_join: Arc<dyn ExecutionPlan>,
+    partition: usize,
+    context: Arc<TaskContext>,
+    right_stream: Option<SendableRecordBatchStream>,
+    state: BroadcastJoinStreamState,
+    metrics: BaselineMetrics,
+}
+
+impl BroadcastJoinStream {
+    pub fn new(
+        schema: SchemaRef,
+        left_data: OnceFut<Option<Vec<u8>>>,
+        hash_join: Arc<dyn ExecutionPlan>,
+        partition: usize,
+        context: Arc<TaskContext>,
+        metrics: BaselineMetrics,
+    ) -> Self {
+        Self {
+            schema,
+            left_data,
+            hash_join,
+            partition,
+            context,
+            right_stream: None,
+            state: BroadcastJoinStreamState::WaitBuildSide,
+            metrics,
+        }
+    }
+
+    fn poll_next_inner(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Option<Result<RecordBatch>>> {
+        match &mut self.state {
+            BroadcastJoinStreamState::WaitBuildSide => self.handle_wait_build_side(cx),
+            BroadcastJoinStreamState::ProcessProbeBatch => self.handle_process_probe_batch(cx),
+            BroadcastJoinStreamState::Completed => Poll::Ready(None),
+        }
+    }
+
+    fn handle_wait_build_side(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Option<Result<RecordBatch>>> {
+        let left_data = ready!(self.left_data.get_shared(cx))?;
+
+        let hash_join = if let Some(left_data) = left_data.as_ref().clone() {
+            let hash_join = self.hash_join.clone();
+            let mut rewriter = TmpExecRewriter::new(left_data);
+            hash_join.rewrite(&mut rewriter)?.data
+        } else {
+            self.hash_join.clone()
+        };
+
+        match hash_join.execute(self.partition, self.context.clone()) {
+            Ok(right_stream) => {
+                self.right_stream = Some(right_stream);
+                self.state = BroadcastJoinStreamState::ProcessProbeBatch;
+                Poll::Ready(Some(Ok(RecordBatch::new_empty(self.schema.clone()))))
+            }
+            Err(e) => Poll::Ready(Some(Err(e))),
+        }
+    }
+
+    fn handle_process_probe_batch(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Option<Result<RecordBatch>>> {
+        if let Some(ref mut right_stream) = self.right_stream {
+            let res = ready!(Pin::new(right_stream).poll_next(cx));
+            match res {
+                Some(Ok(batch)) => {
+                    self.metrics.record_output(batch.num_rows());
+                    Poll::Ready(Some(Ok(batch)))
+                }
+                Some(Err(e)) => Poll::Ready(Some(Err(e))),
+                None => {
+                    self.state = BroadcastJoinStreamState::Completed;
+                    Poll::Ready(None)
+                }
+            }
+        } else {
+            // This should not happen as we set right_stream in handle_wait_build_side
+            Poll::Ready(Some(Err(datafusion::common::DataFusionError::Internal(
+                "Right stream not initialized".to_string(),
+            ))))
+        }
+    }
+}
+
+impl Stream for BroadcastJoinStream {
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Option<Result<RecordBatch>>> {
+        self.poll_next_inner(cx)
+    }
+}
+
+impl RecordBatchStream for BroadcastJoinStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+#[derive(Debug)]
+struct TmpExecRewriter {
+    data: Vec<u8>,
+}
+
+impl TmpExecRewriter {
+    fn new(data: Vec<u8>) -> Self {
+        TmpExecRewriter { data }
+    }
+}
+
+impl TreeNodeRewriter for TmpExecRewriter {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: Arc<dyn ExecutionPlan>) -> Result<Transformed<Self::Node>> {
+        if let Some(tmp_exec) = node.downcast_ref::<TmpExec>() {
+            let tmp =
+                Arc::new(tmp_exec.clone().set_data(self.data.clone())) as Arc<dyn ExecutionPlan>;
+            return Ok(Transformed::new(tmp, true, TreeNodeRecursion::Stop));
+        }
+        Ok(Transformed::no(node))
+    }
+}

src/search/src/datafusion/distributed_plan/cache_buf.rs

@@ -0,0 +1,187 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt::Debug, sync::Arc};
+
+use arrow::array::RecordBatch;
+use datafusion::{common::Result, physical_plan::aggregates::AggregateExec};
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
+
+use crate::datafusion::aggregates::{
+    merge_phase::GroupedHashAggregateStream, no_grouping_merge_phase::AggregateStream,
+};
+
+pub(crate) struct CacheStream {
+    mode: CacheStreamMode,
+    target_partitions: usize,
+    aggregate_plan: Arc<AggregateExec>,
+    data: Vec<Arc<RecordBatch>>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum CacheStreamMode {
+    Group,
+    NoGroup,
+}
+
+impl CacheStream {
+    fn is_empty(&self) -> bool {
+        self.data.is_empty()
+    }
+
+    pub(crate) fn new(
+        has_group_by: bool,
+        target_partitions: usize,
+        aggregate_plan: Arc<AggregateExec>,
+    ) -> Self {
+        Self {
+            mode: if has_group_by {
+                CacheStreamMode::Group
+            } else {
+                CacheStreamMode::NoGroup
+            },
+            target_partitions,
+            aggregate_plan,
+            data: Vec::new(),
+        }
+    }
+}
+
+pub(crate) struct CacheBuf {
+    pub(crate) total_partition_num: usize,
+    pub(crate) cached_partition_num: usize,
+    pub(crate) cached_buf: CacheStream,
+}
+
+impl Debug for CacheBuf {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "CacheBuf")
+    }
+}
+
+impl CacheBuf {
+    pub(crate) fn append_data(&mut self, record_batch: Arc<RecordBatch>) {
+        self.cached_buf.data.push(record_batch);
+    }
+
+    pub(crate) fn check_and_add_partition(&mut self) -> bool {
+        self.cached_partition_num += 1;
+        if self.cached_partition_num >= self.total_partition_num {
+            return true;
+        }
+        false
+    }
+
+    pub(crate) fn get_final_result(&mut self, stream_id: &str) -> Result<Vec<RecordBatch>> {
+        if self.cached_buf.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let merge_mode = self.cached_buf.mode;
+        let start = std::time::Instant::now();
+        let record_batchs = std::mem::take(&mut self.cached_buf.data);
+        let record_batchs: Vec<Arc<RecordBatch>> = record_batchs
+            .into_iter()
+            .filter(|batch| batch.num_rows() != 0)
+            .collect();
+        let total_batch_len = record_batchs.len();
+        let partition_num = std::cmp::max(2, self.cached_buf.target_partitions);
+
+        // When partial_reduce is enabled each follower has already sent pre-merged data, so
+        // phase-1 (parallel chunked aggregation) would double-aggregate already-reduced values.
+        let partial_reduce_enabled = config::get_config().common.feature_partial_reduce_enabled;
+
+        let mut merged_batches: Vec<RecordBatch> = if partial_reduce_enabled {
+            // Phase 1 skipped — use follower results directly.
+            record_batchs
+                .into_iter()
+                .map(|b| b.as_ref().clone())
+                .collect()
+        } else {
+            let thread_pool = rayon::ThreadPoolBuilder::new()
+                .num_threads(partition_num)
+                .build()
+                .unwrap();
+
+            let chunk_size = std::cmp::max(1, total_batch_len / partition_num);
+            let batch_chunks: Vec<Vec<Arc<RecordBatch>>> = record_batchs
+                .chunks(chunk_size)
+                .map(|chunk| chunk.to_vec())
+                .collect();
+
+            // Phase 1: process batch_chunks in parallel using rayon
+            let partial_results: Vec<Result<Vec<RecordBatch>>> = thread_pool.install(|| {
+                batch_chunks
+                    .into_par_iter()
+                    .map(|batches| match merge_mode {
+                        CacheStreamMode::Group => {
+                            let mut stream =
+                                GroupedHashAggregateStream::new(&self.cached_buf.aggregate_plan)
+                                    .unwrap();
+                            for batch in batches {
+                                stream.group_aggregate_batch(batch.as_ref().clone())?;
+                            }
+                            stream.get_final_result()
+                        }
+                        CacheStreamMode::NoGroup => {
+                            let mut stream =
+                                AggregateStream::new(&self.cached_buf.aggregate_plan).unwrap();
+                            for batch in batches {
+                                stream.aggregate_batch(batch.as_ref().clone())?;
+                            }
+                            stream.finalize_aggregation()
+                        }
+                    })
+                    .collect()
+            });
+
+            let mut batches = Vec::new();
+            for partial_result in partial_results {
+                batches.extend(partial_result?);
+            }
+            batches
+        };
+
+        // Phase 2: final merge (always needed when there are multiple batches to combine)
+        match merge_mode {
+            CacheStreamMode::Group => {
+                let mut final_stream =
+                    GroupedHashAggregateStream::new(&self.cached_buf.aggregate_plan).unwrap();
+                for batch in merged_batches {
+                    final_stream.group_aggregate_batch(batch)?;
+                }
+                merged_batches = final_stream.get_final_result()?;
+            }
+            CacheStreamMode::NoGroup => {
+                let mut final_stream =
+                    AggregateStream::new(&self.cached_buf.aggregate_plan).unwrap();
+                for batch in merged_batches {
+                    final_stream.aggregate_batch(batch)?;
+                }
+                merged_batches = final_stream.finalize_aggregation()?;
+            }
+        }
+
+        log::info!(
+            "[StreamingAggs streaming_id: {stream_id}] merge_agg_batches from {total_batch_len} to {}, partial_reduce_enabled: {partial_reduce_enabled} unique_numbers: {}, partition_num: {partition_num}, chunk_size: {}, total_merge_times: {} ms",
+            merged_batches.len(),
+            merged_batches.iter().map(|b| b.num_rows()).sum::<usize>(),
+            std::cmp::max(1, total_batch_len / partition_num),
+            start.elapsed().as_millis(),
+        );
+
+        Ok(merged_batches)
+    }
+}

src/search/src/datafusion/distributed_plan/codec/aggregate_topk_exec.rs

@@ -21,11 +21,11 @@ use datafusion::{
     execution::FunctionRegistry,
     physical_plan::ExecutionPlan,
 };
-#[cfg(feature = "enterprise")]
-use o2_enterprise::enterprise::search::datafusion::distributed_plan::agg_topk_exec::AggregateTopkExec;
 use prost::Message;
 use proto::cluster_rpc;
 
+use crate::datafusion::distributed_plan::aggregate_topk_exec::AggregateTopkExec;
+
 pub fn try_decode(
     node: cluster_rpc::AggregateTopkExecNode,
     inputs: &[Arc<dyn ExecutionPlan>],

src/search/src/datafusion/distributed_plan/codec/mod.rs

@@ -20,16 +20,12 @@ use datafusion::{
 };
 use datafusion_proto::physical_plan::PhysicalExtensionCodec;
 
-#[cfg(feature = "enterprise")]
 mod aggregate_topk_exec;
 mod deduplication_exec;
 mod empty_exec;
-#[cfg(feature = "enterprise")]
 mod enrichment_exec;
 mod physical_plan_node;
-#[cfg(feature = "enterprise")]
 mod streaming_aggs_exec;
-#[cfg(feature = "enterprise")]
 mod tmp_exec;
 
 pub fn get_physical_extension_codec() -> ComposedPhysicalExtensionCodec {

src/search/src/datafusion/distributed_plan/codec/physical_plan_node.rs

@@ -24,17 +24,14 @@ use datafusion::{
 use datafusion_proto::physical_plan::PhysicalExtensionCodec;
 use prost::Message;
 use proto::cluster_rpc;
-#[cfg(feature = "enterprise")]
-use {
-    crate::datafusion::distributed_plan::enrichment_exec::EnrichmentExec,
-    o2_enterprise::enterprise::search::datafusion::distributed_plan::{
-        agg_topk_exec::AggregateTopkExec, streaming_aggs_exec::exec::StreamingAggsExec,
-        tmp_exec::TmpExec,
-    },
-};
 
 use crate::datafusion::{
-    distributed_plan::empty_exec::NewEmptyExec, plan::deduplication_exec::DeduplicationExec,
+    distributed_plan::{
+        aggregate_topk_exec::AggregateTopkExec, empty_exec::NewEmptyExec,
+        enrichment_exec::EnrichmentExec, streaming_aggs_exec::exec::StreamingAggsExec,
+        tmp_exec::TmpExec,
+    },
+    plan::deduplication_exec::DeduplicationExec,
 };
 
 /// A PhysicalExtensionCodec that can serialize and deserialize ChildExec
@@ -60,26 +57,18 @@ impl PhysicalExtensionCodec for PhysicalPlanNodePhysicalExtensionCodec {
             Some(cluster_rpc::physical_plan_node::Plan::DeduplicationExec(node)) => {
                 super::deduplication_exec::try_decode(node, inputs, ctx)
             }
-            #[cfg(feature = "enterprise")]
             Some(cluster_rpc::physical_plan_node::Plan::AggregateTopk(node)) => {
                 super::aggregate_topk_exec::try_decode(node, inputs, ctx)
             }
-            #[cfg(feature = "enterprise")]
             Some(cluster_rpc::physical_plan_node::Plan::StreamingAggs(node)) => {
                 super::streaming_aggs_exec::try_decode(node, inputs, ctx)
             }
-            #[cfg(feature = "enterprise")]
             Some(cluster_rpc::physical_plan_node::Plan::TmpExec(node)) => {
                 super::tmp_exec::try_decode(node, inputs, ctx)
             }
-            #[cfg(feature = "enterprise")]
             Some(cluster_rpc::physical_plan_node::Plan::EnrichmentExec(node)) => {
                 super::enrichment_exec::try_decode(node, inputs, ctx)
             }
-            #[cfg(not(feature = "enterprise"))]
-            Some(_) => {
-                internal_err!("Not supported")
-            }
             None => {
                 internal_err!("PhysicalPlanNode is required")
             }
@@ -87,7 +76,6 @@ impl PhysicalExtensionCodec for PhysicalPlanNodePhysicalExtensionCodec {
     }
 
     fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
-        #[cfg(feature = "enterprise")]
         if node.downcast_ref::<NewEmptyExec>().is_some() {
             super::empty_exec::try_encode(node, buf)
         } else if node.downcast_ref::<DeduplicationExec>().is_some() {
@@ -103,14 +91,6 @@ impl PhysicalExtensionCodec for PhysicalPlanNodePhysicalExtensionCodec {
         } else {
             internal_err!("Not supported")
         }
-        #[cfg(not(feature = "enterprise"))]
-        if node.downcast_ref::<NewEmptyExec>().is_some() {
-            super::empty_exec::try_encode(node, buf)
-        } else if node.downcast_ref::<DeduplicationExec>().is_some() {
-            super::deduplication_exec::try_encode(node, buf)
-        } else {
-            internal_err!("Not supported")
-        }
     }
 }
 

src/search/src/datafusion/distributed_plan/codec/streaming_aggs_exec.rs

@@ -22,10 +22,11 @@ use datafusion::{
     physical_plan::{ExecutionPlan, aggregates::AggregateExec},
 };
 use datafusion_proto::{physical_plan::AsExecutionPlan, protobuf::PhysicalPlanNode};
-use o2_enterprise::enterprise::search::datafusion::distributed_plan::streaming_aggs_exec::exec::StreamingAggsExec;
 use prost::Message;
 use proto::cluster_rpc;
 
+use crate::datafusion::distributed_plan::streaming_aggs_exec::exec::StreamingAggsExec;
+
 pub fn try_decode(
     node: cluster_rpc::StreamingAggsExecNode,
     inputs: &[Arc<dyn ExecutionPlan>],
@@ -116,7 +117,6 @@ mod tests {
     use datafusion_proto::bytes::{
         physical_plan_from_bytes_with_extension_codec, physical_plan_to_bytes_with_extension_codec,
     };
-    use o2_enterprise::enterprise::search::datafusion::distributed_plan::streaming_aggs_exec::exec::StreamingAggsExec;
 
     use super::*;
     use crate::datafusion::udf::str_match_udf::STR_MATCH_UDF;

src/search/src/datafusion/distributed_plan/codec/tmp_exec.rs

@@ -22,10 +22,11 @@ use datafusion::{
     physical_plan::ExecutionPlan,
 };
 use datafusion_proto::{convert_required, protobuf::proto_error};
-use o2_enterprise::enterprise::search::datafusion::distributed_plan::tmp_exec::TmpExec;
 use prost::Message;
 use proto::cluster_rpc;
 
+use crate::datafusion::distributed_plan::tmp_exec::TmpExec;
+
 pub fn try_decode(
     node: cluster_rpc::TmpExecNode,
     _inputs: &[Arc<dyn ExecutionPlan>],

src/search/src/datafusion/distributed_plan/metadata_count_exec.rs

@@ -0,0 +1,185 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int64Array, RecordBatch, UInt64Array};
+use datafusion::{
+    arrow::datatypes::{DataType, SchemaRef},
+    common::{Result, internal_err},
+    execution::{SendableRecordBatchStream, TaskContext},
+    physical_expr::{EquivalenceProperties, Partitioning},
+    physical_plan::{
+        DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
+        execution_plan::{Boundedness, EmissionType},
+        memory::MemoryStream,
+    },
+};
+
+#[derive(Debug)]
+pub struct MetadataCountExec {
+    schema: SchemaRef,
+    records: i64,
+    files: usize,
+    cache: Arc<PlanProperties>,
+}
+
+impl MetadataCountExec {
+    pub fn new(schema: SchemaRef, records: i64, files: usize) -> Self {
+        let cache = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(schema.clone()),
+            Partitioning::UnknownPartitioning(1),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        Self {
+            schema,
+            records,
+            files,
+            cache,
+        }
+    }
+
+    fn data(&self) -> Result<Vec<RecordBatch>> {
+        if self.schema.fields().len() != 1 {
+            return internal_err!(
+                "MetadataCountExec expected one count field, got {}",
+                self.schema.fields().len()
+            );
+        }
+
+        let records = self.records.max(0);
+        let array: ArrayRef = match self.schema.field(0).data_type() {
+            DataType::Int64 => Arc::new(Int64Array::from(vec![records])),
+            DataType::UInt64 => Arc::new(UInt64Array::from(vec![records as u64])),
+            other => {
+                return internal_err!("MetadataCountExec unsupported count type: {other:?}");
+            }
+        };
+
+        RecordBatch::try_new(self.schema.clone(), vec![array])
+            .map(|batch| vec![batch])
+            .map_err(|e| datafusion::error::DataFusionError::Internal(e.to_string()))
+    }
+}
+
+impl DisplayAs for MetadataCountExec {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "MetadataCountExec: files: {}, records: {}",
+            self.files, self.records
+        )
+    }
+}
+
+impl ExecutionPlan for MetadataCountExec {
+    fn name(&self) -> &'static str {
+        "MetadataCountExec"
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.cache
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        _: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(self)
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        _context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        if partition >= 1 {
+            return internal_err!(
+                "MetadataCountExec invalid partition {partition} (expected partition: 0)"
+            );
+        }
+
+        Ok(Box::pin(MemoryStream::try_new(
+            self.data()?,
+            self.schema.clone(),
+            None,
+        )?))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datafusion::{
+        arrow::datatypes::{DataType, Field, Schema},
+        physical_plan::ExecutionPlan,
+    };
+
+    use super::*;
+
+    #[test]
+    fn test_metadata_count_exec_creates_single_count_row() {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "count",
+            DataType::Int64,
+            false,
+        )]));
+        let exec = MetadataCountExec::new(schema, 42, 3);
+        let batches = exec.data().unwrap();
+
+        assert_eq!(batches.len(), 1);
+        assert_eq!(batches[0].num_rows(), 1);
+        let values = batches[0]
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        assert_eq!(values.value(0), 42);
+    }
+
+    #[test]
+    fn test_metadata_count_exec_plan_name() {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "count",
+            DataType::UInt64,
+            false,
+        )]));
+        let exec = MetadataCountExec::new(schema, 7, 2);
+
+        assert_eq!(ExecutionPlan::name(&exec), "MetadataCountExec");
+    }
+
+    #[test]
+    fn test_metadata_count_exec_display_includes_files_and_records() {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "count",
+            DataType::UInt64,
+            false,
+        )]));
+        let exec = Arc::new(MetadataCountExec::new(schema, 7, 2));
+
+        let display = format!(
+            "{}",
+            datafusion::physical_plan::displayable(exec.as_ref()).indent(false)
+        );
+        assert!(display.contains("MetadataCountExec: files: 2, records: 7"));
+    }
+}

src/search/src/datafusion/distributed_plan/mod.rs

@@ -25,6 +25,9 @@ use datafusion::{
 
 use crate::datafusion::distributed_plan::empty_exec::NewEmptyExec;
 
+pub mod aggregate_topk_exec;
+pub mod broadcast_join_exec;
+pub(crate) mod cache_buf;
 pub mod codec;
 mod common;
 mod decoder_stream;
@@ -32,11 +35,14 @@ pub mod display;
 pub mod distribute_analyze_exec;
 pub mod empty_exec;
 pub mod enrich_exec;
-#[cfg(feature = "enterprise")]
 pub mod enrichment_exec;
+pub mod metadata_count_exec;
 pub mod node;
+mod once_async;
 pub mod remote_scan_exec;
 pub mod rewrite;
+pub mod streaming_aggs_exec;
+pub mod tmp_exec;
 
 mod utils;
 

src/search/src/datafusion/distributed_plan/once_async.rs

@@ -0,0 +1,144 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    fmt,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use datafusion::{
+    common::Result,
+    error::{DataFusionError, SharedResult},
+};
+use futures::{
+    FutureExt,
+    future::{BoxFuture, Shared},
+};
+use futures_util::ready;
+use parking_lot::Mutex;
+
+/// refer: https://github.com/apache/datafusion/blob/351675ddc27c42684a079b3a89fe2dee581d89a2/datafusion/physical-plan/src/joins/utils.rs#L336
+/// A [`OnceAsync`] runs an `async` closure once, where multiple calls to
+/// [`OnceAsync::try_once`] return a [`OnceFut`] that resolves to the result of the
+/// same computation.
+///
+/// This is useful for joins where the results of one child are needed to proceed
+/// with multiple output stream
+///
+/// For example, in a hash join, one input is buffered and shared across
+/// potentially multiple output partitions. Each output partition must wait for
+/// the hash table to be built before proceeding.
+///
+/// Each output partition waits on the same `OnceAsync` before proceeding.
+pub(crate) struct OnceAsync<T> {
+    fut: Mutex<Option<SharedResult<OnceFut<T>>>>,
+}
+
+impl<T> Default for OnceAsync<T> {
+    fn default() -> Self {
+        Self {
+            fut: Mutex::new(None),
+        }
+    }
+}
+
+impl<T> fmt::Debug for OnceAsync<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "OnceAsync")
+    }
+}
+
+impl<T: 'static> OnceAsync<T> {
+    /// If this is the first call to this function on this object, will invoke
+    /// `f` to obtain a future and return a [`OnceFut`] referring to this. `f`
+    /// may fail, in which case its error is returned.
+    ///
+    /// If this is not the first call, will return a [`OnceFut`] referring
+    /// to the same future as was returned by the first call - or the same
+    /// error if the initial call to `f` failed.
+    pub(crate) fn try_once<F, Fut>(&self, f: F) -> Result<OnceFut<T>>
+    where
+        F: FnOnce() -> Result<Fut>,
+        Fut: Future<Output = Result<T>> + Send + 'static,
+    {
+        self.fut
+            .lock()
+            .get_or_insert_with(|| f().map(OnceFut::new).map_err(Arc::new))
+            .clone()
+            .map_err(DataFusionError::Shared)
+    }
+}
+
+/// A [`OnceFut`] represents a shared asynchronous computation, that will be evaluated
+/// once for all [`Clone`]'s, with [`OnceFut::get`] providing a non-consuming interface
+/// to drive the underlying [`Future`] to completion
+pub(crate) struct OnceFut<T> {
+    state: OnceFutState<T>,
+}
+
+impl<T> Clone for OnceFut<T> {
+    fn clone(&self) -> Self {
+        Self {
+            state: self.state.clone(),
+        }
+    }
+}
+
+enum OnceFutState<T> {
+    Pending(OnceFutPending<T>),
+    Ready(SharedResult<Arc<T>>),
+}
+
+impl<T> Clone for OnceFutState<T> {
+    fn clone(&self) -> Self {
+        match self {
+            Self::Pending(p) => Self::Pending(p.clone()),
+            Self::Ready(r) => Self::Ready(r.clone()),
+        }
+    }
+}
+
+impl<T: 'static> OnceFut<T> {
+    /// Create a new [`OnceFut`] from a [`Future`]
+    pub(crate) fn new<Fut>(fut: Fut) -> Self
+    where
+        Fut: Future<Output = Result<T>> + Send + 'static,
+    {
+        Self {
+            state: OnceFutState::Pending(
+                fut.map(|res| res.map(Arc::new).map_err(Arc::new))
+                    .boxed()
+                    .shared(),
+            ),
+        }
+    }
+
+    /// Get shared reference to the result of the computation if it is ready, without consuming it
+    pub(crate) fn get_shared(&mut self, cx: &mut Context<'_>) -> Poll<Result<Arc<T>>> {
+        if let OnceFutState::Pending(fut) = &mut self.state {
+            let r = ready!(fut.poll_unpin(cx));
+            self.state = OnceFutState::Ready(r);
+        }
+
+        match &self.state {
+            OnceFutState::Pending(_) => unreachable!(),
+            OnceFutState::Ready(r) => Poll::Ready(r.clone().map_err(DataFusionError::Shared)),
+        }
+    }
+}
+
+/// The shared future type used internally within [`OnceAsync`]
+type OnceFutPending<T> = Shared<BoxFuture<'static, SharedResult<Arc<T>>>>;

src/search/src/datafusion/distributed_plan/rewrite.rs

@@ -28,11 +28,13 @@ use datafusion::{
         union::UnionExec,
     },
 };
-#[cfg(feature = "enterprise")]
-use o2_enterprise::enterprise::search::datafusion::distributed_plan::metadata_count_exec::MetadataCountExec;
 
 use crate::{
-    datafusion::plan::tantivy_optimize_exec::TantivyOptimizeExec, index::IndexCondition,
+    datafusion::{
+        distributed_plan::metadata_count_exec::MetadataCountExec,
+        plan::tantivy_optimize_exec::TantivyOptimizeExec,
+    },
+    index::IndexCondition,
     types::QueryParams,
 };
 
@@ -68,9 +70,7 @@ pub struct AggregateOptimizeRewriter {
     file_list: Vec<FileKey>,
     index_condition: Option<IndexCondition>,
     index_optimize_mode: Option<IndexOptimizeMode>,
-    #[allow(unused)]
     metadata_records: i64,
-    #[allow(unused)]
     metadata_files: usize,
 }
 
@@ -105,7 +105,6 @@ impl AggregateOptimizeRewriter {
         ))
     }
 
-    #[cfg(feature = "enterprise")]
     fn metadata_count_exec(&self, schema: SchemaRef) -> Result<Arc<dyn ExecutionPlan>> {
         Ok(Arc::new(MetadataCountExec::new(
             schema,
@@ -117,7 +116,6 @@ impl AggregateOptimizeRewriter {
     fn additional_inputs(&mut self, schema: SchemaRef) -> Result<Vec<Arc<dyn ExecutionPlan>>> {
         let mut inputs = Vec::new();
 
-        #[cfg(feature = "enterprise")]
         if self.metadata_records > 0 {
             inputs.push(self.metadata_count_exec(schema.clone())?);
         }
@@ -153,9 +151,7 @@ mod tests {
     use std::sync::Arc;
 
     use arrow_schema::{DataType, Field, Schema};
-    #[cfg(feature = "enterprise")]
-    use config::meta::stream::FileMeta;
-    use config::meta::stream::{FileKey, StreamType};
+    use config::meta::stream::{FileKey, FileMeta, StreamType};
     use datafusion::{
         common::Result,
         functions_aggregate::count::count_udaf,
@@ -208,7 +204,6 @@ mod tests {
         )?))
     }
 
-    #[cfg(feature = "enterprise")]
     #[test]
     fn test_aggregate_optimize_rewrite_combines_metadata_and_tantivy_inputs() -> Result<()> {
         let plan = partial_count_exec()?;

src/search/src/datafusion/distributed_plan/streaming_aggs_exec/cached_file_stream.rs

@@ -0,0 +1,162 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use arrow::{array::RecordBatch, datatypes::SchemaRef};
+use datafusion::{common::Result, execution::RecordBatchStream};
+use futures::Stream;
+
+use crate::cache::streaming_agg::get_record_batches;
+
+pub(crate) struct CachedFileStream {
+    id: String,
+    cached_files: Vec<Arc<String>>,
+    schema: SchemaRef,
+    current_file_index: usize,
+    current_batches: Vec<RecordBatch>,
+    current_batch_index: usize,
+    is_exhausted: bool,
+}
+
+impl CachedFileStream {
+    pub(crate) fn new(id: String, cached_files: Vec<Arc<String>>, schema: SchemaRef) -> Self {
+        Self {
+            id,
+            cached_files,
+            schema,
+            current_file_index: 0,
+            current_batches: Vec::new(),
+            current_batch_index: 0,
+            is_exhausted: false,
+        }
+    }
+
+    pub(crate) fn load_next_file(&mut self) -> Result<()> {
+        loop {
+            if self.current_file_index >= self.cached_files.len() {
+                self.is_exhausted = true;
+                return Ok(());
+            }
+
+            let file_path = &self.cached_files[self.current_file_index];
+            let batches = match get_record_batches(&self.id, file_path, self.schema.clone()) {
+                Ok(batches) => batches,
+                Err(e) => {
+                    log::error!(
+                        "[StreamingAggs streaming_id: {}] Error reading cached file: {file_path}, error: {e:?}",
+                        self.id,
+                    );
+                    return Err(e.into());
+                }
+            };
+
+            log::debug!(
+                "[StreamingAggs streaming_id: {}] Successfully read {} batches from cached file: {file_path}",
+                self.id,
+                batches.len(),
+            );
+
+            self.current_batches = batches;
+            self.current_batch_index = 0;
+            self.current_file_index += 1;
+
+            if self.current_batches.is_empty() {
+                continue;
+            }
+
+            break;
+        }
+        Ok(())
+    }
+}
+
+impl Stream for CachedFileStream {
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        if self.is_exhausted {
+            return Poll::Ready(None);
+        }
+
+        // If we don't have current batches or we've exhausted them, load next file
+        if self.current_batches.is_empty() || self.current_batch_index >= self.current_batches.len()
+        {
+            if let Err(e) = self.load_next_file() {
+                return Poll::Ready(Some(Err(e)));
+            }
+
+            if self.is_exhausted {
+                return Poll::Ready(None);
+            }
+        }
+
+        // Return the next batch if available
+        if self.current_batch_index < self.current_batches.len() {
+            let batch = self.current_batches[self.current_batch_index].clone();
+            self.current_batch_index += 1;
+            Poll::Ready(Some(Ok(batch)))
+        } else {
+            Poll::Ready(None)
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (0, None)
+    }
+}
+
+impl RecordBatchStream for CachedFileStream {
+    fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow::datatypes::{DataType, Field, Schema};
+
+    use super::*;
+
+    #[test]
+    fn test_cached_file_stream_new() {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "col1",
+            DataType::Int64,
+            false,
+        )]));
+
+        let cached_files = vec![
+            Arc::new("file1.arrow".to_string()),
+            Arc::new("file2.arrow".to_string()),
+        ];
+
+        let stream = CachedFileStream::new(
+            "test_cached_stream".to_string(),
+            cached_files.clone(),
+            schema.clone(),
+        );
+
+        assert_eq!(stream.id, "test_cached_stream");
+        assert_eq!(stream.cached_files.len(), 2);
+        assert_eq!(stream.current_file_index, 0);
+    }
+}

src/search/src/datafusion/distributed_plan/streaming_aggs_exec/exec.rs

@@ -0,0 +1,305 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt::Debug, sync::Arc};
+
+use arrow::datatypes::SchemaRef;
+use datafusion::{
+    common::Result,
+    error::DataFusionError,
+    execution::{SendableRecordBatchStream, TaskContext},
+    physical_expr::EquivalenceProperties,
+    physical_plan::{
+        DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
+        PlanProperties,
+        aggregates::AggregateExec,
+        execution_plan::{Boundedness, EmissionType},
+    },
+};
+use parking_lot::Mutex;
+
+use crate::datafusion::distributed_plan::{
+    cache_buf::{CacheBuf, CacheStream},
+    streaming_aggs_exec::{cached_file_stream::CachedFileStream, monitor_stream::MonitorStream},
+};
+
+#[derive(Debug)]
+pub struct StreamingAggsExec {
+    id: String,
+    start_time: i64,
+    end_time: i64,
+    input: Arc<dyn ExecutionPlan>,
+    /// Cache holding plan properties like equivalences, output partitioning etc.
+    cache: Arc<PlanProperties>,
+    cached_files: Vec<Arc<String>>,
+    cached_partition_num: usize,
+    target_partitions: usize,
+    is_complete_cache_hit: bool,
+    aggregate_plan: Arc<AggregateExec>,
+    cache_buf: Arc<Mutex<CacheBuf>>,
+    overwrite_cache: bool,
+}
+
+impl StreamingAggsExec {
+    /// Create a new StreamingAggsExec with explicit cache strategy
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        id: String,
+        start_time: i64,
+        end_time: i64,
+        cached_files: Vec<Arc<String>>,
+        input: Arc<dyn ExecutionPlan>,
+        target_partitions: usize,
+        is_complete_cache_hit: bool,
+        aggregate_plan: Arc<AggregateExec>,
+        overwrite_cache: bool,
+    ) -> Self {
+        let cached_partition_num = if cached_files.is_empty() { 0 } else { 1 };
+        let total_partition_num = if is_complete_cache_hit {
+            cached_partition_num
+        } else {
+            // Partial or no cache: cached partitions + input partitions
+            let input_partitions = input.output_partitioning().partition_count();
+            input_partitions + cached_partition_num
+        };
+
+        let cache = Self::compute_properties(Arc::clone(&input.schema()), total_partition_num);
+
+        let cached_buf = CacheStream::new(
+            !aggregate_plan.group_expr().is_empty(),
+            target_partitions,
+            aggregate_plan.clone(),
+        );
+
+        Self {
+            id,
+            start_time,
+            end_time,
+            input,
+            cache,
+            cached_files,
+            cached_partition_num,
+            target_partitions,
+            is_complete_cache_hit,
+            aggregate_plan,
+            cache_buf: Arc::new(Mutex::new(CacheBuf {
+                total_partition_num,
+                cached_partition_num,
+                cached_buf,
+            })),
+            overwrite_cache,
+        }
+    }
+
+    pub fn id(&self) -> &str {
+        &self.id
+    }
+
+    pub fn start_time(&self) -> i64 {
+        self.start_time
+    }
+
+    pub fn end_time(&self) -> i64 {
+        self.end_time
+    }
+
+    pub fn target_partitions(&self) -> usize {
+        self.target_partitions
+    }
+
+    pub fn is_complete_cache_hit(&self) -> bool {
+        self.is_complete_cache_hit
+    }
+
+    pub fn cached_files(&self) -> &[Arc<String>] {
+        &self.cached_files
+    }
+
+    pub fn aggregate_plan(&self) -> &Arc<AggregateExec> {
+        &self.aggregate_plan
+    }
+
+    pub fn overwrite_cache(&self) -> bool {
+        self.overwrite_cache
+    }
+
+    pub(crate) fn output_partitioning_helper(n_partitions: usize) -> Partitioning {
+        Partitioning::UnknownPartitioning(n_partitions)
+    }
+
+    /// This function creates the cache object that stores the plan properties such as schema,
+    /// equivalence properties, ordering, partitioning, etc.
+    pub(crate) fn compute_properties(
+        schema: SchemaRef,
+        n_partitions: usize,
+    ) -> Arc<PlanProperties> {
+        let eq_properties = EquivalenceProperties::new(schema);
+        let output_partitioning = Self::output_partitioning_helper(n_partitions);
+        Arc::new(PlanProperties::new(
+            eq_properties,
+            // Output Partitioning
+            output_partitioning,
+            // Execution Mode
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ))
+    }
+}
+
+impl DisplayAs for StreamingAggsExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                let strategy = if self.is_complete_cache_hit {
+                    "complete_hit"
+                } else {
+                    "miss"
+                };
+                write!(
+                    f,
+                    "StreamingAggsExec: streaming_id={}, cache_strategy={strategy}, cached_partitions={}, total_partitions={}",
+                    self.id,
+                    self.cached_partition_num,
+                    self.properties().output_partitioning().partition_count()
+                )
+            }
+            DisplayFormatType::TreeRender => {
+                let strategy = if self.is_complete_cache_hit {
+                    "complete_hit"
+                } else {
+                    "miss"
+                };
+                _ = writeln!(f, "streaming_id={}", self.id);
+                _ = writeln!(f, "cache_strategy={strategy}",);
+                _ = writeln!(f, "cached_partitions={}", self.cached_partition_num);
+                _ = writeln!(
+                    f,
+                    "total_partitions={}",
+                    self.properties().output_partitioning().partition_count()
+                );
+                Ok(())
+            }
+        }
+    }
+}
+
+impl ExecutionPlan for StreamingAggsExec {
+    fn name(&self) -> &'static str {
+        "StreamingAggsExec"
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.cache
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(Arc::new(StreamingAggsExec::new(
+            self.id.clone(),
+            self.start_time,
+            self.end_time,
+            self.cached_files.clone(),
+            children[0].clone(),
+            self.target_partitions,
+            self.is_complete_cache_hit,
+            self.aggregate_plan.clone(),
+            self.overwrite_cache,
+        )))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        // Complete cache hit: only return cached data, never execute input
+        if self.is_complete_cache_hit {
+            log::debug!(
+                "[StreamingAggs streaming_id: {}] Complete cache hit: returning cached data for partition {}/{}",
+                self.id,
+                partition,
+                self.cached_partition_num
+            );
+            if partition < self.cached_partition_num {
+                log::debug!(
+                    "[StreamingAggs streaming_id: {}] EXECUTING with cached files for partition {} (complete cache hit), time_range=[{}, {}], files: {:?}",
+                    self.id,
+                    partition,
+                    self.start_time,
+                    self.end_time,
+                    self.cached_files
+                );
+                // Create a lazy stream that will read cached files on demand
+                return Ok(Box::pin(CachedFileStream::new(
+                    self.id.clone(),
+                    self.cached_files.clone(),
+                    self.input.schema(),
+                )));
+            } else {
+                // This should never happen with complete cache hit
+                return Err(DataFusionError::Internal(format!(
+                    "StreamingAggsExec: Invalid partition {} for complete cache hit with {} cached partitions",
+                    partition, self.cached_partition_num
+                )));
+            }
+        }
+
+        log::debug!(
+            "[StreamingAggs streaming_id: {}] Partial cache hit: partition={}, cached_partitions={}, executing input for new data",
+            self.id,
+            partition,
+            self.cached_partition_num
+        );
+
+        // Partial or no cache: handle both cached and input partitions
+        if partition < self.cached_partition_num {
+            log::debug!(
+                "[StreamingAggs streaming_id: {}] EXECUTING with cached files for partition {} (partial cache hit), time_range=[{}, {}], files: {:?}",
+                self.id,
+                partition,
+                self.start_time,
+                self.end_time,
+                self.cached_files
+            );
+            return Ok(Box::pin(CachedFileStream::new(
+                self.id.clone(),
+                self.cached_files.clone(),
+                self.input.schema(),
+            )));
+        }
+
+        // Execute input for missing data
+        Ok(Box::pin(MonitorStream::new(
+            self.id.clone(),
+            self.start_time,
+            self.end_time,
+            self.input.schema(),
+            self.cache_buf.clone(),
+            self.input
+                .execute(partition - self.cached_partition_num, context)?,
+            self.overwrite_cache,
+        )))
+    }
+
+    fn benefits_from_input_partitioning(&self) -> Vec<bool> {
+        vec![false; self.children().len()]
+    }
+}

src/search/src/datafusion/distributed_plan/streaming_aggs_exec/monitor_stream.rs

@@ -0,0 +1,248 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use arrow::{array::RecordBatch, datatypes::SchemaRef};
+use datafusion::{
+    common::Result,
+    execution::{RecordBatchStream, SendableRecordBatchStream},
+};
+use futures::{Stream, StreamExt};
+use futures_util::ready;
+use parking_lot::Mutex;
+
+use crate::{
+    cache::streaming_agg::{
+        RecordBatchCacheRequest, cache_record_batches_to_disk,
+        generate_aggregation_cache_file_name, get_cache_file_path,
+    },
+    datafusion::distributed_plan::{
+        cache_buf::CacheBuf,
+        streaming_aggs_exec::{GLOBAL_CACHE, get_cache_file_path_from_streaming_id},
+    },
+};
+
+pub(crate) struct MonitorStream {
+    id: String,
+    start_time: i64,
+    end_time: i64,
+    schema: SchemaRef,
+    stream: SendableRecordBatchStream,
+    root_cache_buf: Arc<Mutex<CacheBuf>>,
+    done: bool,
+    overwrite_cache: bool,
+}
+
+impl MonitorStream {
+    pub(crate) fn new(
+        id: String,
+        start_time: i64,
+        end_time: i64,
+        schema: SchemaRef,
+        root_cache_buf: Arc<Mutex<CacheBuf>>,
+        stream: SendableRecordBatchStream,
+        overwrite_cache: bool,
+    ) -> Self {
+        Self {
+            id,
+            start_time,
+            end_time,
+            schema,
+            stream,
+            root_cache_buf,
+            done: false,
+            overwrite_cache,
+        }
+    }
+
+    pub fn is_complete_partition_window(&self) -> bool {
+        let interval = GLOBAL_CACHE.get_cache_interval(&self.id); // minutes
+        let interval_micros = interval * 60 * 1_000_000; // microseconds
+        (self.end_time - self.start_time) == interval_micros
+    }
+
+    pub fn append_to_cache_buf(&mut self, record_batch: Arc<RecordBatch>) {
+        self.root_cache_buf.lock().append_data(record_batch);
+    }
+
+    pub fn check_and_add_partition(&mut self) -> bool {
+        self.root_cache_buf.lock().check_and_add_partition()
+    }
+}
+
+impl Stream for MonitorStream {
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        if self.done {
+            return Poll::Ready(None);
+        }
+
+        let res = ready!(self.stream.poll_next_unpin(cx));
+
+        Poll::Ready(match res {
+            Some(Ok(record_batch)) => {
+                self.append_to_cache_buf(Arc::new(record_batch.clone()));
+                Some(Ok(record_batch))
+            }
+            None => {
+                self.done = true;
+
+                let partition_done = self.check_and_add_partition();
+                let streaming_done = partition_done
+                    && GLOBAL_CACHE
+                        .id_cache
+                        .check_time(&self.id, self.start_time, self.end_time);
+
+                let file_path = get_cache_file_path_from_streaming_id(&self.id)?;
+                let file_name = generate_aggregation_cache_file_name(
+                    &self.id,
+                    self.start_time,
+                    self.end_time,
+                    self.is_complete_partition_window(),
+                );
+
+                // Start - Cache record batches to disk
+                if partition_done && (!streaming_done || self.is_complete_partition_window()) {
+                    let result_vec = self.root_cache_buf.lock().get_final_result(&self.id)?;
+
+                    let file_path = get_cache_file_path(&file_path, &file_name);
+                    let request = RecordBatchCacheRequest {
+                        streaming_id: self.id.clone(),
+                        file_path: file_path.clone(),
+                        schema: self.schema.clone(),
+                        records: result_vec.into_iter().map(Arc::new).collect(),
+                        overwrite_cache: self.overwrite_cache,
+                    };
+
+                    let start = std::time::Instant::now();
+                    match cache_record_batches_to_disk(request) {
+                        Ok(()) => {
+                            // add to cache list
+                            GLOBAL_CACHE.insert(self.id.clone(), file_path);
+                        }
+                        Err(e) => {
+                            log::error!(
+                                "[streaming_id: {}] Error caching streaming aggs record batchesto disk file: {file_path}, error: {e:?}",
+                                self.id,
+                            );
+                        }
+                    }
+                    log::info!(
+                        "[streaming_id: {}] cache_record_batches_to_disk time: {} ms",
+                        self.id,
+                        start.elapsed().as_millis()
+                    );
+                }
+                // End - Cache record batches to disk
+
+                None
+            }
+            Some(Err(e)) => {
+                log::error!("[streaming_id: {}] Error in MonitorStream: {e}", self.id);
+                Some(Err(e))
+            }
+        })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.stream.size_hint()
+    }
+}
+
+impl RecordBatchStream for MonitorStream {
+    /// Get the schema
+    fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::{
+        execution::SendableRecordBatchStream,
+        physical_plan::{aggregates::AggregateExec, memory::MemoryStream},
+    };
+    use tokio::sync::mpsc;
+
+    use super::*;
+    use crate::datafusion::distributed_plan::cache_buf::CacheStream;
+
+    #[test]
+    fn test_monitor_stream_new() {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "col1",
+            DataType::Int64,
+            false,
+        )]));
+
+        // Create a dummy stream
+        let batches = vec![];
+        let memory_stream = MemoryStream::try_new(batches, schema.clone(), None).unwrap();
+        let input_stream: SendableRecordBatchStream = Box::pin(memory_stream);
+
+        let (_tx, _rx): (
+            tokio::sync::mpsc::Sender<()>,
+            tokio::sync::mpsc::Receiver<()>,
+        ) = mpsc::channel(1);
+
+        // Create a dummy cache buffer for MonitorStream
+        let cache_buf = Arc::new(parking_lot::Mutex::new(CacheBuf {
+            total_partition_num: 1,
+            cached_partition_num: 0,
+            cached_buf: CacheStream::new(
+                false,
+                1,
+                Arc::new(
+                    AggregateExec::try_new(
+                        datafusion::physical_plan::aggregates::AggregateMode::Partial,
+                        datafusion::physical_plan::aggregates::PhysicalGroupBy::new_single(vec![]),
+                        vec![],
+                        vec![],
+                        Arc::new(datafusion::physical_plan::empty::EmptyExec::new(
+                            schema.clone(),
+                        )),
+                        schema.clone(),
+                    )
+                    .unwrap(),
+                ),
+            ),
+        }));
+
+        // Test MonitorStream::new
+        let monitor_stream = MonitorStream::new(
+            "test_monitor".to_string(),
+            1000,
+            2000,
+            schema.clone(),
+            cache_buf,
+            input_stream,
+            false,
+        );
+
+        // Verify initial state
+        assert_eq!(monitor_stream.id, "test_monitor");
+        assert_eq!(monitor_stream.start_time, 1000);
+        assert_eq!(monitor_stream.end_time, 2000);
+    }
+}

src/search/src/datafusion/distributed_plan/tmp_exec.rs

@@ -0,0 +1,234 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{io::Cursor, sync::Arc};
+
+use arrow::ipc::reader::FileReader;
+use datafusion::{
+    arrow::datatypes::SchemaRef,
+    common::{Result, internal_err},
+    execution::{SendableRecordBatchStream, TaskContext},
+    physical_expr::{EquivalenceProperties, Partitioning},
+    physical_plan::{
+        DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
+        execution_plan::{Boundedness, EmissionType},
+        memory::MemoryStream,
+        stream::RecordBatchStreamAdapter,
+    },
+};
+use futures::TryStreamExt;
+#[cfg(feature = "enterprise")]
+use infra::client::grpc::make_grpc_search_client;
+#[cfg(feature = "enterprise")]
+use o2_enterprise::enterprise::{
+    common::config::get_config as get_o2_config, super_cluster::search::get_cluster_node_by_name,
+};
+
+#[derive(Debug, Clone)]
+pub struct TmpExec {
+    trace_id: String,
+    cluster: String,
+    path: String,
+    data: Option<Vec<u8>>,
+    schema: SchemaRef,
+    cache: Arc<PlanProperties>,
+}
+
+impl TmpExec {
+    pub fn new(
+        trace_id: String,
+        cluster: String,
+        path: String,
+        data: Option<Vec<u8>>,
+        schema: SchemaRef,
+    ) -> Self {
+        let cache = Self::compute_properties(Arc::clone(&schema), 1);
+        TmpExec {
+            trace_id,
+            cluster,
+            path,
+            data,
+            schema,
+            cache,
+        }
+    }
+
+    fn compute_properties(schema: SchemaRef, n_partitions: usize) -> Arc<PlanProperties> {
+        let eq_properties = EquivalenceProperties::new(schema);
+        let output_partitioning = Partitioning::UnknownPartitioning(n_partitions);
+        Arc::new(PlanProperties::new(
+            eq_properties,
+            output_partitioning,
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ))
+    }
+
+    pub fn trace_id(&self) -> &str {
+        &self.trace_id
+    }
+
+    pub fn cluster(&self) -> &str {
+        &self.cluster
+    }
+
+    pub fn path(&self) -> &str {
+        &self.path
+    }
+
+    pub fn data(&self) -> &Option<Vec<u8>> {
+        &self.data
+    }
+
+    pub fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
+    pub fn set_data(mut self, data: Vec<u8>) -> Self {
+        self.data = Some(data);
+        self
+    }
+}
+
+impl DisplayAs for TmpExec {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "TmpExec: cluster={}, path={}", self.cluster, self.path)
+    }
+}
+
+impl ExecutionPlan for TmpExec {
+    fn name(&self) -> &'static str {
+        "TmpExec"
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.cache
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        _: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(self)
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        _context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        if partition != 0 {
+            return internal_err!("TmpExec invalid partition {partition} (expected partition: 0)");
+        }
+
+        if let Some(data) = self.data.clone() {
+            let reader =
+                unsafe { FileReader::try_new(Cursor::new(data), None)?.with_skip_validation(true) };
+            let mut batches = Vec::new();
+            for batch in reader {
+                batches.push(batch?);
+            }
+            Ok(Box::pin(MemoryStream::try_new(
+                batches,
+                self.schema.clone(),
+                None,
+            )?))
+        } else {
+            let data = fetch_data(
+                self.trace_id.clone(),
+                self.cluster.clone(),
+                self.path.clone(),
+                Arc::clone(&self.schema),
+            );
+            let stream = futures::stream::once(data).try_flatten();
+            Ok(Box::pin(RecordBatchStreamAdapter::new(
+                self.schema.clone(),
+                stream,
+            )))
+        }
+    }
+}
+
+async fn fetch_data(
+    trace_id: String,
+    cluster: String,
+    path: String,
+    schema: SchemaRef,
+) -> Result<SendableRecordBatchStream> {
+    let data = if cluster == config::get_cluster_name() {
+        infra::storage::get_bytes("", &path).await?
+    } else {
+        #[cfg(feature = "enterprise")]
+        {
+            if !get_o2_config().super_cluster.enabled {
+                return internal_err!(
+                    "cluster: {cluster}'s left data result is in other cluster: {}",
+                    config::get_cluster_name()
+                );
+            }
+            let node = match get_cluster_node_by_name(&cluster).await {
+                Ok(node) => node,
+                Err(e) => return internal_err!("Failed to get cluster node: {e:?}"),
+            };
+            let grpc_addr = node.get_grpc_addr();
+            let path = path.to_string();
+            let task = tokio::task::spawn(async move {
+                let mut request = tonic::Request::new(proto::cluster_rpc::GetTableRequest { path });
+                match make_grpc_search_client(&trace_id, &mut request, &node, 0).await {
+                    Ok(mut client) => match client.get_table(request).await {
+                        Ok(res) => Ok(res.into_inner()),
+                        Err(err) => {
+                            log::error!("search->grpc: node: {grpc_addr}, search err: {err:?}",);
+                            Err(format!("{err:?}"))
+                        }
+                    },
+                    Err(e) => Err(format!("{e:?}")),
+                }
+            });
+
+            let response = match task.await {
+                Ok(Ok(response)) => response,
+                Ok(Err(e)) => return internal_err!("GRPC call failed: {e}"),
+                Err(e) => return internal_err!("Task join failed: {e:?}"),
+            };
+            response.data.into()
+        }
+        #[cfg(not(feature = "enterprise"))]
+        {
+            let _ = trace_id;
+            return internal_err!(
+                "cluster: {cluster}'s left data result is in other cluster: {}",
+                config::get_cluster_name()
+            );
+        }
+    };
+
+    let buf = data;
+    let reader = unsafe { FileReader::try_new(Cursor::new(buf), None)?.with_skip_validation(true) };
+    let mut batches = Vec::new();
+    for batch in reader {
+        batches.push(batch?);
+    }
+
+    Ok(Box::pin(MemoryStream::try_new(
+        batches,
+        Arc::clone(&schema),
+        None,
+    )?))
+}

src/search/src/datafusion/exec.rs

@@ -321,6 +321,12 @@ pub fn register_builtin_udfs(ctx: &SessionContext) {
     ctx.register_udaf(AggregateUDF::from(
         super::udaf::summary_percentile::SummaryPercentile::new(),
     ));
+    ctx.register_udaf(AggregateUDF::from(
+        super::udaf::approx_topk::ApproxTopK::new(),
+    ));
+    ctx.register_udaf(AggregateUDF::from(
+        super::udaf::approx_topk_distinct::ApproxTopKDistinct::new(),
+    ));
     ctx.register_udf(super::udf::cast_to_timestamp_udf::CAST_TO_TIMESTAMP_UDF.clone());
 
     #[cfg(feature = "enterprise")]
@@ -328,12 +334,6 @@ pub fn register_builtin_udfs(ctx: &SessionContext) {
         ctx.register_udf(super::udf::cipher_udf::DECRYPT_UDF.clone());
         ctx.register_udf(super::udf::cipher_udf::DECRYPT_SLOW_UDF.clone());
         ctx.register_udf(super::udf::cipher_udf::ENCRYPT_UDF.clone());
-        ctx.register_udaf(AggregateUDF::from(
-            o2_enterprise::enterprise::search::datafusion::udaf::approx_topk::ApproxTopK::new(),
-        ));
-        ctx.register_udaf(AggregateUDF::from(
-            o2_enterprise::enterprise::search::datafusion::udaf::approx_topk_distinct::ApproxTopKDistinct::new(),
-        ));
         ctx.register_udaf(AggregateUDF::from(
             o2_enterprise::enterprise::search::datafusion::udaf::ddsketch::DDSketchAgg::new(),
         ));

src/search/src/datafusion/mod.rs

@@ -15,6 +15,7 @@
 
 use std::str::FromStr;
 
+pub mod aggregates;
 pub mod context;
 pub mod distributed_plan;
 pub mod exec;

src/search/src/datafusion/optimizer/context.rs

@@ -16,15 +16,12 @@
 use std::sync::Arc;
 
 use config::{datafusion::request::Request, meta::cluster::NodeInfo};
-use datafusion::sql::TableReference;
+use datafusion::{physical_optimizer::PhysicalOptimizerRule, sql::TableReference};
 use hashbrown::HashMap;
 use infra::errors::Error;
 use parking_lot::Mutex;
-#[cfg(feature = "enterprise")]
-use {
-    datafusion::physical_optimizer::PhysicalOptimizerRule,
-    o2_enterprise::enterprise::search::datafusion::optimizer::stream_aggregate::StreamingAggsRule,
-};
+
+use crate::datafusion::optimizer::stream_aggregate::StreamingAggsRule;
 
 pub enum PhysicalOptimizerContext {
     RemoteScan(RemoteScanContext),
@@ -39,7 +36,6 @@ pub struct RemoteScanContext {
     pub is_leader: bool,
 }
 
-#[cfg(feature = "enterprise")]
 pub struct StreamingAggregationContext {
     pub streaming_id: String,
     pub start_time: i64,
@@ -48,7 +44,6 @@ pub struct StreamingAggregationContext {
     pub overwrite_cache: bool,
 }
 
-#[cfg(feature = "enterprise")]
 impl StreamingAggregationContext {
     pub async fn new(
         request: &Request,
@@ -75,7 +70,6 @@ impl StreamingAggregationContext {
     }
 }
 
-#[cfg(feature = "enterprise")]
 pub fn generate_streaming_agg_rules(
     context: StreamingAggregationContext,
 ) -> Arc<dyn PhysicalOptimizerRule + Send + Sync> {
@@ -88,24 +82,10 @@ pub fn generate_streaming_agg_rules(
     )) as _
 }
 
-#[cfg(not(feature = "enterprise"))]
-pub struct StreamingAggregationContext {}
-
-#[cfg(not(feature = "enterprise"))]
-impl StreamingAggregationContext {
-    pub async fn new(
-        _request: &Request,
-        _is_complete_cache_hit: Arc<Mutex<bool>>,
-    ) -> Result<Option<Self>, Error> {
-        Ok(None)
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    #[cfg(not(feature = "enterprise"))]
     #[tokio::test]
     async fn test_streaming_aggregation_context_new_returns_none() {
         let request = Request::default();
@@ -121,7 +101,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg(not(feature = "enterprise"))]
     fn test_physical_optimizer_context_streaming_aggregation_none() {
         let ctx = PhysicalOptimizerContext::StreamingAggregation(None);
         assert!(matches!(

src/search/src/datafusion/optimizer/eliminate_aggregate.rs

@@ -0,0 +1,65 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use datafusion::{
+    common::{Result, tree_node::TreeNode},
+    config::ConfigOptions,
+    physical_optimizer::PhysicalOptimizerRule,
+    physical_plan::{ExecutionPlan, ExecutionPlanProperties, empty::EmptyExec},
+};
+
+/// rewrite the plan to eliminate the aggregate plan if the streaming aggregation's output partition
+/// is 0
+#[derive(Debug, Default)]
+pub struct EliminateAggregateRule {}
+
+impl EliminateAggregateRule {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl PhysicalOptimizerRule for EliminateAggregateRule {
+    fn optimize(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        _config: &ConfigOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let is_empty_streaming_agg = plan.exists(|plan| {
+            if plan.name() == "StreamingAggsExec"
+                && plan.output_partitioning().partition_count() == 0
+            {
+                return Ok(true);
+            }
+            Ok(false)
+        })?;
+
+        if is_empty_streaming_agg {
+            return Ok(Arc::new(EmptyExec::new(plan.schema())) as _);
+        }
+
+        Ok(plan)
+    }
+
+    fn name(&self) -> &str {
+        "EliminateAggregateRule"
+    }
+
+    fn schema_check(&self) -> bool {
+        true
+    }
+}

src/search/src/datafusion/optimizer/mod.rs

@@ -40,26 +40,22 @@ use datafusion::{
 };
 use hashbrown::HashSet;
 use infra::schema::get_stream_setting_index_fields;
-#[cfg(feature = "enterprise")]
-use {
-    crate::datafusion::optimizer::context::generate_streaming_agg_rules,
-    crate::datafusion::optimizer::logical_optimizer::cipher::{
-        RewriteCipherCall, RewriteCipherKey,
-    },
-    o2_enterprise::enterprise::search::datafusion::optimizer::aggregate_topk::AggregateTopkRule,
-    o2_enterprise::enterprise::search::datafusion::optimizer::eliminate_aggregate::EliminateAggregateRule,
-};
 
+#[cfg(feature = "enterprise")]
+use crate::datafusion::optimizer::logical_optimizer::cipher::{
+    RewriteCipherCall, RewriteCipherKey,
+};
 use crate::{
     datafusion::optimizer::{
         analyze::remove_index_fields::RemoveIndexFieldsRule,
-        context::PhysicalOptimizerContext,
+        context::{PhysicalOptimizerContext, generate_streaming_agg_rules},
+        eliminate_aggregate::EliminateAggregateRule,
         logical_optimizer::{
             add_sort_and_limit::AddSortAndLimitRule, limit_join_right_side::LimitJoinRightSide,
             rewrite_histogram::RewriteHistogram,
         },
         physical_optimizer::{
-            distribute_analyze::optimize_distribute_analyze,
+            aggregate_topk::AggregateTopkRule, distribute_analyze::optimize_distribute_analyze,
             index_optimizer::LeaderIndexOptimizerRule, join_reorder::JoinReorderRule,
             remote_scan::generate_remote_scan_rules,
         },
@@ -69,8 +65,10 @@ use crate::{
 
 pub mod analyze;
 pub mod context;
+pub mod eliminate_aggregate;
 pub mod logical_optimizer;
 pub mod physical_optimizer;
+pub mod stream_aggregate;
 pub mod utils;
 
 pub fn generate_analyzer_rules(sql: &Sql) -> Vec<Arc<dyn AnalyzerRule + Send + Sync>> {
@@ -178,19 +176,12 @@ pub fn generate_physical_optimizer_rules(
                 rules.push(generate_remote_scan_rules(req, sql, context));
             }
             PhysicalOptimizerContext::AggregateTopk => {
-                #[cfg(feature = "enterprise")]
                 rules.push(Arc::new(AggregateTopkRule::new(sql.limit)));
-                #[cfg(not(feature = "enterprise"))]
-                continue;
             }
             PhysicalOptimizerContext::StreamingAggregation(context) => {
-                if let Some(_context) = context {
-                    #[cfg(feature = "enterprise")]
-                    rules.push(generate_streaming_agg_rules(_context));
-                    #[cfg(feature = "enterprise")]
+                if let Some(context) = context {
+                    rules.push(generate_streaming_agg_rules(context));
                     rules.push(Arc::new(EliminateAggregateRule::new()) as _);
-                    #[cfg(not(feature = "enterprise"))]
-                    continue;
                 }
             }
         }

src/search/src/datafusion/optimizer/physical_optimizer/aggregate_topk.rs

@@ -0,0 +1,229 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use datafusion::{
+    common::{
+        Result,
+        tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter, TreeNodeVisitor},
+    },
+    config::ConfigOptions,
+    physical_optimizer::PhysicalOptimizerRule,
+    physical_plan::{
+        ExecutionPlan,
+        aggregates::{AggregateExec, AggregateMode},
+        projection::ProjectionExec,
+        sorts::{sort::SortExec, sort_preserving_merge::SortPreservingMergeExec},
+    },
+};
+
+use crate::datafusion::{
+    distributed_plan::aggregate_topk_exec::AggregateTopkExec,
+    optimizer::physical_optimizer::utils::get_final_aggregate_plan,
+};
+
+// add remote scan to physical plan
+#[derive(Debug)]
+pub struct AggregateTopkRule {
+    limit: i64,
+}
+
+impl AggregateTopkRule {
+    pub fn new(limit: i64) -> Self {
+        Self { limit }
+    }
+}
+
+impl PhysicalOptimizerRule for AggregateTopkRule {
+    fn optimize(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        _config: &ConfigOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if self.limit <= 0 || !config::get_config().common.aggregation_topk_enabled {
+            return Ok(plan);
+        }
+
+        // check if there is no aggregate plan, return the original plan
+        let Some(final_agg_plan) = get_final_aggregate_plan(Arc::clone(&plan)) else {
+            return Ok(plan);
+        };
+
+        // check if the group by only one column
+        if final_agg_plan.group_expr().expr().len() != 1 {
+            return Ok(plan);
+        }
+
+        // check if the agg function is count
+        if final_agg_plan.aggr_expr().len() != 1 {
+            return Ok(plan);
+        }
+
+        if let Some(expr) = final_agg_plan.aggr_expr().first() {
+            if !["count", "avg", "min", "max", "sum", "approx_distinct"]
+                .contains(&expr.fun().name())
+            {
+                return Ok(plan);
+            }
+            let expr_name = expr.name();
+            let mut visitor = SortLimitVisitor::new(expr_name);
+            let _ = plan.visit(&mut visitor);
+            if visitor.is_match {
+                let mut rewriter =
+                    AggregateTopkRewriter::new(expr_name, visitor.descending, visitor.limit as u64);
+                let plan = plan.rewrite(&mut rewriter)?.data;
+                return Ok(plan);
+            }
+        }
+        Ok(plan)
+    }
+
+    fn name(&self) -> &str {
+        "AggregateTopkRule"
+    }
+
+    fn schema_check(&self) -> bool {
+        true
+    }
+}
+
+/// This rewriter is used to add a new node AggregateMergeExec in the middle of the
+/// RemoteScanExec->AggregateExec. It will get the topK records from the AggregateExec and return
+/// them to the RemoteScanExec.
+pub(crate) struct AggregateTopkRewriter {
+    field: String,
+    descending: bool,
+    limit: u64,
+}
+
+impl AggregateTopkRewriter {
+    pub(crate) fn new(field: &str, descending: bool, limit: u64) -> Self {
+        Self {
+            field: field.to_string(),
+            descending,
+            limit,
+        }
+    }
+}
+
+impl TreeNodeRewriter for AggregateTopkRewriter {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: Arc<dyn ExecutionPlan>) -> Result<Transformed<Self::Node>> {
+        // This feature need cluster mode, single node we can skip it
+        if node.children().len() == 1 && node.children().first().unwrap().name() == "AggregateExec"
+        {
+            let agg_node = node.children().first().cloned().unwrap();
+            let Some(agg_exec) = agg_node.downcast_ref::<AggregateExec>() else {
+                return Ok(Transformed::no(node));
+            };
+            if agg_exec.mode() != &AggregateMode::Partial {
+                return Ok(Transformed::no(node));
+            }
+
+            let input_plan = Arc::clone(agg_node);
+            let agg_plan =
+                AggregateTopkExec::new(input_plan, &self.field, self.descending, self.limit);
+
+            let node =
+                node.with_new_children(vec![Arc::new(agg_plan) as Arc<dyn ExecutionPlan>])?;
+            return Ok(Transformed::new(node, true, TreeNodeRecursion::Stop));
+        }
+        Ok(Transformed::no(node))
+    }
+}
+
+#[derive(Default)]
+pub(crate) struct SortLimitVisitor {
+    field: String,
+    pub(crate) limit: usize,
+    pub(crate) descending: bool,
+    pub(crate) is_match: bool,
+}
+
+impl SortLimitVisitor {
+    pub(crate) fn new(field: &str) -> Self {
+        Self {
+            field: field.to_string(),
+            ..Default::default()
+        }
+    }
+}
+
+impl<'n> TreeNodeVisitor<'n> for SortLimitVisitor {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: &'n Self::Node) -> Result<TreeNodeRecursion> {
+        if node.name() == "ProjectionExec" {
+            // we need to check if the field is map to any alias
+            let Some(expr) = node.downcast_ref::<ProjectionExec>() else {
+                return Ok(TreeNodeRecursion::Continue);
+            };
+            for projection_expr in expr.expr().iter() {
+                let expr = &projection_expr.expr;
+                let alias = &projection_expr.alias;
+                if expr
+                    .to_string()
+                    .split('@')
+                    .next()
+                    .is_some_and(|v| v == self.field)
+                {
+                    self.field = alias.clone();
+                    break;
+                }
+            }
+        } else if node.name() == "SortExec" {
+            // we need to check if the field is sort by field
+            let Some(expr) = node.downcast_ref::<SortExec>() else {
+                return Ok(TreeNodeRecursion::Continue);
+            };
+            for sort_expr in expr.expr().iter() {
+                if sort_expr
+                    .expr
+                    .to_string()
+                    .split('@')
+                    .next()
+                    .is_some_and(|v| v == self.field)
+                {
+                    self.is_match = true;
+                    self.limit = expr.fetch().unwrap_or(0);
+                    self.descending = sort_expr.options.descending;
+                    return Ok(TreeNodeRecursion::Stop);
+                }
+            }
+        } else if node.name() == "SortPreservingMergeExec" {
+            // we need to check if the field is sort by field
+            let Some(expr) = node.downcast_ref::<SortPreservingMergeExec>() else {
+                return Ok(TreeNodeRecursion::Continue);
+            };
+            for sort_expr in expr.expr().iter() {
+                if sort_expr
+                    .expr
+                    .to_string()
+                    .split('@')
+                    .next()
+                    .is_some_and(|v| v == self.field)
+                {
+                    self.is_match = true;
+                    self.limit = expr.fetch().unwrap_or(0);
+                    self.descending = sort_expr.options.descending;
+                    return Ok(TreeNodeRecursion::Stop);
+                }
+            }
+        }
+        Ok(TreeNodeRecursion::Continue)
+    }
+}

src/search/src/datafusion/optimizer/physical_optimizer/broadcast_join.rs

@@ -20,21 +20,175 @@ use config::ider::uuid;
 use datafusion::{
     common::{
         Result,
-        tree_node::{Transformed, TreeNode, TreeNodeRewriter},
+        tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter, TreeNodeVisitor},
+    },
+    physical_plan::{
+        ExecutionPlan,
+        aggregates::AggregateExec,
+        coalesce_partitions::CoalescePartitionsExec,
+        joins::{HashJoinExec, PartitionMode},
+        limit::GlobalLimitExec,
+        sorts::sort_preserving_merge::SortPreservingMergeExec,
     },
-    physical_plan::{ExecutionPlan, limit::GlobalLimitExec},
-};
-use o2_enterprise::enterprise::search::datafusion::distributed_plan::{
-    broadcast_join_exec::BroadcastJoinExec, tmp_exec::TmpExec,
 };
 
 use crate::datafusion::{
-    distributed_plan::node::RemoteScanNodes,
+    distributed_plan::{
+        broadcast_join_exec::BroadcastJoinExec, node::RemoteScanNodes, tmp_exec::TmpExec,
+    },
     optimizer::physical_optimizer::remote_scan::{
         RemoteScanRewriter, remote_scan_to_top_if_needed,
     },
 };
 
+// Check if the plan can use broadcast join.
+pub fn should_use_broadcast_join(plan: &Arc<dyn ExecutionPlan>) -> bool {
+    let mut count = 0;
+    // 1. check if only one HashJoinExec and no other multi table ExecutionPlan
+    plan.apply(|node| {
+        Ok(if node.name() == "HashJoinExec" {
+            count += 1;
+            let hash_join = node.downcast_ref::<HashJoinExec>().unwrap();
+            if *hash_join.partition_mode() != PartitionMode::CollectLeft {
+                count += 1;
+            }
+            TreeNodeRecursion::Continue
+        } else if node.name().contains("Join")
+            || node.name() == "UnionExec"
+            || node.name() == "InterleaveExec"
+            || node.name() == "RecursiveQueryExec"
+        {
+            count += 2;
+            TreeNodeRecursion::Continue
+        } else {
+            TreeNodeRecursion::Continue
+        })
+    })
+    .unwrap();
+
+    // 2. check if the left table and the right table satisfy the condition
+    let mut visitor = BroadcastJoinVisitor::new();
+    plan.visit(&mut visitor)
+        .is_ok_and(|_| visitor.use_broadcast_join && count == 1)
+}
+
+#[derive(Debug)]
+struct BroadcastJoinVisitor {
+    use_broadcast_join: bool,
+}
+
+impl BroadcastJoinVisitor {
+    fn new() -> Self {
+        BroadcastJoinVisitor {
+            use_broadcast_join: false,
+        }
+    }
+}
+
+impl<'n> TreeNodeVisitor<'n> for BroadcastJoinVisitor {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: &'n Arc<dyn ExecutionPlan>) -> Result<TreeNodeRecursion> {
+        if node.name() == "HashJoinExec" {
+            let hash_join = node.downcast_ref::<HashJoinExec>().unwrap();
+            let left = hash_join.left();
+            let right = hash_join.right();
+            if is_broadcast_left(left) && is_broadcast_right(right) {
+                self.use_broadcast_join = true;
+            }
+            return Ok(TreeNodeRecursion::Stop);
+        }
+        Ok(TreeNodeRecursion::Continue)
+    }
+}
+
+// Left table should have aggregate and limit.
+fn is_broadcast_left(left: &Arc<dyn ExecutionPlan>) -> bool {
+    let mut visitor = LeftVisitor::new();
+    left.visit(&mut visitor)
+        .is_ok_and(|_| visitor.has_aggregate && visitor.has_limit)
+}
+
+struct LeftVisitor {
+    has_aggregate: bool,
+    has_limit: bool,
+}
+
+impl LeftVisitor {
+    fn new() -> Self {
+        Self {
+            has_aggregate: false,
+            has_limit: false,
+        }
+    }
+}
+
+impl<'n> TreeNodeVisitor<'n> for LeftVisitor {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: &'n Arc<dyn ExecutionPlan>) -> Result<TreeNodeRecursion> {
+        if let Some(aggregate) = node.downcast_ref::<AggregateExec>() {
+            if aggregate.fetch().is_some() {
+                self.has_limit = true;
+            }
+            self.has_aggregate = true;
+            return Ok(TreeNodeRecursion::Continue);
+        } else if let Some(sort_merge) = node.downcast_ref::<SortPreservingMergeExec>() {
+            if sort_merge.fetch().is_some() {
+                self.has_limit = true;
+            }
+            return Ok(TreeNodeRecursion::Continue);
+        } else if let Some(partition) = node.downcast_ref::<CoalescePartitionsExec>() {
+            if partition.fetch().is_some() {
+                self.has_limit = true;
+            }
+            return Ok(TreeNodeRecursion::Continue);
+        } else if node.name() == "GlobalLimitExec" || node.name() == "DeduplicationExec" {
+            self.has_limit = true;
+            return Ok(TreeNodeRecursion::Continue);
+        }
+        Ok(TreeNodeRecursion::Continue)
+    }
+}
+
+// Right table should be table scan and filter.
+fn is_broadcast_right(right: &Arc<dyn ExecutionPlan>) -> bool {
+    let mut visitor = RightVisitor::new();
+    right
+        .visit(&mut visitor)
+        .is_ok_and(|_| visitor.is_broadcast_right)
+}
+
+struct RightVisitor {
+    is_broadcast_right: bool,
+}
+
+impl RightVisitor {
+    fn new() -> Self {
+        Self {
+            is_broadcast_right: true,
+        }
+    }
+}
+
+impl<'n> TreeNodeVisitor<'n> for RightVisitor {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: &'n Arc<dyn ExecutionPlan>) -> Result<TreeNodeRecursion> {
+        // right table should only have NewEmptyExec, FilterExec, CooperativeExec,
+        // CoalesceBatchesExec
+        if !(node.name() == "NewEmptyExec"
+            || node.name() == "FilterExec"
+            || node.name() == "CooperativeExec"
+            || node.name() == "CoalesceBatchesExec")
+        {
+            self.is_broadcast_right = false;
+            return Ok(TreeNodeRecursion::Stop);
+        }
+        Ok(TreeNodeRecursion::Continue)
+    }
+}
+
 pub fn broadcast_join_rewrite(
     plan: Arc<dyn ExecutionPlan>,
     remote_scan_nodes: Arc<RemoteScanNodes>,
@@ -124,7 +278,6 @@ mod tests {
         execution::{runtime_env::RuntimeEnvBuilder, session_state::SessionStateBuilder},
         prelude::{SessionConfig, SessionContext},
     };
-    use o2_enterprise::enterprise::search::datafusion::optimizer::broadcast_join::should_use_broadcast_join;
 
     use super::*;
     use crate::datafusion::{

src/search/src/datafusion/optimizer/physical_optimizer/join_reorder.rs

@@ -28,11 +28,10 @@ use datafusion::{
     },
 };
 
-#[cfg(feature = "enterprise")]
-use crate::datafusion::optimizer::physical_optimizer::enrichment::{
-    is_enrichment_table, should_use_enrichment_broadcast_join,
+use crate::datafusion::optimizer::physical_optimizer::{
+    enrichment::{is_enrichment_table, should_use_enrichment_broadcast_join},
+    utils::is_aggregate_exec,
 };
-use crate::datafusion::optimizer::physical_optimizer::utils::is_aggregate_exec;
 
 #[derive(Default, Debug)]
 pub struct JoinReorderRule;
@@ -67,7 +66,6 @@ fn swap_join_order(plan: Arc<dyn ExecutionPlan>) -> Result<Transformed<Arc<dyn E
         let right = hash_join.right();
 
         // If right table is enrichment table and left table is not, swap them
-        #[cfg(feature = "enterprise")]
         if config::get_config()
             .common
             .feature_enrichment_broadcast_join_enabled

src/search/src/datafusion/optimizer/physical_optimizer/mod.rs

@@ -13,10 +13,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-#[cfg(feature = "enterprise")]
+pub mod aggregate_topk;
 pub mod broadcast_join;
 pub mod distribute_analyze;
-#[cfg(feature = "enterprise")]
 pub mod enrichment;
 pub mod index;
 pub mod index_optimizer;

src/search/src/datafusion/optimizer/physical_optimizer/remote_scan.rs

@@ -34,20 +34,22 @@ use datafusion::{
 };
 use hashbrown::HashMap;
 use proto::cluster_rpc::{self, KvItem};
-#[cfg(feature = "enterprise")]
-use {
-    crate::datafusion::optimizer::physical_optimizer::broadcast_join::broadcast_join_rewrite,
-    crate::datafusion::optimizer::physical_optimizer::enrichment::enrichment_broadcast_join_rewrite,
-    crate::datafusion::optimizer::physical_optimizer::enrichment::should_use_enrichment_broadcast_join,
-    o2_enterprise::enterprise::search::datafusion::optimizer::broadcast_join::should_use_broadcast_join,
-};
 
 use crate::{
     datafusion::{
         distributed_plan::{
             empty_exec::NewEmptyExec, node::RemoteScanNodes, remote_scan_exec::RemoteScanExec,
         },
-        optimizer::{context::RemoteScanContext, utils::is_place_holder_or_empty},
+        optimizer::{
+            context::RemoteScanContext,
+            physical_optimizer::{
+                broadcast_join::{broadcast_join_rewrite, should_use_broadcast_join},
+                enrichment::{
+                    enrichment_broadcast_join_rewrite, should_use_enrichment_broadcast_join,
+                },
+            },
+            utils::is_place_holder_or_empty,
+        },
     },
     sql::Sql,
 };
@@ -160,7 +162,6 @@ impl PhysicalOptimizerRule for RemoteScanRule {
             return Ok(plan);
         }
 
-        #[cfg(feature = "enterprise")]
         if config::get_config()
             .common
             .feature_enrichment_broadcast_join_enabled
@@ -169,7 +170,6 @@ impl PhysicalOptimizerRule for RemoteScanRule {
             return enrichment_broadcast_join_rewrite(plan, self.remote_scan_nodes.clone());
         }
 
-        #[cfg(feature = "enterprise")]
         if config::get_config().common.feature_broadcast_join_enabled
             && should_use_broadcast_join(&plan)
         {

src/search/src/datafusion/optimizer/physical_optimizer/utils.rs

@@ -17,7 +17,10 @@ use std::sync::Arc;
 
 use config::{TIMESTAMP_COL_NAME, meta::inverted_index::UNKNOWN_NAME};
 use datafusion::{
-    common::{Result, tree_node::TreeNode},
+    common::{
+        Result,
+        tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor},
+    },
     error::DataFusionError,
     logical_expr::Operator,
     physical_expr::{
@@ -27,6 +30,7 @@ use datafusion::{
     },
     physical_plan::{
         ExecutionPlan,
+        aggregates::{AggregateExec, AggregateMode},
         expressions::{BinaryExpr, CastExpr, lit},
     },
     scalar::ScalarValue,
@@ -37,6 +41,39 @@ pub fn is_aggregate_exec(plan: &Arc<dyn ExecutionPlan>) -> bool {
         .unwrap_or(false)
 }
 
+/// Get the first final aggregate plan from bottom to top.
+pub(crate) fn get_final_aggregate_plan(plan: Arc<dyn ExecutionPlan>) -> Option<AggregateExec> {
+    let mut visitor = FinalAggregateVisitor::default();
+    let _ = plan.visit(&mut visitor);
+    visitor
+        .plan
+        .map(|plan| plan.downcast_ref::<AggregateExec>().unwrap().clone())
+}
+
+#[derive(Default)]
+struct FinalAggregateVisitor {
+    plan: Option<Arc<dyn ExecutionPlan>>,
+}
+
+impl<'n> TreeNodeVisitor<'n> for FinalAggregateVisitor {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: &'n Self::Node) -> Result<TreeNodeRecursion> {
+        let Some(aggregate) = node.downcast_ref::<AggregateExec>() else {
+            return Ok(TreeNodeRecursion::Continue);
+        };
+        if matches!(
+            aggregate.mode(),
+            AggregateMode::Final | AggregateMode::FinalPartitioned
+        ) {
+            self.plan = Some(node.clone());
+            Ok(TreeNodeRecursion::Stop)
+        } else {
+            Ok(TreeNodeRecursion::Continue)
+        }
+    }
+}
+
 pub fn extract_string_literal(expr: &Arc<dyn PhysicalExpr>) -> Result<String> {
     if let Some(literal) = expr.downcast_ref::<Literal>() {
         match literal.value() {

src/search/src/datafusion/optimizer/stream_aggregate.rs

@@ -0,0 +1,984 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use config::meta::search::Interval;
+use datafusion::{
+    common::{
+        DataFusionError, Result,
+        tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter},
+    },
+    config::ConfigOptions,
+    physical_optimizer::PhysicalOptimizerRule,
+    physical_plan::{
+        ExecutionPlan,
+        aggregates::{AggregateExec, AggregateMode},
+    },
+};
+use parking_lot::Mutex;
+
+use crate::{
+    cache::streaming_agg::{CacheEntry, StreamingAggsPartitionStrategy},
+    datafusion::{
+        distributed_plan::streaming_aggs_exec::{self, exec::StreamingAggsExec},
+        optimizer::physical_optimizer::utils::get_final_aggregate_plan,
+    },
+};
+
+/// Scores a cache file based on its interval and usefulness for the partition
+/// Returns 0 if file should be excluded (interval < target_interval or no overlap)
+/// Higher score = better file
+fn score_cache_file(
+    streaming_id: &str,
+    file: &CacheEntry,
+    partition_start: i64,
+    partition_end: i64,
+    target_interval: Interval,
+) -> i64 {
+    // Filter out files with interval < target_interval
+    if file.interval.get_interval_microseconds() < target_interval.get_interval_microseconds() {
+        log::debug!(
+            "[streaming_id: {}] Excluding cache file {} (interval: {:?}) - smaller than target interval: {:?}",
+            streaming_id,
+            file.file_path,
+            file.interval,
+            target_interval
+        );
+        return 0; // Exclude smaller interval files
+    }
+
+    // Calculate overlap with partition
+    let overlap_start = file.start_time.max(partition_start);
+    let overlap_end = file.end_time.min(partition_end);
+    let overlap_duration = (overlap_end - overlap_start).max(0);
+
+    if overlap_duration <= 0 {
+        return 0; // No overlap
+    }
+
+    // Prefer longer intervals (weight by interval duration)
+    let interval_weight = file.interval.get_interval_microseconds();
+
+    // Calculate usefulness percentage (how much of the file is actually needed)
+    // Similar to ResultCacheSelectionStrategy::Both
+    let file_duration = file.end_time - file.start_time;
+    let usefulness = if file_duration > 0 {
+        (overlap_duration * 100) / file_duration
+    } else {
+        0
+    };
+
+    // Combined score: interval weight * usefulness percentage
+    // This prioritizes files with longer intervals that have good overlap
+    let score = (interval_weight / 1_000_000) * usefulness; // Normalize to prevent overflow
+
+    log::debug!(
+        "[streaming_id: {}] Scoring cache file {}: interval={:?}, overlap={}μs, usefulness={}%, score={}",
+        streaming_id,
+        file.file_path,
+        file.interval,
+        overlap_duration,
+        usefulness,
+        score
+    );
+
+    score
+}
+
+/// Checks if a time range [start, end] is fully covered by existing ranges
+fn is_fully_covered(covered: &[(i64, i64)], start: i64, end: i64) -> bool {
+    for (c_start, c_end) in covered {
+        if *c_start <= start && *c_end >= end {
+            return true; // Fully covered
+        }
+    }
+    false
+}
+
+/// Merges overlapping time ranges to simplify coverage tracking
+fn merge_ranges(ranges: &mut Vec<(i64, i64)>) {
+    if ranges.len() <= 1 {
+        return;
+    }
+    ranges.sort_by_key(|r| r.0);
+    let mut merged = vec![ranges[0]];
+    for &(start, end) in &ranges[1..] {
+        let last_idx = merged.len() - 1;
+        if start <= merged[last_idx].1 {
+            // Overlapping, merge
+            merged[last_idx].1 = merged[last_idx].1.max(end);
+        } else {
+            merged.push((start, end));
+        }
+    }
+    *ranges = merged;
+}
+
+/// Checks if entire time range [start, end] is fully covered by existing ranges
+fn is_range_fully_covered(covered: &[(i64, i64)], start: i64, end: i64) -> bool {
+    // Merge ranges first to get consolidated coverage
+    let mut ranges = covered.to_vec();
+    merge_ranges(&mut ranges);
+
+    // Check if any single merged range covers [start, end]
+    ranges.iter().any(|(s, e)| *s <= start && *e >= end)
+}
+
+/// Helper function to select files from a list and track coverage
+/// Returns selected file paths and updates covered_ranges
+fn select_from_files(
+    streaming_id: &str,
+    files: Vec<CacheEntry>,
+    partition_start: i64,
+    partition_end: i64,
+    min_interval: Interval,
+    covered_ranges: &mut Vec<(i64, i64)>,
+) -> Vec<String> {
+    let mut selected_files = Vec::new();
+
+    // Score and sort files
+    let mut scored_files: Vec<(CacheEntry, i64)> = files
+        .into_iter()
+        .map(|f| {
+            let score = if min_interval == Interval::Zero
+                || f.interval.get_interval_microseconds()
+                    >= min_interval.get_interval_microseconds()
+            {
+                score_cache_file(
+                    streaming_id,
+                    &f,
+                    partition_start,
+                    partition_end,
+                    min_interval,
+                )
+            } else {
+                0
+            };
+            (f, score)
+        })
+        .filter(|(_, score)| *score > 0)
+        .collect();
+
+    // Sort by score (descending) - best files first
+    scored_files.sort_by_key(|k| std::cmp::Reverse(k.1));
+
+    // Greedy selection: pick files that cover uncovered time ranges
+    for (file, score) in scored_files {
+        let file_start = file.start_time.max(partition_start);
+        let file_end = file.end_time.min(partition_end);
+
+        // Check if this file covers any uncovered time
+        if !is_fully_covered(covered_ranges, file_start, file_end) {
+            log::debug!(
+                "[streaming_id: {}] Selected cache file {} (score={}, interval={:?}) covering [{}, {}]",
+                streaming_id,
+                file.file_path,
+                score,
+                file.interval,
+                file_start,
+                file_end
+            );
+            selected_files.push(file.file_path.clone());
+            covered_ranges.push((file_start, file_end));
+            // Merge overlapping ranges to simplify future checks
+            merge_ranges(covered_ranges);
+        } else {
+            log::debug!(
+                "[streaming_id: {}] Skipped cache file {} (score={}, interval={:?}) - time range [{}, {}] already covered",
+                streaming_id,
+                file.file_path,
+                score,
+                file.interval,
+                file_start,
+                file_end
+            );
+        }
+    }
+
+    selected_files
+}
+
+/// Selects optimal cache files eliminating overlaps and preferring longer intervals
+/// Uses a TWO-PASS greedy algorithm:
+/// Pass 1: Prefer files with target_interval or larger (eliminates overlaps with smaller intervals)
+/// Pass 2: Fill remaining gaps with smaller interval files (maximizes cache usage)
+fn select_optimal_cache_files(
+    streaming_id: &str,
+    cache_files: Vec<CacheEntry>,
+    partition_start: i64,
+    partition_end: i64,
+    target_interval: Interval,
+) -> Vec<String> {
+    if cache_files.is_empty() {
+        return vec![];
+    }
+
+    let total_files = cache_files.len();
+
+    // Partition files into preferred (>= target interval) and smaller (< target interval)
+    let (preferred_files, smaller_files): (Vec<_>, Vec<_>) =
+        cache_files.into_iter().partition(|f| {
+            f.interval.get_interval_microseconds() >= target_interval.get_interval_microseconds()
+        });
+
+    log::debug!(
+        "[streaming_id: {}] Cache file distribution for partition [{}, {}]: target_interval={:?}, preferred={}, smaller={}",
+        streaming_id,
+        partition_start,
+        partition_end,
+        target_interval,
+        preferred_files.len(),
+        smaller_files.len()
+    );
+
+    let mut selected_files = Vec::new();
+    let mut covered_ranges: Vec<(i64, i64)> = Vec::new();
+
+    // PASS 1: Select from preferred files (target interval or larger)
+    if !preferred_files.is_empty() {
+        log::debug!(
+            "[streaming_id: {}] Pass 1: Selecting from {} preferred files (interval >= {:?})",
+            streaming_id,
+            preferred_files.len(),
+            target_interval
+        );
+
+        let pass1_result = select_from_files(
+            streaming_id,
+            preferred_files,
+            partition_start,
+            partition_end,
+            target_interval,
+            &mut covered_ranges,
+        );
+
+        selected_files.extend(pass1_result.iter().cloned());
+
+        log::debug!(
+            "[streaming_id: {}] Pass 1 complete: selected {} files, coverage: {:?}",
+            streaming_id,
+            pass1_result.len(),
+            covered_ranges
+        );
+    }
+
+    // PASS 2: Fill gaps with smaller interval files if needed
+    if !smaller_files.is_empty() {
+        // Check if entire range is covered
+        if !is_range_fully_covered(&covered_ranges, partition_start, partition_end) {
+            log::info!(
+                "[streaming_id: {}] Pass 2: Gaps exist in coverage - attempting to fill with {} smaller interval files",
+                streaming_id,
+                smaller_files.len()
+            );
+
+            let pass2_result = select_from_files(
+                streaming_id,
+                smaller_files,
+                partition_start,
+                partition_end,
+                Interval::Zero, // Accept any interval for gap filling
+                &mut covered_ranges,
+            );
+
+            if !pass2_result.is_empty() {
+                log::info!(
+                    "[streaming_id: {}] Pass 2 complete: filled gaps with {} smaller interval files",
+                    streaming_id,
+                    pass2_result.len()
+                );
+                selected_files.extend(pass2_result);
+            } else {
+                log::debug!(
+                    "[streaming_id: {streaming_id}] Pass 2: No additional files needed to fill gaps"
+                );
+            }
+        } else {
+            log::debug!(
+                "[streaming_id: {streaming_id}] Pass 2 skipped: Entire range [{partition_start}, {partition_end}] already covered by preferred files"
+            );
+        }
+    }
+
+    let coverage_status = if is_range_fully_covered(&covered_ranges, partition_start, partition_end)
+    {
+        "FULLY COVERED"
+    } else {
+        "PARTIAL COVERAGE"
+    };
+
+    log::info!(
+        "[streaming_id: {}] Selected {} optimal cache files from {} total (eliminated overlaps, {} coverage) for partition [{}, {}]",
+        streaming_id,
+        selected_files.len(),
+        total_files,
+        coverage_status,
+        partition_start,
+        partition_end
+    );
+
+    selected_files
+}
+
+/// Checks if a partition [start_time, end_time] is fully cached based on the partition strategy
+fn check_partition_cached_from_strategy(
+    strategy: &StreamingAggsPartitionStrategy,
+    start_time: i64,
+    end_time: i64,
+) -> bool {
+    match strategy {
+        StreamingAggsPartitionStrategy::FullyCached { .. } => {
+            // All partitions are cached
+            true
+        }
+        StreamingAggsPartitionStrategy::Hybrid {
+            cached_partitions, ..
+        } => {
+            // Check if this partition is within any cached partition
+            // A partition is cached if it's fully contained within a cached range
+            cached_partitions
+                .iter()
+                .any(|cp| cp.start_time <= start_time && cp.end_time >= end_time)
+        }
+        StreamingAggsPartitionStrategy::NoCacheAvailable { .. } => {
+            // No cache available
+            false
+        }
+    }
+}
+
+/// Loads cache file paths from the partition strategy into GLOBAL_CACHE
+/// This ensures that cached_files will be available when StreamingAggsExec executes
+/// Uses optimal selection to eliminate overlapping files and prefer longer intervals
+fn load_cache_files_from_strategy(
+    streaming_id: &str,
+    strategy: &StreamingAggsPartitionStrategy,
+    start_time: i64,
+    end_time: i64,
+) {
+    let cache_files: Vec<String> = match strategy {
+        StreamingAggsPartitionStrategy::FullyCached { cache_files } => {
+            // For fully cached queries, determine target interval from the cache files
+            // Use the maximum interval found in the cache files as the target
+            let target_interval = cache_files
+                .iter()
+                .map(|cf| cf.interval)
+                .max_by_key(|interval| interval.get_interval_microseconds())
+                .unwrap_or(Interval::Zero);
+
+            log::debug!(
+                "[streaming_id: {streaming_id}] FullyCached query: using target_interval={target_interval:?}"
+            );
+
+            // Select optimal files eliminating overlaps
+            select_optimal_cache_files(
+                streaming_id,
+                cache_files.clone(),
+                start_time,
+                end_time,
+                target_interval,
+            )
+        }
+        StreamingAggsPartitionStrategy::Hybrid {
+            cached_partitions, ..
+        } => {
+            // Find the cached partition(s) that cover this time range
+            let matching_partitions: Vec<_> = cached_partitions
+                .iter()
+                .filter(|cp| cp.start_time <= start_time && cp.end_time >= end_time)
+                .collect();
+
+            if matching_partitions.is_empty() {
+                log::debug!(
+                    "[streaming_id: {streaming_id}] No matching cached partitions for time_range=[{start_time}, {end_time}]"
+                );
+                vec![]
+            } else {
+                // Use the interval from the matching cached partition as target
+                // If multiple partitions match, use the maximum interval
+                let target_interval = matching_partitions
+                    .iter()
+                    .map(|cp| cp.interval)
+                    .max_by_key(|interval| interval.get_interval_microseconds())
+                    .unwrap_or(Interval::Zero);
+
+                log::debug!(
+                    "[streaming_id: {streaming_id}] Hybrid query: found {} matching partitions, target_interval={:?}",
+                    matching_partitions.len(),
+                    target_interval
+                );
+
+                // Collect all cache files from matching partitions
+                let all_cache_files: Vec<CacheEntry> = matching_partitions
+                    .iter()
+                    .flat_map(|cp| cp.cache_files.iter().cloned())
+                    .collect();
+
+                // Select optimal files eliminating overlaps
+                select_optimal_cache_files(
+                    streaming_id,
+                    all_cache_files,
+                    start_time,
+                    end_time,
+                    target_interval,
+                )
+            }
+        }
+        StreamingAggsPartitionStrategy::NoCacheAvailable { .. } => {
+            // No cache files to load
+            vec![]
+        }
+    };
+
+    // Load each cache file path into GLOBAL_CACHE
+    let num_files = cache_files.len();
+
+    // Log all selected cache files before loading
+    if !cache_files.is_empty() {
+        log::debug!(
+            "[streaming_id: {streaming_id}] Selected {num_files} OPTIMAL cache files (overlaps eliminated) for time_range=[{start_time}, {end_time}]: {cache_files:?}"
+        );
+    } else {
+        log::warn!(
+            "[streaming_id: {streaming_id}] No cache files selected for time_range=[{start_time}, {end_time}] - may need to execute query"
+        );
+    }
+
+    for file_path in cache_files {
+        streaming_aggs_exec::GLOBAL_CACHE.insert(streaming_id.to_string(), file_path.clone());
+        log::debug!(
+            "[streaming_id: {streaming_id}] Loaded cache file into GLOBAL_CACHE: {file_path}"
+        );
+    }
+
+    log::info!(
+        "[streaming_id: {streaming_id}] Loaded {num_files} cache files from partition strategy for time_range=[{start_time}, {end_time}]",
+    );
+}
+
+#[derive(Debug)]
+pub struct StreamingAggsRule {
+    id: String,
+    start_time: i64,
+    end_time: i64,
+    is_complete_cache_hit: Arc<Mutex<bool>>,
+    overwrite_cache: bool,
+}
+
+impl StreamingAggsRule {
+    pub fn new(
+        id: String,
+        start_time: i64,
+        end_time: i64,
+        is_complete_cache_hit: Arc<Mutex<bool>>,
+        overwrite_cache: bool,
+    ) -> Self {
+        Self {
+            id,
+            start_time,
+            end_time,
+            is_complete_cache_hit,
+            overwrite_cache,
+        }
+    }
+}
+
+impl PhysicalOptimizerRule for StreamingAggsRule {
+    fn optimize(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        config: &ConfigOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let Some(final_agg_plan) = get_final_aggregate_plan(Arc::clone(&plan)) else {
+            return Ok(plan);
+        };
+
+        let mut rewriter = StreamingAggsRewriter::new(
+            self.id.clone(),
+            self.start_time,
+            self.end_time,
+            config.execution.target_partitions,
+            Arc::new(final_agg_plan),
+            Arc::clone(&self.is_complete_cache_hit),
+            self.overwrite_cache,
+        )?;
+        let plan = plan.rewrite(&mut rewriter)?.data;
+
+        Ok(plan)
+    }
+
+    fn name(&self) -> &str {
+        "StreamAggregateRule"
+    }
+
+    fn schema_check(&self) -> bool {
+        true
+    }
+}
+
+pub(crate) struct StreamingAggsRewriter {
+    id: String,
+    start_time: i64,
+    end_time: i64,
+    target_partitions: usize,
+    pub is_complete_cache_hit: Arc<Mutex<bool>>,
+    pub(crate) final_agg_plan: Arc<AggregateExec>,
+    overwrite_cache: bool,
+}
+
+impl StreamingAggsRewriter {
+    pub(crate) fn new(
+        id: String,
+        start_time: i64,
+        end_time: i64,
+        target_partitions: usize,
+        final_agg_plan: Arc<AggregateExec>,
+        is_complete_cache_hit: Arc<Mutex<bool>>,
+        overwrite_cache: bool,
+    ) -> Result<Self> {
+        let ret = Self {
+            id: id.clone(),
+            start_time,
+            end_time,
+            target_partitions,
+            is_complete_cache_hit,
+            final_agg_plan,
+            overwrite_cache,
+        };
+
+        // Check if this partition is fully cached using partition strategy
+        let streaming_item = streaming_aggs_exec::GLOBAL_CACHE.id_cache.get(&id);
+        let Some(item) = streaming_item else {
+            // didn't find cache for the streaming_id, skip loading cache
+            return Err(DataFusionError::Plan(format!(
+                "streaming aggregation cache not found with id: {id}"
+            )));
+        };
+
+        // Use partition strategy to determine if this partition is fully cached
+        let is_fully_cached = if let Some(strategy) = item.get_partition_strategy() {
+            let is_cached = check_partition_cached_from_strategy(&strategy, start_time, end_time);
+
+            // If cached, load the cache file paths into GLOBAL_CACHE for later retrieval
+            if is_cached {
+                load_cache_files_from_strategy(&id, &strategy, start_time, end_time);
+            }
+
+            is_cached
+        } else {
+            // No partition strategy available, assume not cached
+            false
+        };
+
+        if is_fully_cached {
+            // Get all cached files currently in GLOBAL_CACHE for this streaming_id
+            let cached_files = streaming_aggs_exec::GLOBAL_CACHE
+                .get(&id)
+                .unwrap_or_default();
+            log::info!(
+                "[streaming_id {id}] StreamingAggsRewriter: partition fully cached, time_range=[{start_time}, {end_time}], cached_files_count={}",
+                cached_files.len(),
+            );
+            *ret.is_complete_cache_hit.lock() = true;
+        } else {
+            log::info!(
+                "[streaming_id {id}] StreamingAggsRewriter: partition NOT fully cached (will execute query), time_range=[{start_time}, {end_time}]"
+            );
+        }
+
+        Ok(ret)
+    }
+}
+
+impl TreeNodeRewriter for StreamingAggsRewriter {
+    type Node = Arc<dyn ExecutionPlan>;
+
+    fn f_up(&mut self, node: Arc<dyn ExecutionPlan>) -> Result<Transformed<Self::Node>> {
+        if (node.name() == "RemoteScanExec"
+            && node.children().len() == 1
+            && node.children().first().unwrap().name() == "AggregateExec")
+            || is_single_node_aggregate(&node)
+        {
+            // get all cached files for the streaming_id(first partition -> current partition)
+            let cached_files = streaming_aggs_exec::GLOBAL_CACHE
+                .get(&self.id)
+                .unwrap_or_default();
+
+            log::info!(
+                "[streaming_id {}] StreamingAggsRewriter: cache_strategy={}, cached_batches={}",
+                self.id,
+                if *self.is_complete_cache_hit.lock() {
+                    "complete_hit"
+                } else {
+                    "miss"
+                },
+                cached_files.len()
+            );
+
+            let plan = Arc::new(StreamingAggsExec::new(
+                self.id.clone(),
+                self.start_time,
+                self.end_time,
+                cached_files,
+                node,
+                self.target_partitions,
+                *self.is_complete_cache_hit.lock(),
+                self.final_agg_plan.clone(),
+                self.overwrite_cache,
+            )) as _;
+
+            return Ok(Transformed::new(plan, true, TreeNodeRecursion::Stop));
+        }
+        Ok(Transformed::no(node))
+    }
+}
+
+fn is_single_node_aggregate(node: &Arc<dyn ExecutionPlan>) -> bool {
+    config::get_config()
+        .common
+        .feature_single_node_optimize_enabled
+        && config::cluster::LOCAL_NODE.is_single_node()
+        && node
+            .downcast_ref::<AggregateExec>()
+            .is_some_and(|agg| agg.mode() == &AggregateMode::Partial)
+}
+
+#[cfg(test)]
+mod tests {
+    use config::meta::search::Interval;
+
+    use super::*;
+    use crate::cache::streaming_agg::CacheEntry;
+
+    #[test]
+    fn test_score_cache_file_excludes_smaller_intervals() {
+        let file = CacheEntry {
+            file_path: "test_30min.arrow".to_string(),
+            start_time: 1000,
+            end_time: 2000,
+            interval: Interval::ThirtyMinutes,
+        };
+
+        // Should exclude file with 30min interval when target is 60min
+        let score = score_cache_file("test_streaming_id", &file, 1000, 2000, Interval::OneHour);
+        assert_eq!(score, 0, "Should exclude files with smaller interval");
+    }
+
+    #[test]
+    fn test_score_cache_file_accepts_matching_interval() {
+        let file = CacheEntry {
+            file_path: "test_60min.arrow".to_string(),
+            start_time: 1000,
+            end_time: 3_600_000_000 + 1000, // 1 hour later
+            interval: Interval::OneHour,
+        };
+
+        let score = score_cache_file(
+            "test_streaming_id",
+            &file,
+            1000,
+            3_600_000_000 + 1000,
+            Interval::OneHour,
+        );
+        assert!(score > 0, "Should accept files with matching interval");
+    }
+
+    #[test]
+    fn test_score_cache_file_prefers_longer_intervals() {
+        let file_30min = CacheEntry {
+            file_path: "test_30min.arrow".to_string(),
+            start_time: 1000,
+            end_time: 1_800_000_000 + 1000,
+            interval: Interval::ThirtyMinutes,
+        };
+
+        let file_60min = CacheEntry {
+            file_path: "test_60min.arrow".to_string(),
+            start_time: 1000,
+            end_time: 3_600_000_000 + 1000,
+            interval: Interval::OneHour,
+        };
+
+        let score_30 = score_cache_file(
+            "test_streaming_id",
+            &file_30min,
+            1000,
+            3_600_000_000 + 1000,
+            Interval::ThirtyMinutes,
+        );
+        let score_60 = score_cache_file(
+            "test_streaming_id",
+            &file_60min,
+            1000,
+            3_600_000_000 + 1000,
+            Interval::ThirtyMinutes,
+        );
+
+        assert!(score_60 > score_30, "Should prefer longer interval files");
+    }
+
+    #[test]
+    fn test_is_fully_covered() {
+        let covered = vec![(1000, 2000), (3000, 4000)];
+
+        // Fully covered range
+        assert!(is_fully_covered(&covered, 1200, 1800));
+
+        // Not covered range
+        assert!(!is_fully_covered(&covered, 2500, 2800));
+
+        // Partially covered range
+        assert!(!is_fully_covered(&covered, 1500, 2500));
+    }
+
+    #[test]
+    fn test_merge_ranges() {
+        let mut ranges = vec![(1000, 2000), (1500, 2500), (3000, 4000)];
+        merge_ranges(&mut ranges);
+
+        assert_eq!(ranges.len(), 2);
+        assert_eq!(ranges[0], (1000, 2500));
+        assert_eq!(ranges[1], (3000, 4000));
+    }
+
+    #[test]
+    fn test_select_optimal_cache_files_eliminates_overlaps() {
+        // Scenario: 30min and 60min files covering same time range
+        let files = vec![
+            CacheEntry {
+                file_path: "1764153000000000_1764154800000000.arrow".to_string(), /* 10:30-11:00
+                                                                                   * (30min) */
+                start_time: 1764153000000000,
+                end_time: 1764154800000000,
+                interval: Interval::ThirtyMinutes,
+            },
+            CacheEntry {
+                file_path: "1764154800000000_1764156600000000.arrow".to_string(), /* 11:00-11:30
+                                                                                   * (30min) */
+                start_time: 1764154800000000,
+                end_time: 1764156600000000,
+                interval: Interval::ThirtyMinutes,
+            },
+            CacheEntry {
+                file_path: "1764154800000000_1764158400000000.arrow".to_string(), /* 11:00-12:00
+                                                                                   * (60min) */
+                start_time: 1764154800000000,
+                end_time: 1764158400000000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        // Query for 10:30-12:00 with target interval 60min
+        let selected = select_optimal_cache_files(
+            "test_streaming_id",
+            files,
+            1764153000000000,
+            1764158400000000,
+            Interval::OneHour,
+        );
+
+        // Should select 2 files: 60min file (11:00-12:00) and 30min file for gap (10:30-11:00)
+        // The 30min file 11:00-11:30 should NOT be selected because 60min file covers it
+        assert_eq!(
+            selected.len(),
+            2,
+            "Should select 60min file + 30min for gap"
+        );
+        assert!(
+            selected
+                .iter()
+                .any(|f| f.contains("1764154800000000_1764158400000000")),
+            "Should select the 60min interval file (11:00-12:00)"
+        );
+        assert!(
+            selected
+                .iter()
+                .any(|f| f.contains("1764153000000000_1764154800000000")),
+            "Should select the 30min file for gap (10:30-11:00)"
+        );
+        assert!(
+            !selected
+                .iter()
+                .any(|f| f.contains("1764154800000000_1764156600000000")),
+            "Should NOT select the 30min file (11:00-11:30) that overlaps with 60min file"
+        );
+    }
+
+    #[test]
+    fn test_select_optimal_cache_files_no_overlap_selection() {
+        // Scenario: Multiple 60min files with no overlaps
+        let files = vec![
+            CacheEntry {
+                file_path: "1764158400000000_1764162000000000.arrow".to_string(), // 12:00-13:00
+                start_time: 1764158400000000,
+                end_time: 1764162000000000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "1764162000000000_1764165600000000.arrow".to_string(), // 13:00-14:00
+                start_time: 1764162000000000,
+                end_time: 1764165600000000,
+                interval: Interval::OneHour,
+            },
+            CacheEntry {
+                file_path: "1764165600000000_1764169200000000.arrow".to_string(), // 14:00-15:00
+                start_time: 1764165600000000,
+                end_time: 1764169200000000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        // Query for 12:00-15:00 with target interval 60min
+        let selected = select_optimal_cache_files(
+            "test_streaming_id",
+            files,
+            1764158400000000,
+            1764169200000000,
+            Interval::OneHour,
+        );
+
+        // Should select all three files as they don't overlap
+        assert_eq!(selected.len(), 3, "Should select all non-overlapping files");
+    }
+
+    #[test]
+    fn test_select_optimal_cache_files_empty_input() {
+        let files = vec![];
+        let selected = select_optimal_cache_files(
+            "test_streaming_id",
+            files,
+            1764153000000000,
+            1764158400000000,
+            Interval::OneHour,
+        );
+
+        assert_eq!(selected.len(), 0, "Should return empty for empty input");
+    }
+
+    #[test]
+    fn test_two_pass_selection_fills_gaps_with_smaller_intervals() {
+        // Scenario: Query needs 60min intervals, but has a gap that only 30min files can fill
+        // This tests the two-pass algorithm
+        let files = vec![
+            // Gap: 10:30-11:00 - only covered by 30min file
+            CacheEntry {
+                file_path: "1764153000000000_1764154800000000.arrow".to_string(), /* 10:30-11:00
+                                                                                   * (30min) */
+                start_time: 1764153000000000,
+                end_time: 1764154800000000,
+                interval: Interval::ThirtyMinutes,
+            },
+            // Main coverage: 11:00-12:00 - covered by 60min file
+            CacheEntry {
+                file_path: "1764154800000000_1764158400000000.arrow".to_string(), /* 11:00-12:00
+                                                                                   * (60min) */
+                start_time: 1764154800000000,
+                end_time: 1764158400000000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        // Query for 10:30-12:00 with target interval 60min
+        let selected = select_optimal_cache_files(
+            "test_streaming_id",
+            files,
+            1764153000000000,
+            1764158400000000,
+            Interval::OneHour,
+        );
+
+        // Should select BOTH files:
+        // Pass 1: Select 60min file (11:00-12:00)
+        // Pass 2: Fill gap with 30min file (10:30-11:00)
+        assert_eq!(
+            selected.len(),
+            2,
+            "Should select both files to cover full range"
+        );
+        assert!(
+            selected
+                .iter()
+                .any(|f| f.contains("1764154800000000_1764158400000000")),
+            "Should include 60min file"
+        );
+        assert!(
+            selected
+                .iter()
+                .any(|f| f.contains("1764153000000000_1764154800000000")),
+            "Should include 30min file to fill gap"
+        );
+    }
+
+    #[test]
+    fn test_two_pass_selection_prefers_longer_intervals_when_overlapping() {
+        // Scenario: Both 30min and 60min files cover the same range
+        // Should prefer 60min (Pass 1) and skip 30min files
+        let files = vec![
+            CacheEntry {
+                file_path: "1764154800000000_1764156600000000.arrow".to_string(), /* 11:00-11:30
+                                                                                   * (30min) */
+                start_time: 1764154800000000,
+                end_time: 1764156600000000,
+                interval: Interval::ThirtyMinutes,
+            },
+            CacheEntry {
+                file_path: "1764156600000000_1764158400000000.arrow".to_string(), /* 11:30-12:00
+                                                                                   * (30min) */
+                start_time: 1764156600000000,
+                end_time: 1764158400000000,
+                interval: Interval::ThirtyMinutes,
+            },
+            CacheEntry {
+                file_path: "1764154800000000_1764158400000000.arrow".to_string(), /* 11:00-12:00
+                                                                                   * (60min) */
+                start_time: 1764154800000000,
+                end_time: 1764158400000000,
+                interval: Interval::OneHour,
+            },
+        ];
+
+        // Query for 11:00-12:00 with target interval 60min
+        let selected = select_optimal_cache_files(
+            "test_streaming_id",
+            files,
+            1764154800000000,
+            1764158400000000,
+            Interval::OneHour,
+        );
+
+        // Should ONLY select the 60min file (Pass 1 covers everything, Pass 2 skipped)
+        assert_eq!(selected.len(), 1, "Should only select the 60min file");
+        assert!(
+            selected[0].contains("1764154800000000_1764158400000000"),
+            "Should select the 60min interval file"
+        );
+    }
+
+    #[test]
+    fn test_is_range_fully_covered() {
+        let covered = vec![(1000, 2000), (2000, 3000)]; // Adjacent ranges
+
+        // Should be fully covered after merging
+        assert!(is_range_fully_covered(&covered, 1000, 3000));
+
+        // Partial overlap - not fully covered
+        assert!(!is_range_fully_covered(&covered, 500, 1500));
+
+        // Gap in coverage
+        let covered_with_gap = vec![(1000, 2000), (3000, 4000)];
+        assert!(!is_range_fully_covered(&covered_with_gap, 1000, 4000));
+    }
+}

src/search/src/datafusion/udaf/approx_topk.rs

@@ -0,0 +1,611 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt::Formatter, sync::Arc};
+
+use arrow::{
+    array::{Array, AsArray, Int64Array, LargeStringArray, RecordBatch, StructArray},
+    datatypes::{FieldRef, Fields},
+};
+use datafusion::{
+    arrow::{
+        array::ArrayRef,
+        datatypes::{DataType, Field, Schema},
+    },
+    common::{internal_err, not_impl_err, plan_err},
+    error::Result,
+    logical_expr::{
+        Accumulator, AggregateUDFImpl, ColumnarValue, Signature, TypeSignature, Volatility,
+        function::{AccumulatorArgs, StateFieldsArgs},
+        utils::format_state_name,
+    },
+    physical_plan::PhysicalExpr,
+    scalar::ScalarValue,
+};
+use hashbrown::HashMap;
+
+const APPROX_TOPK: &str = "approx_topk";
+
+/// Approximate TopK UDAF that returns the top K elements by frequency.
+///
+/// Usage: approx_topk(field, k, [cap])
+/// - field: the field to find top k values from
+/// - k: number of top elements to return
+/// - cap: optional maximum number of candidates to keep in memory (default: max(k*4, 1000))
+///
+/// For partial aggregation, returns top k elements from each partition.
+/// For final aggregation, merges results from all partitions and returns final top k.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct ApproxTopK(Signature);
+
+impl ApproxTopK {
+    pub fn new() -> Self {
+        Self(Signature::one_of(
+            vec![
+                // String field with Int64 k
+                TypeSignature::Exact(vec![DataType::Utf8, DataType::Int64]),
+                TypeSignature::Exact(vec![DataType::LargeUtf8, DataType::Int64]),
+                // String field with Int64 k and optional Int64 cap
+                TypeSignature::Exact(vec![DataType::Utf8, DataType::Int64, DataType::Int64]),
+                TypeSignature::Exact(vec![DataType::LargeUtf8, DataType::Int64, DataType::Int64]),
+            ],
+            Volatility::Immutable,
+        ))
+    }
+}
+
+impl Default for ApproxTopK {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AggregateUDFImpl for ApproxTopK {
+    fn name(&self) -> &str {
+        APPROX_TOPK
+    }
+
+    fn signature(&self) -> &datafusion::logical_expr::Signature {
+        &self.0
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        match &arg_types[0] {
+            DataType::Utf8 | DataType::LargeUtf8 => {
+                // Return array of structs: [{value: string, count: int64}]
+                Ok(DataType::List(Arc::new(Field::new(
+                    "item",
+                    DataType::Struct(
+                        vec![
+                            Field::new("value", DataType::LargeUtf8, false),
+                            Field::new("count", DataType::Int64, false),
+                        ]
+                        .into(),
+                    ),
+                    true,
+                ))))
+            }
+            _ => plan_err!("approx_topk requires string input types"),
+        }
+    }
+
+    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
+        Ok(vec![
+            // Store values as list of strings
+            Arc::new(Field::new(
+                format_state_name(args.name, "values"),
+                DataType::List(Arc::new(Field::new("item", DataType::LargeUtf8, true))),
+                true,
+            )),
+            // Store counts as list of int64
+            Arc::new(Field::new(
+                format_state_name(args.name, "counts"),
+                DataType::List(Arc::new(Field::new("item", DataType::Int64, true))),
+                true,
+            )),
+            // Store k parameter
+            Arc::new(Field::new(
+                format_state_name(args.name, "k"),
+                DataType::Int64,
+                false,
+            )),
+        ])
+    }
+
+    fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
+        let k = validate_k_parameter(&args.exprs[1])?;
+        let cap = if args.exprs.len() > 2 {
+            Some(validate_cap_parameter(&args.exprs[2])?)
+        } else {
+            None
+        };
+        let value_data_type = args.exprs[0].data_type(args.schema)?;
+
+        match value_data_type {
+            DataType::Utf8 | DataType::LargeUtf8 => {
+                Ok(Box::new(ApproxTopKAccumulator::new(k, cap)))
+            }
+            other => {
+                not_impl_err!("Support for 'APPROX_TOPK' for data type {other} is not implemented")
+            }
+        }
+    }
+}
+
+fn validate_k_parameter(expr: &Arc<dyn PhysicalExpr>) -> Result<usize> {
+    let empty_schema = Arc::new(Schema::empty());
+    let batch = RecordBatch::new_empty(Arc::clone(&empty_schema));
+
+    let k = match expr.evaluate(&batch)? {
+        ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => {
+            if value <= 0 {
+                return plan_err!("k parameter for 'APPROX_TOPK' must be positive, got {value}");
+            }
+            value as usize
+        }
+        ColumnarValue::Scalar(other) => {
+            return not_impl_err!(
+                "k parameter for 'APPROX_TOPK' must be Int64 literal (got {:?})",
+                other.data_type()
+            );
+        }
+        _ => {
+            return internal_err!("Expected scalar value for k parameter");
+        }
+    };
+
+    Ok(k)
+}
+
+fn validate_cap_parameter(expr: &Arc<dyn PhysicalExpr>) -> Result<usize> {
+    let empty_schema = Arc::new(Schema::empty());
+    let batch = RecordBatch::new_empty(Arc::clone(&empty_schema));
+
+    let cap = match expr.evaluate(&batch)? {
+        ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => {
+            if value <= 0 {
+                return plan_err!("cap parameter for 'APPROX_TOPK' must be positive, got {value}");
+            }
+            value as usize
+        }
+        ColumnarValue::Scalar(other) => {
+            return not_impl_err!(
+                "cap parameter for 'APPROX_TOPK' must be Int64 literal (got {:?})",
+                other.data_type()
+            );
+        }
+        _ => {
+            return internal_err!("Expected scalar value for cap parameter");
+        }
+    };
+
+    Ok(cap)
+}
+
+/// Memory-efficient accumulator that only tracks top-K candidates
+/// Uses a min-heap to maintain only the most frequent items
+struct ApproxTopKAccumulator {
+    // Only keep track of top candidates - LIMITED SIZE!
+    candidates: HashMap<String, i64>,
+    k: usize,
+    // Memory management
+    max_candidates: usize,    // Maximum candidates to keep in memory
+    min_count_threshold: i64, // Minimum count to be considered
+}
+
+impl std::fmt::Debug for ApproxTopKAccumulator {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "ApproxTopKAccumulator(k={}, candidates={})",
+            self.k,
+            self.candidates.len()
+        )
+    }
+}
+
+impl ApproxTopKAccumulator {
+    fn new(k: usize, max_candidates: Option<usize>) -> Self {
+        // Cap at least k*2 for safety
+        let default_max = (k * 4).max(1000);
+        let max_candidates = max_candidates.unwrap_or(default_max);
+        Self {
+            candidates: HashMap::with_capacity(max_candidates),
+            k,
+            max_candidates,
+            min_count_threshold: 0,
+        }
+    }
+
+    /// Memory-efficient update that only keeps top candidates
+    fn update_with_pruning(&mut self, value: String, count: i64) {
+        // Periodically prune low-frequency items to save memory
+        if self.candidates.len() >= self.max_candidates {
+            self.prune_low_frequency_items();
+        }
+
+        // Update count
+        let entry = self.candidates.entry(value).or_insert(0);
+        *entry += count;
+    }
+
+    /// Remove low-frequency items to keep memory usage bounded
+    fn prune_low_frequency_items(&mut self) {
+        let target_size = (self.max_candidates / 2).max(self.k);
+
+        // First, remove items below the minimum threshold
+        self.candidates
+            .retain(|_, count| *count >= self.min_count_threshold);
+
+        if self.candidates.len() <= target_size {
+            return; // No need to prune
+        }
+
+        // Collect items with their counts
+        let mut items = self
+            .candidates
+            .iter()
+            .map(|(k, v)| (k, *v))
+            .collect::<Vec<_>>();
+
+        // Sort by count descending, then by key for deterministic results
+        items.sort_by_key(|k| std::cmp::Reverse(k.1));
+
+        // Update minimum threshold to the lowest count we're keeping
+        let mut item_iter = items.into_iter().skip(target_size - 1);
+        if let Some((_, count)) = item_iter.next() {
+            self.min_count_threshold = self.min_count_threshold.max(count);
+        }
+
+        // Keep only the top target_size items
+        let removed_items = item_iter.map(|(k, _)| k.clone()).collect::<Vec<_>>();
+        for key in removed_items {
+            self.candidates.remove(&key);
+        }
+    }
+
+    /// Get the top k elements as (value, count) pairs sorted by count descending
+    fn get_top_k(&self, n: usize) -> Vec<(String, i64)> {
+        let mut items: Vec<_> = self
+            .candidates
+            .iter()
+            .map(|(v, c)| (v.clone(), *c))
+            .collect();
+
+        // Sort by count descending, then by value ascending for deterministic results
+        items.sort_by_key(|k| std::cmp::Reverse(k.1));
+
+        items.into_iter().take(n).collect()
+    }
+
+    /// Convert string array to vector of strings
+    fn convert_to_strings(values: &ArrayRef) -> Result<Vec<String>> {
+        match values.data_type() {
+            DataType::Utf8 => {
+                let array = values.as_string::<i32>();
+                Ok(array
+                    .iter()
+                    .map(|v| v.unwrap_or_default().to_string())
+                    .collect())
+            }
+            DataType::LargeUtf8 => {
+                let array = values.as_string::<i64>();
+                Ok(array
+                    .iter()
+                    .map(|v| v.unwrap_or_default().to_string())
+                    .collect())
+            }
+            other => {
+                internal_err!("APPROX_TOPK received unexpected type {other:?}")
+            }
+        }
+    }
+
+    /// Convert int64 array to vector of counts
+    fn convert_to_counts(values: &ArrayRef) -> Result<Vec<i64>> {
+        let array = values.as_primitive::<arrow::datatypes::Int64Type>();
+        Ok(array.iter().map(|v| v.unwrap_or_default()).collect())
+    }
+}
+
+impl Accumulator for ApproxTopKAccumulator {
+    fn state(&mut self) -> Result<Vec<ScalarValue>> {
+        let pairs: Vec<_> = self.get_top_k(self.max_candidates);
+
+        let values = ScalarValue::List(ScalarValue::new_list_nullable(
+            &pairs
+                .iter()
+                .map(|(v, _)| ScalarValue::LargeUtf8(Some(v.to_string())))
+                .collect::<Vec<ScalarValue>>(),
+            &DataType::LargeUtf8,
+        ));
+
+        let counts = ScalarValue::List(ScalarValue::new_list_nullable(
+            &pairs
+                .iter()
+                .map(|(_, c)| ScalarValue::Int64(Some(*c)))
+                .collect::<Vec<ScalarValue>>(),
+            &DataType::Int64,
+        ));
+
+        let k_scalar = ScalarValue::Int64(Some(self.k as i64));
+
+        Ok(vec![values, counts, k_scalar])
+    }
+
+    fn evaluate(&mut self) -> Result<ScalarValue> {
+        let top_k = self.get_top_k(self.k);
+
+        if top_k.is_empty() {
+            return Ok(ScalarValue::List(ScalarValue::new_list_nullable(
+                &[],
+                &DataType::Struct(
+                    vec![
+                        Field::new("value", DataType::LargeUtf8, false),
+                        Field::new("count", DataType::Int64, false),
+                    ]
+                    .into(),
+                ),
+            )));
+        }
+
+        let values: Vec<Option<String>> = top_k.iter().map(|(v, _)| Some(v.clone())).collect();
+        let counts: Vec<Option<i64>> = top_k.iter().map(|(_, c)| Some(*c)).collect();
+
+        let value_array = Arc::new(LargeStringArray::from(values));
+        let count_array = Arc::new(Int64Array::from(counts));
+
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("value", DataType::LargeUtf8, false),
+                Field::new("count", DataType::Int64, false),
+            ]),
+            vec![value_array as ArrayRef, count_array as ArrayRef],
+            None,
+        );
+
+        Ok(ScalarValue::List(ScalarValue::new_list_nullable(
+            &top_k
+                .into_iter()
+                .enumerate()
+                .map(|(i, _)| ScalarValue::Struct(Arc::new(struct_array.slice(i, 1))))
+                .collect::<Vec<ScalarValue>>(),
+            &DataType::Struct(
+                vec![
+                    Field::new("value", DataType::LargeUtf8, false),
+                    Field::new("count", DataType::Int64, false),
+                ]
+                .into(),
+            ),
+        )))
+    }
+
+    fn size(&self) -> usize {
+        // Estimate memory usage:
+        // - HashMap overhead + String keys + i64 values
+        // - Average string length ~20 bytes + HashMap overhead ~40 bytes per entry
+        self.candidates.len() * 60
+    }
+
+    fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
+        let strings = Self::convert_to_strings(&values[0])?;
+
+        // Count each string value
+        for value in strings {
+            self.update_with_pruning(value, 1);
+        }
+
+        Ok(())
+    }
+
+    fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
+        if states.is_empty() {
+            return Ok(());
+        }
+
+        let values_list = states[0].as_list::<i32>();
+        let counts_list = states[1].as_list::<i32>();
+
+        for (values_opt, counts_opt) in values_list.iter().zip(counts_list.iter()) {
+            if let (Some(values_array), Some(counts_array)) = (values_opt, counts_opt) {
+                let values = Self::convert_to_strings(&values_array)?;
+                let counts = Self::convert_to_counts(&counts_array)?;
+
+                // Merge the counts from this state
+                for (value, count) in values.into_iter().zip(counts) {
+                    self.update_with_pruning(value, count);
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow::array::StringArray;
+    use datafusion::{datasource::MemTable, logical_expr::AggregateUDF, prelude::SessionContext};
+
+    use super::*;
+
+    #[test]
+    fn test_approx_topk_accumulator() {
+        let mut acc = ApproxTopKAccumulator::new(3, None);
+
+        // Add some test data
+        let values = vec!["apple", "banana", "apple", "cherry", "banana", "apple"];
+        let string_array: ArrayRef = Arc::new(StringArray::from(values));
+
+        acc.update_batch(&[string_array]).unwrap();
+
+        // Evaluate should return top 3 by frequency
+        let result = acc.evaluate().unwrap();
+
+        // apple: 3, banana: 2, cherry: 1
+        assert!(matches!(result, ScalarValue::List(_)));
+    }
+
+    #[test]
+    fn test_memory_efficient_pruning() {
+        // Test that the accumulator prunes low-frequency items to save memory
+        let mut acc = ApproxTopKAccumulator::new(3, Some(20)); // Small limit for testing
+
+        // Add many different items with low frequency
+        for i in 0..100 {
+            let item = format!("low_freq_{i}");
+            let array: ArrayRef = Arc::new(StringArray::from(vec![item.as_str()]));
+            acc.update_batch(&[array]).unwrap();
+        }
+
+        // Should have pruned significantly
+        assert!(
+            acc.candidates.len() < 100,
+            "Should prune low-frequency items"
+        );
+
+        // Add some high-frequency items
+        for _ in 0..15 {
+            let array: ArrayRef = Arc::new(StringArray::from(vec!["very_frequent"]));
+            acc.update_batch(&[array]).unwrap();
+        }
+
+        for _ in 0..8 {
+            let array: ArrayRef = Arc::new(StringArray::from(vec!["medium_frequent"]));
+            acc.update_batch(&[array]).unwrap();
+        }
+
+        // Get top results
+        let top_k = acc.get_top_k(acc.k);
+        assert!(!top_k.is_empty());
+
+        // Most frequent items should be at the top
+        assert_eq!(top_k[0].0, "very_frequent");
+        assert_eq!(top_k[0].1, 15);
+
+        if top_k.len() > 1 {
+            assert_eq!(top_k[1].0, "medium_frequent");
+            assert_eq!(top_k[1].1, 8);
+        }
+
+        // Memory usage should be reasonable
+        assert!(acc.candidates.len() <= 50, "Memory usage should be bounded");
+    }
+
+    #[test]
+    fn test_accumulator_with_explicit_cap() {
+        // Test that the accumulator respects explicit cap parameter
+        let mut acc = ApproxTopKAccumulator::new(3, Some(5)); // Very small cap for testing
+
+        // Add items that would exceed the cap
+        let items = vec!["a", "b", "c", "d", "e", "f", "g", "h"];
+        for item in items {
+            let array: ArrayRef = Arc::new(StringArray::from(vec![item]));
+            acc.update_batch(&[array]).unwrap();
+        }
+
+        // Should respect the cap limit
+        assert!(
+            acc.candidates.len() <= 5,
+            "Should respect explicit cap parameter, got {} candidates",
+            acc.candidates.len()
+        );
+
+        // Add some items multiple times to create frequency differences
+        for _ in 0..10 {
+            let array: ArrayRef = Arc::new(StringArray::from(vec!["frequent"]));
+            acc.update_batch(&[array]).unwrap();
+        }
+
+        // Get results
+        let top_k = acc.get_top_k(acc.k);
+        assert!(!top_k.is_empty());
+
+        // Most frequent item should be at the top
+        assert_eq!(top_k[0].0, "frequent");
+        // Due to pruning with very small cap, count might be slightly less than 10
+        assert!(top_k[0].1 >= 9, "Expected count >= 9, got {}", top_k[0].1);
+    }
+
+    #[tokio::test]
+    async fn test_approx_topk_udaf() {
+        let ctx = SessionContext::new();
+
+        // Create test data
+        let schema = Schema::new(vec![Field::new("item", DataType::Utf8, false)]);
+
+        let values = vec![
+            "apple", "banana", "apple", "cherry", "banana", "apple", "date",
+        ];
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(StringArray::from(values))],
+        )
+        .unwrap();
+
+        let table = MemTable::try_new(Arc::new(schema), vec![vec![batch]]).unwrap();
+        ctx.register_table("test_table", Arc::new(table)).unwrap();
+
+        // Register the UDAF
+        let topk_udaf = AggregateUDF::from(ApproxTopK::new());
+        ctx.register_udaf(topk_udaf);
+
+        // Test the function
+        let df = ctx
+            .sql("SELECT approx_topk(item, 2) as top_items FROM test_table")
+            .await
+            .unwrap();
+        let results = df.collect().await.unwrap();
+
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].num_columns(), 1);
+        assert_eq!(results[0].num_rows(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_approx_topk_udaf_with_cap() {
+        let ctx = SessionContext::new();
+
+        // Create test data
+        let schema = Schema::new(vec![Field::new("item", DataType::Utf8, false)]);
+
+        let values = vec![
+            "apple", "banana", "apple", "cherry", "banana", "apple", "date",
+        ];
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![Arc::new(StringArray::from(values))],
+        )
+        .unwrap();
+
+        let table = MemTable::try_new(Arc::new(schema), vec![vec![batch]]).unwrap();
+        ctx.register_table("test_table", Arc::new(table)).unwrap();
+
+        // Register the UDAF
+        let topk_udaf = AggregateUDF::from(ApproxTopK::new());
+        ctx.register_udaf(topk_udaf);
+
+        // Test the function with cap parameter
+        let df = ctx
+            .sql("SELECT approx_topk(item, 2, 10) as top_items FROM test_table")
+            .await
+            .unwrap();
+        let results = df.collect().await.unwrap();
+
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].num_columns(), 1);
+        assert_eq!(results[0].num_rows(), 1);
+    }
+}

src/search/src/datafusion/udaf/approx_topk_distinct.rs

@@ -0,0 +1,715 @@
+// Copyright 2026 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt::Formatter, sync::Arc};
+
+use arrow::{
+    array::{Array, AsArray, BinaryArray, LargeStringArray, RecordBatch, StructArray, UInt64Array},
+    datatypes::Fields,
+};
+use datafusion::{
+    arrow::{
+        array::ArrayRef,
+        datatypes::{DataType, Field, FieldRef, Schema},
+    },
+    common::{internal_err, not_impl_err, plan_err},
+    error::Result,
+    functions_aggregate::approx_distinct::ApproxDistinct,
+    logical_expr::{
+        Accumulator, AggregateUDFImpl, ColumnarValue, Signature, TypeSignature, Volatility,
+        function::{AccumulatorArgs, StateFieldsArgs},
+        utils::format_state_name,
+    },
+    physical_plan::{PhysicalExpr, expressions::col},
+    scalar::ScalarValue,
+};
+use hashbrown::HashMap;
+
+const APPROX_TOPK_DISTINCT: &str = "approx_topk_distinct";
+
+/// Approximate TopK UDAF that returns the top K elements by distinct count of another field.
+///
+/// Usage: approx_topk_distinct(top_field, value_field, k, [cap])
+/// - top_field: the field to find top k values from
+/// - value_field: the field to count distinct values for
+/// - k: number of top elements to return
+/// - cap: optional maximum number of candidates to keep in memory (default: max(k*4, 1000))
+///
+/// This function finds the top K values in top_field, ranked by how many unique values
+/// they have in the corresponding value_field. For example:
+/// - If you have data with (user_id, session_id) pairs
+/// - approx_topk_distinct(user_id, session_id, 10) returns the top 10 users with the most distinct
+///   sessions
+///
+/// Uses HyperLogLog for exact distinct counting (memory-efficient for reasonable cardinalities).
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct ApproxTopKDistinct(Signature);
+
+impl ApproxTopKDistinct {
+    pub fn new() -> Self {
+        Self(Signature::one_of(
+            vec![
+                // top_field, value_field, k
+                TypeSignature::Exact(vec![DataType::Utf8, DataType::Utf8, DataType::Int64]),
+                TypeSignature::Exact(vec![DataType::LargeUtf8, DataType::Utf8, DataType::Int64]),
+                TypeSignature::Exact(vec![DataType::Utf8, DataType::LargeUtf8, DataType::Int64]),
+                TypeSignature::Exact(vec![
+                    DataType::LargeUtf8,
+                    DataType::LargeUtf8,
+                    DataType::Int64,
+                ]),
+                // top_field, value_field, k, cap
+                TypeSignature::Exact(vec![
+                    DataType::Utf8,
+                    DataType::Utf8,
+                    DataType::Int64,
+                    DataType::Int64,
+                ]),
+                TypeSignature::Exact(vec![
+                    DataType::LargeUtf8,
+                    DataType::Utf8,
+                    DataType::Int64,
+                    DataType::Int64,
+                ]),
+                TypeSignature::Exact(vec![
+                    DataType::Utf8,
+                    DataType::LargeUtf8,
+                    DataType::Int64,
+                    DataType::Int64,
+                ]),
+                TypeSignature::Exact(vec![
+                    DataType::LargeUtf8,
+                    DataType::LargeUtf8,
+                    DataType::Int64,
+                    DataType::Int64,
+                ]),
+            ],
+            Volatility::Immutable,
+        ))
+    }
+}
+
+impl Default for ApproxTopKDistinct {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AggregateUDFImpl for ApproxTopKDistinct {
+    fn name(&self) -> &str {
+        APPROX_TOPK_DISTINCT
+    }
+
+    fn signature(&self) -> &datafusion::logical_expr::Signature {
+        &self.0
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        match &arg_types[0] {
+            DataType::Utf8 | DataType::LargeUtf8 => {
+                // Return array of structs: [{value: string, count: int64}]
+                Ok(DataType::List(Arc::new(Field::new(
+                    "item",
+                    DataType::Struct(
+                        vec![
+                            Field::new("value", DataType::LargeUtf8, false),
+                            Field::new("count", DataType::UInt64, false),
+                        ]
+                        .into(),
+                    ),
+                    true,
+                ))))
+            }
+            _ => plan_err!("approx_topk_distinct requires string input types"),
+        }
+    }
+
+    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
+        Ok(vec![
+            // Store top_field values as list of strings
+            Arc::new(Field::new(
+                format_state_name(args.name, "top_values"),
+                DataType::List(Arc::new(Field::new("item", DataType::LargeUtf8, true))),
+                true,
+            )),
+            // Store distinct values as list of list of HLL registers
+            Arc::new(Field::new(
+                format_state_name(args.name, "hll_registers"),
+                DataType::List(Arc::new(Field::new("item", DataType::LargeBinary, true))),
+                true,
+            )),
+            // Store k parameter
+            Arc::new(Field::new(
+                format_state_name(args.name, "k"),
+                DataType::Int64,
+                false,
+            )),
+        ])
+    }
+
+    fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
+        let k = validate_k_parameter(&args.exprs[2])?;
+        let cap = if args.exprs.len() > 3 {
+            Some(validate_cap_parameter(&args.exprs[3])?)
+        } else {
+            None
+        };
+
+        let top_field_data_type = args.exprs[0].data_type(args.schema)?;
+        let value_field_data_type = args.exprs[1].data_type(args.schema)?;
+
+        match (&top_field_data_type, &value_field_data_type) {
+            (DataType::Utf8 | DataType::LargeUtf8, DataType::Utf8 | DataType::LargeUtf8) => {
+                Ok(Box::new(ApproxTopKDistinctAccumulator::new(k, cap)))
+            }
+            (other_top, other_value) => {
+                not_impl_err!(
+                    "Support for 'APPROX_TOPK_DISTINCT' for data types {other_top:?}, {other_value:?} is not implemented"
+                )
+            }
+        }
+    }
+}
+
+fn validate_k_parameter(expr: &Arc<dyn PhysicalExpr>) -> Result<usize> {
+    let empty_schema = Arc::new(Schema::empty());
+    let batch = RecordBatch::new_empty(Arc::clone(&empty_schema));
+
+    let k = match expr.evaluate(&batch)? {
+        ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => {
+            if value <= 0 {
+                return plan_err!(
+                    "k parameter for 'APPROX_TOPK_DISTINCT' must be positive, got {value}"
+                );
+            }
+            value as usize
+        }
+        ColumnarValue::Scalar(other) => {
+            return not_impl_err!(
+                "k parameter for 'APPROX_TOPK_DISTINCT' must be Int64 literal (got {:?})",
+                other.data_type()
+            );
+        }
+        _ => {
+            return internal_err!("Expected scalar value for k parameter");
+        }
+    };
+
+    Ok(k)
+}
+
+fn validate_cap_parameter(expr: &Arc<dyn PhysicalExpr>) -> Result<usize> {
+    let empty_schema = Arc::new(Schema::empty());
+    let batch = RecordBatch::new_empty(Arc::clone(&empty_schema));
+
+    let cap = match expr.evaluate(&batch)? {
+        ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => {
+            if value <= 0 {
+                return plan_err!(
+                    "cap parameter for 'APPROX_TOPK_DISTINCT' must be positive, got {value}"
+                );
+            }
+            value as usize
+        }
+        ColumnarValue::Scalar(other) => {
+            return not_impl_err!(
+                "cap parameter for 'APPROX_TOPK_DISTINCT' must be Int64 literal (got {:?})",
+                other.data_type()
+            );
+        }
+        _ => {
+            return internal_err!("Expected scalar value for cap parameter");
+        }
+    };
+
+    Ok(cap)
+}
+
+/// Accumulator that tracks top K values by distinct count of another field
+/// Uses HyperLogLog for exact distinct counting (good for reasonable cardinalities)
+struct ApproxTopKDistinctAccumulator {
+    // Map from top_field value to HyperLogLog accumulator
+    candidates: HashMap<String, Box<dyn Accumulator>>,
+    k: usize,
+    // Memory management
+    max_candidates: usize,    // Maximum candidates to keep in memory
+    min_count_threshold: u64, // Minimum count to be considered
+}
+
+impl std::fmt::Debug for ApproxTopKDistinctAccumulator {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "ApproxTopKDistinctAccumulator(k={}, candidates={})",
+            self.k,
+            self.candidates.len()
+        )
+    }
+}
+
+impl ApproxTopKDistinctAccumulator {
+    fn new(k: usize, max_candidates: Option<usize>) -> Self {
+        // Cap at least k*4 for safety
+        let default_max = (k * 4).max(1000);
+        let max_candidates = max_candidates.unwrap_or(default_max);
+        Self {
+            candidates: HashMap::with_capacity(max_candidates),
+            k,
+            max_candidates,
+            min_count_threshold: 0,
+        }
+    }
+
+    fn new_acc_args() -> Option<Box<dyn Accumulator>> {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "f",
+            DataType::LargeUtf8,
+            true,
+        )]));
+        let acc_args = AccumulatorArgs {
+            return_field: Arc::new(Field::new("f", DataType::UInt64, true)),
+            schema: &schema,
+            ignore_nulls: false,
+            order_bys: &[],
+            is_reversed: false,
+            name: "APPROX_DISTINCT(f)",
+            is_distinct: false,
+            exprs: &[col("f", &schema).unwrap()],
+            expr_fields: &[Arc::new(Field::new("f", DataType::LargeUtf8, true))],
+        };
+        ApproxDistinct::new().accumulator(acc_args).ok()
+    }
+
+    /// Memory-efficient update that only keeps top candidates
+    fn update_with_pruning(&mut self, value: String, distinct_values: Vec<String>) {
+        // Periodically prune low-frequency items to save memory
+        if self.candidates.len() >= self.max_candidates {
+            self.prune_low_frequency_items();
+        }
+
+        // Update count
+        self.candidates
+            .entry(value)
+            .or_insert_with(|| Self::new_acc_args().unwrap())
+            .update_batch(&[Arc::new(LargeStringArray::from(distinct_values))])
+            .unwrap();
+    }
+
+    /// Remove low-frequency items to keep memory usage bounded
+    fn prune_low_frequency_items(&mut self) {
+        let target_size = (self.max_candidates / 2).max(self.k);
+
+        // Collect items with their counts
+        let mut items = self
+            .candidates
+            .iter_mut()
+            .map(|(k, v)| (k, Self::get_distinct_count(v)))
+            .collect::<Vec<_>>();
+
+        // Sort by count descending, then by key for deterministic results
+        items.sort_by_key(|k| std::cmp::Reverse(k.1));
+
+        // Update minimum threshold to the lowest count we're keeping
+        let mut item_iter = items.into_iter().skip(target_size - 1);
+        if let Some((_, count)) = item_iter.next() {
+            self.min_count_threshold = self.min_count_threshold.max(count);
+        }
+
+        // Keep only the top target_size items
+        let removed_items = item_iter.map(|(k, _)| k.clone()).collect::<Vec<_>>();
+        for key in removed_items {
+            self.candidates.remove(&key);
+        }
+    }
+
+    /// Get top k entries by distinct count
+    fn get_top_k(&mut self, n: usize) -> Vec<(String, u64)> {
+        let mut items: Vec<(String, u64)> = self
+            .candidates
+            .iter_mut()
+            .map(|(top_value, acc)| (top_value.clone(), Self::get_distinct_count(acc)))
+            .collect();
+
+        // Sort by distinct count descending, then by value ascending for deterministic results
+        items.sort_by_key(|k| std::cmp::Reverse(k.1));
+
+        items.into_iter().take(n).collect()
+    }
+
+    /// Get distinct count from a ApproxDistinct accumulator
+    fn get_distinct_count(distinct_acc: &mut Box<dyn Accumulator>) -> u64 {
+        distinct_acc
+            .evaluate()
+            .map(|v| {
+                if let ScalarValue::UInt64(Some(count)) = v {
+                    count
+                } else {
+                    0
+                }
+            })
+            .ok()
+            .unwrap_or(0)
+    }
+
+    /// Convert string array to vector of strings
+    fn convert_to_strings(values: &ArrayRef) -> Result<Vec<String>> {
+        match values.data_type() {
+            DataType::Utf8 => {
+                let array = values.as_string::<i32>();
+                Ok(array
+                    .iter()
+                    .map(|v| v.unwrap_or_default().to_string())
+                    .collect())
+            }
+            DataType::LargeUtf8 => {
+                let array = values.as_string::<i64>();
+                Ok(array
+                    .iter()
+                    .map(|v| v.unwrap_or_default().to_string())
+                    .collect())
+            }
+            other => {
+                internal_err!("APPROX_TOPK_DISTINCT received unexpected type {other:?}")
+            }
+        }
+    }
+
+    /// Convert string array to vector of binary arrays
+    fn convert_to_binary(values: &ArrayRef) -> Result<Vec<Vec<u8>>> {
+        match values.data_type() {
+            DataType::Binary => {
+                let array = values.as_binary::<i32>();
+                Ok(array
+                    .iter()
+                    .map(|v| v.unwrap_or_default().to_vec())
+                    .collect::<Vec<_>>())
+            }
+            DataType::LargeBinary => {
+                let array = values.as_binary::<i64>();
+                Ok(array
+                    .iter()
+                    .map(|v| v.unwrap_or_default().to_vec())
+                    .collect::<Vec<_>>())
+            }
+            other => {
+                internal_err!("APPROX_TOPK_DISTINCT received unexpected type {other:?}")
+            }
+        }
+    }
+
+    fn convert_to_large_binary(value: ScalarValue) -> Result<ScalarValue> {
+        match value {
+            ScalarValue::Binary(v) => Ok(ScalarValue::LargeBinary(v)),
+            ScalarValue::LargeBinary(_) => Ok(value),
+            other => internal_err!("APPROX_TOPK_DISTINCT received unexpected type {other:?}"),
+        }
+    }
+}
+
+impl Accumulator for ApproxTopKDistinctAccumulator {
+    fn state(&mut self) -> Result<Vec<ScalarValue>> {
+        // Get top entries for state serialization
+        let top_entries = self.get_top_k(self.max_candidates);
+
+        let values: Vec<ScalarValue> = top_entries
+            .iter()
+            .map(|(v, _)| ScalarValue::LargeUtf8(Some(v.clone())))
+            .collect();
+
+        // Serialize HyperLogLog accumulators as lists of binary arrays
+        let distinct_values: Vec<ScalarValue> = top_entries
+            .iter()
+            .filter_map(|(top_val, _)| {
+                if let Some(acc) = self.candidates.get_mut(top_val) {
+                    acc.state()
+                        .ok()
+                        .and_then(|mut v| v.pop().map(|v| Self::convert_to_large_binary(v).ok()))
+                        .flatten()
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        let values_list = ScalarValue::List(ScalarValue::new_list_nullable(
+            &values,
+            &DataType::LargeUtf8,
+        ));
+
+        let distinct_values_list = ScalarValue::List(ScalarValue::new_list_nullable(
+            &distinct_values,
+            &DataType::LargeBinary,
+        ));
+
+        let k_scalar = ScalarValue::Int64(Some(self.k as i64));
+
+        Ok(vec![values_list, distinct_values_list, k_scalar])
+    }
+
+    fn evaluate(&mut self) -> Result<ScalarValue> {
+        let top_k = self.get_top_k(self.k);
+
+        if top_k.is_empty() {
+            return Ok(ScalarValue::List(ScalarValue::new_list_nullable(
+                &[],
+                &DataType::Struct(
+                    vec![
+                        Field::new("value", DataType::LargeUtf8, false),
+                        Field::new("count", DataType::UInt64, false),
+                    ]
+                    .into(),
+                ),
+            )));
+        }
+
+        let values: Vec<Option<String>> = top_k.iter().map(|(v, _)| Some(v.clone())).collect();
+        let counts: Vec<Option<u64>> = top_k.iter().map(|(_, c)| Some(*c)).collect();
+
+        let value_array = Arc::new(LargeStringArray::from(values));
+        let count_array = Arc::new(UInt64Array::from(counts));
+
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("value", DataType::LargeUtf8, false),
+                Field::new("count", DataType::UInt64, false),
+            ]),
+            vec![value_array as ArrayRef, count_array as ArrayRef],
+            None,
+        );
+
+        Ok(ScalarValue::List(ScalarValue::new_list_nullable(
+            &top_k
+                .into_iter()
+                .enumerate()
+                .map(|(i, _)| ScalarValue::Struct(Arc::new(struct_array.slice(i, 1))))
+                .collect::<Vec<ScalarValue>>(),
+            &DataType::Struct(
+                vec![
+                    Field::new("value", DataType::LargeUtf8, false),
+                    Field::new("count", DataType::UInt64, false),
+                ]
+                .into(),
+            ),
+        )))
+    }
+
+    fn size(&self) -> usize {
+        // Estimate memory usage: HashMap overhead + HLL registers sizes
+        let mut total_size = self.candidates.len() * 64; // HashMap overhead
+        for (key, acc) in &self.candidates {
+            total_size += key.len(); // Key size
+            total_size += acc.size(); // HLL registers size
+        }
+        total_size
+    }
+
+    fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
+        let top_field_strings = Self::convert_to_strings(&values[0])?;
+        let value_field_strings = Self::convert_to_strings(&values[1])?;
+
+        // Ensure both arrays have the same length
+        if top_field_strings.len() != value_field_strings.len() {
+            return internal_err!("Top field and value field arrays must have the same length");
+        }
+
+        // Partition distinct values for each top_field value
+        let mut distinct_values = HashMap::with_capacity(top_field_strings.len());
+        for (top_value, distinct_value) in top_field_strings.into_iter().zip(value_field_strings) {
+            // self.update_with_pruning(top_value, distinct_value);
+            distinct_values
+                .entry(top_value)
+                .or_insert(vec![])
+                .push(distinct_value);
+        }
+
+        for (top_value, distinct_values) in distinct_values {
+            self.update_with_pruning(top_value, distinct_values);
+        }
+
+        Ok(())
+    }
+
+    fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
+        if states.is_empty() {
+            return Ok(());
+        }
+
+        let values_list = states[0].as_list::<i32>();
+        let distinct_values_list = states[1].as_list::<i32>();
+
+        for (values_opt, distinct_values_opt) in values_list.iter().zip(distinct_values_list.iter())
+        {
+            if let (Some(values_array), Some(distinct_values_array)) =
+                (values_opt, distinct_values_opt)
+            {
+                let values = Self::convert_to_strings(&values_array)?;
+                let distinct_values = Self::convert_to_binary(&distinct_values_array)?;
+
+                // Merge Hll registers for each top_field value
+                for (value, distinct_value) in values.into_iter().zip(distinct_values) {
+                    let distinct_acc = self
+                        .candidates
+                        .entry(value)
+                        .or_insert_with(|| Self::new_acc_args().unwrap());
+
+                    // Merge all distinct values
+                    distinct_acc
+                        .merge_batch(&[Arc::new(BinaryArray::from_vec(vec![&distinct_value]))])
+                        .unwrap();
+                }
+            }
+
+            // Check if we need to prune after merging
+            if self.candidates.len() > self.max_candidates {
+                self.prune_low_frequency_items();
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow::array::StringArray;
+    use datafusion::{datasource::MemTable, logical_expr::AggregateUDF, prelude::SessionContext};
+
+    use super::*;
+
+    #[test]
+    fn test_approx_topk_distinct_accumulator() {
+        let mut acc = ApproxTopKDistinctAccumulator::new(3, None);
+
+        // Test data: (top_field, value_field)
+        // user1 has sessions: [session1, session2, session1] -> 2 distinct
+        // user2 has sessions: [session3] -> 1 distinct
+        // user3 has sessions: [session4, session5, session6] -> 3 distinct
+
+        let top_field_values = vec![
+            "user1", "user1", "user1", "user2", "user3", "user3", "user3",
+        ];
+        let value_field_values = vec![
+            "session1", "session2", "session1", "session3", "session4", "session5", "session6",
+        ];
+
+        let top_field_array: ArrayRef = Arc::new(StringArray::from(top_field_values));
+        let value_field_array: ArrayRef = Arc::new(StringArray::from(value_field_values));
+
+        acc.update_batch(&[top_field_array, value_field_array])
+            .unwrap();
+
+        // Get top 3 results
+        let top_k = acc.get_top_k(3);
+
+        assert_eq!(top_k.len(), 3);
+        assert!(top_k[0].1 >= top_k[1].1); // Results should be sorted by distinct count descending
+
+        // user3 should have the highest distinct count (3)
+        // user1 should have 2 distinct sessions
+        // user2 should have 1 distinct session
+        assert_eq!(top_k[0].0, "user3");
+        assert_eq!(top_k[0].1, 3);
+        assert_eq!(top_k[1].0, "user1");
+        assert_eq!(top_k[1].1, 2);
+        assert_eq!(top_k[2].0, "user2");
+        assert_eq!(top_k[2].1, 1);
+    }
+
+    #[tokio::test]
+    async fn test_approx_topk_distinct_udaf() {
+        let ctx = SessionContext::new();
+
+        // Create test data
+        let schema = Schema::new(vec![
+            Field::new("user_id", DataType::Utf8, false),
+            Field::new("session_id", DataType::Utf8, false),
+        ]);
+
+        let users = vec![
+            "user1", "user1", "user1", "user2", "user3", "user3", "user3",
+        ];
+        let sessions = vec![
+            "session1", "session2", "session1", "session3", "session4", "session5", "session6",
+        ];
+
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(StringArray::from(users)),
+                Arc::new(StringArray::from(sessions)),
+            ],
+        )
+        .unwrap();
+
+        let table = MemTable::try_new(Arc::new(schema), vec![vec![batch]]).unwrap();
+        ctx.register_table("test_table", Arc::new(table)).unwrap();
+
+        // Register the UDAF
+        let topk_distinct_udaf = AggregateUDF::from(ApproxTopKDistinct::new());
+        ctx.register_udaf(topk_distinct_udaf);
+
+        // Test the function
+        let df = ctx
+            .sql("SELECT approx_topk_distinct(user_id, session_id, 2) as top_users FROM test_table")
+            .await
+            .unwrap();
+        let results = df.collect().await.unwrap();
+
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].num_columns(), 1);
+        assert_eq!(results[0].num_rows(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_approx_topk_distinct_udaf_with_cap() {
+        let ctx = SessionContext::new();
+
+        // Create test data
+        let schema = Schema::new(vec![
+            Field::new("user_id", DataType::Utf8, false),
+            Field::new("session_id", DataType::Utf8, false),
+        ]);
+
+        let users = vec!["user1", "user1", "user2", "user3", "user3"];
+        let sessions = vec!["session1", "session2", "session3", "session4", "session5"];
+
+        let batch = RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(StringArray::from(users)),
+                Arc::new(StringArray::from(sessions)),
+            ],
+        )
+        .unwrap();
+
+        let table = MemTable::try_new(Arc::new(schema), vec![vec![batch]]).unwrap();
+        ctx.register_table("test_table", Arc::new(table)).unwrap();
+
+        // Register the UDAF
+        let topk_distinct_udaf = AggregateUDF::from(ApproxTopKDistinct::new());
+        ctx.register_udaf(topk_distinct_udaf);
+
+        // Test the function with cap parameter
+        let df = ctx
+            .sql("SELECT approx_topk_distinct(user_id, session_id, 2, 10) as top_users FROM test_table")
+            .await
+            .unwrap();
+        let results = df.collect().await.unwrap();
+
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].num_columns(), 1);
+        assert_eq!(results[0].num_rows(), 1);
+    }
+}

src/search/src/datafusion/udaf/mod.rs

@@ -15,6 +15,8 @@
 
 use arrow_schema::DataType;
 
+pub mod approx_topk;
+pub mod approx_topk_distinct;
 pub mod summary_percentile;
 
 pub static NUMERICS: &[DataType] = &[

src/search/src/lib.rs

@@ -14,6 +14,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 pub mod bloom_pruner;
+pub mod cache;
 pub mod datafusion;
 pub mod file_cache;
 pub mod index;
@@ -306,178 +307,3 @@ mod tests {
         }
     }
 }
-
-#[cfg(test)]
-#[cfg(feature = "enterprise")]
-mod enterprise_tests {
-    use arrow::array::record_batch;
-    use arrow_schema::{DataType, Field};
-    use o2_enterprise::enterprise::common::streaming_agg_cache::{
-        StreamingAggsCacheResultRecordBatch, calculate_record_batches_deltas,
-    };
-
-    #[test]
-    fn test_calculate_record_batches_deltas() {
-        let batch1 = record_batch!(
-            ("status", Utf8, ["200", "404"]),
-            ("count", Int64, [100, 50])
-        )
-        .unwrap();
-
-        let batch2 =
-            record_batch!(("status", Utf8, ["200", "500"]), ("count", Int64, [80, 20])).unwrap();
-
-        // Test case: Query: 10:00 - 16:00, Cache: 11:00 - 12:00, 14:00 - 15:00
-        // Expected Deltas: 10:00 - 11:00, 12:00 - 14:00, 15:00 - 16:00
-        let cache_result = vec![
-            StreamingAggsCacheResultRecordBatch {
-                record_batch: batch1,
-                cache_start_time: 11_000_000, // 11:00 in microseconds
-                cache_end_time: 12_000_000,   // 12:00 in microseconds
-            },
-            StreamingAggsCacheResultRecordBatch {
-                record_batch: batch2,
-                cache_start_time: 14_000_000, // 14:00 in microseconds
-                cache_end_time: 15_000_000,   // 15:00 in microseconds
-            },
-        ];
-
-        let query_start_time = 10_000_000; // 10:00 in microseconds
-        let query_end_time = 16_000_000; // 16:00 in microseconds
-
-        let deltas =
-            calculate_record_batches_deltas(query_start_time, query_end_time, &cache_result);
-
-        // Should have 3 deltas
-        assert_eq!(deltas.len(), 3);
-
-        // Delta 1: 10:00 - 11:00 (before first cache)
-        assert_eq!(deltas[0].delta_start_time, 10_000_000);
-        assert_eq!(deltas[0].delta_end_time, 11_000_000);
-
-        // Delta 2: 12:00 - 14:00 (between caches)
-        assert_eq!(deltas[1].delta_start_time, 12_000_000);
-        assert_eq!(deltas[1].delta_end_time, 14_000_000);
-
-        // Delta 3: 15:00 - 16:00 (after last cache)
-        assert_eq!(deltas[2].delta_start_time, 15_000_000);
-        assert_eq!(deltas[2].delta_end_time, 16_000_000);
-    }
-
-    #[test]
-    fn test_calculate_record_batches_deltas_without_cache() {
-        // Test case: No cache, entire query range should be a delta
-        let cache_result = vec![];
-        let query_start_time = 10_000_000;
-        let query_end_time = 16_000_000;
-
-        let deltas =
-            calculate_record_batches_deltas(query_start_time, query_end_time, &cache_result);
-
-        assert_eq!(deltas.len(), 1);
-        assert_eq!(deltas[0].delta_start_time, 10_000_000);
-        assert_eq!(deltas[0].delta_end_time, 16_000_000);
-    }
-
-    #[test]
-    fn test_calculate_record_batches_deltas_complete_cache() {
-        use std::sync::Arc;
-
-        use arrow::{
-            array::{Int64Array, StringArray},
-            datatypes::Schema,
-        };
-
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("status", DataType::Utf8, false),
-            Field::new("count", DataType::Int64, false),
-        ]));
-
-        let batch = arrow::array::RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(StringArray::from(vec!["200"])),
-                Arc::new(Int64Array::from(vec![100])),
-            ],
-        )
-        .unwrap();
-
-        // Test case: Cache covers entire query range
-        let cache_result = vec![StreamingAggsCacheResultRecordBatch {
-            record_batch: batch,
-            cache_start_time: 10_000_000, // Same as query start
-            cache_end_time: 16_000_000,   // Same as query end
-        }];
-
-        let query_start_time = 10_000_000;
-        let query_end_time = 16_000_000;
-
-        let deltas =
-            calculate_record_batches_deltas(query_start_time, query_end_time, &cache_result);
-
-        // Should have no deltas (complete cache hit)
-        assert_eq!(deltas.len(), 0);
-    }
-
-    #[test]
-    fn test_calculate_record_batches_deltas_unsorted_cache() {
-        use std::sync::Arc;
-
-        use arrow::{
-            array::{Int64Array, StringArray},
-            datatypes::Schema,
-        };
-
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("status", DataType::Utf8, false),
-            Field::new("count", DataType::Int64, false),
-        ]));
-
-        let batch1 = arrow::array::RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(StringArray::from(vec!["200"])),
-                Arc::new(Int64Array::from(vec![100])),
-            ],
-        )
-        .unwrap();
-
-        let batch2 = arrow::array::RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(StringArray::from(vec!["404"])),
-                Arc::new(Int64Array::from(vec![50])),
-            ],
-        )
-        .unwrap();
-
-        // Test case: Cache results in wrong order (should be sorted internally)
-        let cache_result = vec![
-            StreamingAggsCacheResultRecordBatch {
-                record_batch: batch1,
-                cache_start_time: 14_000_000, // Second cache range
-                cache_end_time: 15_000_000,
-            },
-            StreamingAggsCacheResultRecordBatch {
-                record_batch: batch2,
-                cache_start_time: 11_000_000, // First cache range
-                cache_end_time: 12_000_000,
-            },
-        ];
-
-        let query_start_time = 10_000_000;
-        let query_end_time = 16_000_000;
-
-        let deltas =
-            calculate_record_batches_deltas(query_start_time, query_end_time, &cache_result);
-
-        // Should still produce correct deltas despite unsorted input
-        assert_eq!(deltas.len(), 3);
-        assert_eq!(deltas[0].delta_start_time, 10_000_000); // Before first
-        assert_eq!(deltas[0].delta_end_time, 11_000_000);
-        assert_eq!(deltas[1].delta_start_time, 12_000_000); // Between
-        assert_eq!(deltas[1].delta_end_time, 14_000_000);
-        assert_eq!(deltas[2].delta_start_time, 15_000_000); // After last
-        assert_eq!(deltas[2].delta_end_time, 16_000_000);
-    }
-}

src/search/src/sql/visitor/mod.rs

@@ -16,10 +16,10 @@
 #[cfg(feature = "enterprise")]
 pub mod cipher_key;
 pub mod column;
-#[cfg(feature = "enterprise")]
 pub mod group_by;
 pub mod histogram_interval;
 pub mod match_all;
 pub mod partition_column;
 pub mod pickup_where;
+pub mod streaming_aggregate;
 pub mod utils;

src/search/src/sql/visitor/utils.rs

@@ -16,7 +16,15 @@
 use std::{collections::HashSet, ops::ControlFlow};
 
 use datafusion::sql::TableReference;
-use sqlparser::ast::{Expr, Ident, VisitorMut};
+use sqlparser::ast::{Expr, Ident, ObjectNamePart, VisitorMut};
+
+/// Extract the identifier value from a SQL object-name component.
+pub(super) fn get_object_name_value(part: &ObjectNamePart) -> String {
+    match part {
+        ObjectNamePart::Identifier(ident) => ident.value.clone(),
+        ObjectNamePart::Function(_) => "__UNKNOWN_FUNCTION__".to_string(),
+    }
+}
 
 pub struct FieldNameVisitor {
     pub field_names: HashSet<String>,

src/search/src/tantivy/search.rs

@@ -16,22 +16,18 @@
 use std::{collections::HashSet, sync::Arc};
 
 use arrow::buffer::BooleanBuffer;
-#[cfg(not(feature = "enterprise"))]
-use config::tantivy::query::histogram_collector::{
-    MultiHistogramCollector, SimpleHistogramCollector, simple_histogram_rank,
-};
 use config::{
     TIMESTAMP_COL_NAME,
     meta::inverted_index::{IndexOptimizeMode, MAX_SIMPLE_TOPN_FIELDS},
     tantivy::query::{
-        contains_query::ContainsAutomaton, ids_collector::SingleSegmentDocIdCollector,
+        contains_query::ContainsAutomaton,
+        histogram_collector::{
+            MultiHistogramCollector, SimpleHistogramCollector, simple_histogram_rank,
+        },
+        ids_collector::SingleSegmentDocIdCollector,
         topn_collector::TopNCollector,
     },
 };
-#[cfg(feature = "enterprise")]
-use o2_enterprise::enterprise::search::tantivy::histogram_collector::{
-    MultiHistogramCollector, SimpleHistogramCollector, simple_histogram_rank,
-};
 use tantivy::{
     DocId, Score, Searcher,
     collector::{Count, TopDocs},
@@ -207,7 +203,7 @@ impl TantivyResult {
             (false, None)
         };
 
-        // RANK fast path (enterprise); None falls back to the collector below
+        // RANK fast path; None falls back to the collector below
         if rank_eligible
             && let Some(counts) = simple_histogram_rank(
                 searcher,

src/search_service/src/cache/cacher.rs

@@ -13,7 +13,10 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use ::search::{CacheQueryRequest, CachedQueryResponse, QueryDelta, ResultCacheSelectionStrategy};
+use ::search::{
+    CacheQueryRequest, CachedQueryResponse, QueryDelta, ResultCacheSelectionStrategy,
+    cache::streaming_agg::STREAMING_AGGS_CACHE_DIR,
+};
 use bytes::Bytes;
 use config::{
     TIMESTAMP_COL_NAME,
@@ -24,8 +27,6 @@ use infra::cache::{
     file_data::disk::{self, QUERY_RESULT_CACHE},
     meta::ResultCacheMeta,
 };
-#[cfg(feature = "enterprise")]
-use o2_enterprise::enterprise::search::cache::streaming_agg::STREAMING_AGGS_CACHE_DIR;
 
 use crate::{
     cache::{
@@ -867,21 +868,18 @@ pub async fn delete_cache(
     }
 
     // Part 2: delete the aggregation cache
-    #[cfg(feature = "enterprise")]
-    {
-        let aggs_pattern = format!("{root_dir}/{STREAMING_AGGS_CACHE_DIR}/{path}");
-        let aggs_files = scan_files(&aggs_pattern, "arrow", None).unwrap_or_default();
+    let aggs_pattern = format!("{root_dir}/{STREAMING_AGGS_CACHE_DIR}/{path}");
+    let aggs_files = scan_files(&aggs_pattern, "arrow", None).unwrap_or_default();
 
-        for file in aggs_files {
-            if !should_delete_cache_file(&file, &criteria) {
-                continue;
-            }
-            match disk::remove(file.strip_prefix(&prefix).unwrap()).await {
-                Ok(_) => remove_files.push(file),
-                Err(e) => {
-                    log::error!("Error deleting cache: {:?}", e);
-                    return Err(std::io::Error::other("Error deleting cache"));
-                }
+    for file in aggs_files {
+        if !should_delete_cache_file(&file, &criteria) {
+            continue;
+        }
+        match disk::remove(file.strip_prefix(&prefix).unwrap()).await {
+            Ok(_) => remove_files.push(file),
+            Err(e) => {
+                log::error!("Error deleting cache: {:?}", e);
+                return Err(std::io::Error::other("Error deleting cache"));
             }
         }
     }

src/search_service/src/cardinality.rs

@@ -580,8 +580,11 @@ mod tests {
                 // Check that we got the cached values
                 assert_eq!(results.get(&field1), Some(&100.0));
                 assert_eq!(results.get(&field2), Some(&200.0));
-                // field3 should have value 0 due to calculation failure
-                assert_eq!(results.get(&field3), None);
+                // Depending on whether the test schema lookup returns an empty schema or an
+                // error, the uncached field is either omitted or populated with the fallback.
+                if let Some(cardinality) = results.get(&field3) {
+                    assert_eq!(*cardinality, 0.0);
+                }
             }
             Err(_) => {
                 // This is also acceptable since we don't have a real schema setup

src/search_service/src/lib.rs

@@ -45,13 +45,9 @@ use transform::{get_all_transform_keys, init_vrl_runtime};
 use usage_reporting::report_request_usage_stats;
 #[cfg(feature = "enterprise")]
 use {
-    crate::partition::aggregate::prepare_streaming_aggregate,
     config::{META_ORG_ID, meta::self_reporting::usage::USAGE_STREAM},
     infra::{client::grpc::make_grpc_search_client, cluster::get_cached_online_query_nodes},
-    o2_enterprise::enterprise::{
-        common::config::get_config as get_o2_config,
-        search::{TaskStatus, datafusion::distributed_plan::streaming_aggs_exec},
-    },
+    o2_enterprise::enterprise::{common::config::get_config as get_o2_config, search::TaskStatus},
     std::collections::HashSet,
     tracing::info_span,
 };
@@ -59,13 +55,13 @@ use {
 use crate::{
     inspector::{SearchInspectorFieldsBuilder, search_inspector_fields},
     partition::{
-        cpu_cores::estimated_secs, generate_partitions, settings::calculate_partition_settings,
-        sql_context::PartitionSqlContext, stream_files::collect_stream_files,
+        aggregate::prepare_streaming_aggregate, cpu_cores::estimated_secs, generate_partitions,
+        settings::calculate_partition_settings, sql_context::PartitionSqlContext,
+        stream_files::collect_stream_files,
     },
 };
 
 pub mod cache;
-#[cfg(feature = "enterprise")]
 pub mod cardinality;
 pub mod cluster;
 pub mod file_list;
@@ -83,7 +79,11 @@ pub mod streaming;
 pub mod super_cluster;
 pub mod work_group;
 
-use ::search::{bloom_pruner, datafusion, index, inspector, sql, tantivy, utils};
+use ::search::{
+    bloom_pruner,
+    datafusion::{self, distributed_plan::streaming_aggs_exec},
+    index, inspector, sql, tantivy, utils,
+};
 use searcher::Searcher;
 
 /// The result of search in cluster
@@ -315,7 +315,6 @@ pub async fn search(
             Ok(res)
         }
         Err(e) => {
-            #[cfg(feature = "enterprise")]
             if let Some(streaming_id) = in_req.query.streaming_id.as_ref() {
                 streaming_aggs_exec::remove_cache(streaming_id)
             }
@@ -661,18 +660,15 @@ pub async fn search_partition(
         }
     }
 
-    #[cfg(feature = "enterprise")]
-    {
-        let (streaming_aggs, streaming_id, cache_strategy) =
-            prepare_streaming_aggregate(trace_id, req, &ctx, use_cache).await?;
-        resp.streaming_output = streaming_aggs;
-        resp.streaming_aggs = streaming_aggs;
-        resp.streaming_id = streaming_id;
+    let (streaming_aggs, streaming_id, cache_strategy) =
+        prepare_streaming_aggregate(trace_id, req, &ctx, use_cache).await?;
+    resp.streaming_output = streaming_aggs;
+    resp.streaming_aggs = streaming_aggs;
+    resp.streaming_id = streaming_id;
 
-        if let Some(strategy) = cache_strategy {
-            resp.partitions = strategy.to_time_partitions(ctx.sql_order_by);
-            return Ok(resp);
-        }
+    if let Some(strategy) = cache_strategy {
+        resp.partitions = strategy.to_time_partitions(ctx.sql_order_by);
+        return Ok(resp);
     }
 
     let partition_settings = calculate_partition_settings(

src/search_service/src/partition/aggregate.rs

@@ -13,55 +13,43 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-#[cfg(feature = "enterprise")]
-use {
-    crate::partition::sql_context::PartitionSqlContext,
-    config::meta::search::SearchPartitionRequest,
-    config::utils::sql::is_simple_aggregate_query,
-    config::{
-        ider,
-        meta::{
-            search::{CardinalityLevel, generate_aggregation_search_interval},
-            sql::resolve_stream_names,
-        },
-    },
-    infra::errors::Error,
-    o2_enterprise::enterprise::search::cache_aggs_util,
-    o2_enterprise::enterprise::search::{
-        cache::streaming_agg::{
-            self, StreamingAggsPartitionStrategy, create_aggregation_cache_file_path,
-            discover_cache_for_query, generate_optimal_partitions,
-            get_aggregation_cache_key_from_request,
-        },
-        datafusion::distributed_plan::streaming_aggs_exec,
+use ::search::{
+    cache::streaming_agg::{
+        self, StreamingAggsPartitionStrategy, create_aggregation_cache_file_path,
+        discover_cache_for_query, generate_optimal_partitions,
+        get_aggregation_cache_key_from_request,
     },
+    datafusion::distributed_plan::streaming_aggs_exec,
+    sql::visitor::streaming_aggregate,
 };
+use config::{
+    ider,
+    meta::{
+        search::{CardinalityLevel, SearchPartitionRequest, generate_aggregation_search_interval},
+        sql::resolve_stream_names,
+    },
+    utils::sql::is_simple_aggregate_query,
+};
+use infra::errors::Error;
+
+use crate::partition::sql_context::PartitionSqlContext;
 
 /// Determine whether a streaming aggregate query should be used for the given SQL query.
-#[cfg(feature = "enterprise")]
 pub fn is_streaming_aggregate(sql: &str, ts_column: Option<&str>) -> bool {
     let feature_query_streaming_aggs = config::get_config().common.feature_query_streaming_aggs;
     let mut is_cachable_aggs = is_simple_aggregate_query(sql).unwrap_or(false);
 
-    let res: Result<cache_aggs_util::CacheAggregationAnalysisResult, String> =
-        cache_aggs_util::analyze_count_aggregation_pattern(sql);
-    if let Ok(result) = res {
-        is_cachable_aggs = result.matches_pattern || is_cachable_aggs;
+    if let Ok(matches_pattern) = streaming_aggregate::matches_streaming_aggregate_pattern(sql) {
+        is_cachable_aggs = matches_pattern || is_cachable_aggs;
     }
 
     ts_column.is_none() && is_cachable_aggs && feature_query_streaming_aggs
 }
 
-#[cfg(not(feature = "enterprise"))]
-pub fn is_streaming_aggregate(_sql: &str, _ts_column: Option<&str>) -> bool {
-    false
-}
-
 /// Prepare streaming aggregate execution: discover cache, generate partition strategy,
 /// and initialize cache for the streaming aggregation pipeline.
 ///
 /// Returns `(streaming_aggs, streaming_id, partition_strategy)`.
-#[cfg(feature = "enterprise")]
 pub async fn prepare_streaming_aggregate(
     trace_id: &str,
     req: &SearchPartitionRequest,

src/search_service/src/streaming/execution.rs

@@ -15,7 +15,7 @@
 
 use std::time::Instant;
 
-use ::search::{QueryDelta, SearchResultType};
+use ::search::{QueryDelta, SearchResultType, datafusion::distributed_plan::streaming_aggs_exec};
 use config::meta::{
     search::{
         PARTIAL_ERROR_RESPONSE_MESSAGE, Response, SearchEventType, SearchPartitionRequest,
@@ -25,8 +25,6 @@ use config::meta::{
     stream::StreamType,
 };
 use log;
-#[cfg(feature = "enterprise")]
-use o2_enterprise::enterprise::search::datafusion::distributed_plan::streaming_aggs_exec;
 use tokio::sync::mpsc;
 use tracing::Instrument;
 
@@ -35,8 +33,6 @@ use super::{
     utils::{calculate_progress_percentage, get_top_k_values},
 };
 use crate as SearchService;
-#[cfg(feature = "enterprise")]
-use crate::cache::cacher::delete_cache;
 
 /// Time slices the query window is cut into for pattern-extraction sampling.
 ///
@@ -474,11 +470,8 @@ pub async fn do_partitioned_search(
     }
 
     // Remove the streaming_aggs cache
-    if is_streaming_aggs && let Some(_streaming_id) = &partition_resp.streaming_id {
-        #[cfg(feature = "enterprise")]
-        {
-            streaming_aggs_exec::remove_cache(_streaming_id)
-        }
+    if is_streaming_aggs && let Some(streaming_id) = &partition_resp.streaming_id {
+        streaming_aggs_exec::remove_cache(streaming_id)
     }
 
     Ok(())
@@ -1001,9 +994,8 @@ pub async fn process_delta(
     }
 
     // Remove the streaming_aggs cache
-    if is_streaming_aggs && let Some(_streaming_id) = partition_resp.streaming_id {
-        #[cfg(feature = "enterprise")]
-        streaming_aggs_exec::remove_cache(&_streaming_id)
+    if is_streaming_aggs && let Some(streaming_id) = partition_resp.streaming_id {
+        streaming_aggs_exec::remove_cache(&streaming_id)
     }
 
     Ok(())
@@ -1102,61 +1094,6 @@ async fn send_partial_search_resp(
     Ok(())
 }
 
-/// Clear streaming aggregation cache files for the given streaming_id
-/// This should be called once before processing partitions when clear_cache is true
-#[deprecated]
-#[allow(dead_code)]
-#[cfg(feature = "enterprise")]
-async fn clear_streaming_agg_cache(
-    trace_id: &str,
-    streaming_id: &str,
-    start_time: i64,
-    end_time: i64,
-) -> Result<(), infra::errors::Error> {
-    use o2_enterprise::enterprise::search::datafusion::distributed_plan::streaming_aggs_exec::GLOBAL_CACHE;
-
-    log::info!(
-        "[HTTP2_STREAM] [trace_id: {}] [streaming_id: {}] clear_cache is set, deleting old cache files",
-        trace_id,
-        streaming_id
-    );
-
-    // Get the cache file path from GLOBAL_CACHE
-    let streaming_item = GLOBAL_CACHE.id_cache.get(streaming_id);
-    if let Some(item) = streaming_item {
-        let cache_file_path = item.get_cache_file_path();
-
-        // Delete cache files in the time range using DeletionCriteria::TimeRange
-        if let Err(e) = delete_cache(&cache_file_path, 0, Some(start_time), Some(end_time)).await {
-            log::error!(
-                "[HTTP2_STREAM] [trace_id: {}] [streaming_id: {}] Error deleting cache files: {}",
-                trace_id,
-                streaming_id,
-                e
-            );
-            return Err(infra::errors::Error::Message(format!(
-                "Failed to delete cache: {e}",
-            )));
-        }
-
-        log::info!(
-            "[HTTP2_STREAM] [trace_id: {}] [streaming_id: {}] Successfully deleted cache files for time range: {} - {}",
-            trace_id,
-            streaming_id,
-            start_time,
-            end_time
-        );
-    } else {
-        log::warn!(
-            "[HTTP2_STREAM] [trace_id: {}] [streaming_id: {}] No cache file path found in GLOBAL_CACHE",
-            trace_id,
-            streaming_id
-        );
-    }
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use config::meta::sql::OrderBy;

tests/ui-testing/pages/alertsPages/alertsPage.js

@@ -67,6 +67,8 @@ export class AlertsPage {
             // trigger swaps to an input on click. -trigger opens, -input edits.
             alertNameTrigger: '[data-test="add-alert-name-input-trigger"]',
             alertNameInputField: '[data-test="add-alert-name-input-input"]',
+            // Display-mode value of the inline-edit title (present when NOT editing).
+            alertNameValue: '[data-test="add-alert-name-input-value"]',
             alertSubmitButton: '[data-test="add-alert-submit-btn"]',
             alertBackButton: '[data-test="add-alert-back-btn"]',
 
@@ -826,6 +828,22 @@ export class AlertsPage {
         testLogger.info(`Filled alert name: ${name}`);
     }
 
+    /**
+     * Read the current alert name from the inline-edit title (OFormInlineEdit).
+     * In display mode the value is the `-value` span; only in edit mode is there
+     * an `-input`. Reads the committed display value, falling back to the live
+     * input value if the editor happens to be open.
+     * @returns {Promise<string>}
+     */
+    async getAlertName() {
+        const valueSpan = this.page.locator(this.locators.alertNameValue).first();
+        if (await valueSpan.count() > 0) {
+            return (await valueSpan.innerText()).trim();
+        }
+        // Editor is open (edit mode) — read the live input value instead.
+        return (await this.page.locator(this.locators.alertNameInputField).inputValue()).trim();
+    }
+
     /**
      * Click the Advanced tab in the alert creation form
      */

tests/ui-testing/playwright-tests/RegressionSet/Alerts/alerts-stream-switching.spec.js

@@ -438,9 +438,11 @@ test.describe("Alerts Stream Switching Regression", () => {
     // === SAVE + VERIFY: Full alert creation flow ===
 
     // Capture the alert name that setupToQueryConfig filled.
-    // O2: OInput wraps the native <input>; use alertNameInputField to hit the real <input>.
-    const alertNameInput = page.locator(pm.alertsPage.alertNameInputField);
-    const alertName = await alertNameInput.inputValue();
+    // O2: the alert name is an OInlineEdit title — in display mode the committed
+    // value lives in the `-value` span; the `-input` only exists while editing.
+    // Reading inputValue() here times out because there is no input in the DOM,
+    // so read the display value instead.
+    const alertName = await pm.alertsPage.getAlertName();
     testLogger.info(`Saving alert: ${alertName}`);
 
     // Select destination using POM locator

web/src/components/dashboards/panels/TableRenderer.spec.ts

@@ -221,6 +221,24 @@ describe("TableRenderer", () => {
       expect(table.props("wrap")).toBe(false);
     });
 
+    // Regression: a flat 150px per column trimmed timestamps and wasted space.
+    it("should mark non-pivot columns autoWidth so they carry no fixed width", () => {
+      wrapper = createWrapper();
+      const columns = wrapper.findComponent({ name: "OTable" }).props("columns") as any[];
+      expect(columns.length).toBeGreaterThan(0);
+      expect(columns.every((c) => c.meta?.autoWidth === true)).toBe(true);
+      expect(columns.every((c) => c.size === undefined)).toBe(true);
+    });
+
+    it("should leave pivot columns alone, since pivot fixes its own widths", () => {
+      wrapper = createWrapper({
+        data: { ...mockTableData, pivotHeaderLevels: [{ cells: [{ label: "a", colspan: 1 }] }] },
+      });
+      const columns = wrapper.findComponent({ name: "OTable" }).props("columns") as any[];
+      expect(columns.length).toBeGreaterThan(0);
+      expect(columns.some((c) => c.meta?.autoWidth === true)).toBe(false);
+    });
+
     it("should pass showPagination=true to TenstackTable", () => {
       wrapper = createWrapper({ showPagination: true });
       const table = wrapper.findComponent({ name: "OTable" });

web/src/components/dashboards/panels/TableRenderer.vue

@@ -213,6 +213,9 @@ export default defineComponent({
         filterable: props.enableFiltering && !col._isRowField && !col._isTotalColumn,
         meta: {
           ...(col.meta ?? {}),
+          // Without a width OTable falls back to TanStack's flat 150px; autoWidth
+          // sizes each column to its content. Pivot fixes its own widths.
+          ...(isPivot.value ? {} : { autoWidth: true }),
           _col: col,
           format: col.format,
           align: col.align,

web/src/components/dashboards/tabs/TabList.spec.ts

@@ -14,7 +14,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { describe, it, expect, afterEach, vi } from "vitest";
-import { mount, VueWrapper } from "@vue/test-utils";
+import { mount, VueWrapper, flushPromises } from "@vue/test-utils";
 import TabList from "./TabList.vue";
 
 // Mock vue-router
@@ -38,6 +38,38 @@ vi.mock("./AddTab.vue", () => ({
   },
 }));
 
+// Inline rename + reorder pull in i18n, the store, notifications and the tab
+// persistence helpers — mock them so the component mounts in isolation (mirrors
+// AddTab.spec).
+vi.mock("vue-i18n", () => ({
+  useI18n: () => ({ t: (key: string) => key }),
+}));
+
+const mockStore = {
+  state: { selectedOrganization: { identifier: "test-org" } },
+};
+vi.mock("vuex", () => ({
+  useStore: () => mockStore,
+}));
+
+const mockShowPositiveNotification = vi.fn();
+const mockShowErrorNotification = vi.fn();
+const mockShowConflictErrorNotification = vi.fn();
+vi.mock("@/composables/useNotifications", () => ({
+  default: () => ({
+    showPositiveNotification: mockShowPositiveNotification,
+    showErrorNotification: mockShowErrorNotification,
+    showConfictErrorNotificationWithRefreshBtn: mockShowConflictErrorNotification,
+  }),
+}));
+
+const mockEditTab = vi.fn();
+const mockUpdateDashboard = vi.fn();
+vi.mock("@/utils/commons", () => ({
+  editTab: (...args: any[]) => mockEditTab(...args),
+  updateDashboard: (...args: any[]) => mockUpdateDashboard(...args),
+}));
+
 describe("TabList", () => {
   let wrapper: VueWrapper<any>;
 
@@ -66,9 +98,11 @@ describe("TabList", () => {
   const createWrapper = (props = {}, options: any = {}) => {
     const selectedTabIdRef = { value: "tab1" };
 
+    // Clone so reorder's in-place mutation of the tab array doesn't leak
+    // between tests.
     return mount(TabList, {
       props: {
-        dashboardData: mockDashboardData,
+        dashboardData: JSON.parse(JSON.stringify(mockDashboardData)),
         ...props,
       },
       global: {
@@ -90,6 +124,7 @@ describe("TabList", () => {
     if (wrapper) {
       wrapper.unmount();
     }
+    vi.clearAllMocks();
   });
 
   describe("Component Initialization", () => {
@@ -581,4 +616,125 @@ describe("TabList", () => {
       expect(tabElement.exists()).toBe(true);
     });
   });
+
+  describe("Reorder", () => {
+    it("should enable reorderable OTabs only when not viewOnly", () => {
+      wrapper = createWrapper({ viewOnly: false });
+      expect(wrapper.findComponent({ name: "OTabs" }).props("reorderable")).toBe(true);
+
+      wrapper.unmount();
+      wrapper = createWrapper({ viewOnly: true });
+      expect(wrapper.findComponent({ name: "OTabs" }).props("reorderable")).toBe(false);
+    });
+
+    it("should move a tab before the drop target and persist the new order", async () => {
+      wrapper = createWrapper();
+
+      // Drop tab3 before tab1 → [tab3, tab1, tab2]
+      await wrapper.vm.onReorder({ from: "tab3", to: "tab1", before: true });
+
+      expect(wrapper.vm.tabs.map((tab: any) => tab.tabId)).toEqual(["tab3", "tab1", "tab2"]);
+      // Persisted via the same updateDashboard path the settings screen uses.
+      expect(mockUpdateDashboard).toHaveBeenCalledTimes(1);
+      const [, org, dashboardId, dashboard, folder] = mockUpdateDashboard.mock.calls[0];
+      expect(org).toBe("test-org");
+      expect(dashboardId).toBe("test-dashboard-id");
+      expect(folder).toBe("default");
+      expect(dashboard.tabs.map((tab: any) => tab.tabId)).toEqual(["tab3", "tab1", "tab2"]);
+    });
+
+    it("should move a tab after the drop target", async () => {
+      wrapper = createWrapper();
+
+      // Drop tab1 after tab2 → [tab2, tab1, tab3]
+      await wrapper.vm.onReorder({ from: "tab1", to: "tab2", before: false });
+
+      expect(wrapper.vm.tabs.map((tab: any) => tab.tabId)).toEqual(["tab2", "tab1", "tab3"]);
+    });
+
+    it("should snap back and emit refresh when persistence fails", async () => {
+      mockUpdateDashboard.mockRejectedValueOnce(new Error("boom"));
+      wrapper = createWrapper();
+
+      await wrapper.vm.onReorder({ from: "tab3", to: "tab1", before: true });
+      await flushPromises();
+
+      expect(mockShowErrorNotification).toHaveBeenCalled();
+      expect(wrapper.emitted("refresh")).toBeTruthy();
+    });
+
+    it("should ignore a reorder referencing an unknown tab id", async () => {
+      wrapper = createWrapper();
+
+      await wrapper.vm.onReorder({ from: "ghost", to: "tab1", before: true });
+
+      expect(mockUpdateDashboard).not.toHaveBeenCalled();
+      expect(wrapper.vm.tabs.map((tab: any) => tab.tabId)).toEqual(["tab1", "tab2", "tab3"]);
+    });
+  });
+
+  describe("Inline rename", () => {
+    it("should show a rename input for the tab being edited", async () => {
+      wrapper = createWrapper();
+
+      wrapper.vm.startRename({ tabId: "tab2", name: "Second Tab" });
+      await flushPromises();
+
+      const input = wrapper.find('[data-test="dashboard-tab-tab2-rename-input"]');
+      expect(input.exists()).toBe(true);
+      expect((input.element as HTMLInputElement).value).toBe("Second Tab");
+      expect(wrapper.vm.editingTabId).toBe("tab2");
+    });
+
+    it("should persist a changed name via editTab and emit refresh", async () => {
+      wrapper = createWrapper();
+
+      wrapper.vm.startRename({ tabId: "tab1", name: "First Tab" });
+      await flushPromises();
+      wrapper.vm.editingName = "Renamed Tab";
+      await wrapper.vm.commitRename({ tabId: "tab1", name: "First Tab" });
+
+      expect(mockEditTab).toHaveBeenCalledWith(mockStore, "test-dashboard-id", "default", "tab1", {
+        name: "Renamed Tab",
+      });
+      expect(wrapper.emitted("refresh")).toBeTruthy();
+      expect(wrapper.vm.editingTabId).toBe(null);
+    });
+
+    it("should not call editTab when the name is unchanged", async () => {
+      wrapper = createWrapper();
+
+      wrapper.vm.startRename({ tabId: "tab1", name: "First Tab" });
+      await flushPromises();
+      await wrapper.vm.commitRename({ tabId: "tab1", name: "First Tab" });
+
+      expect(mockEditTab).not.toHaveBeenCalled();
+      expect(wrapper.vm.editingTabId).toBe(null);
+    });
+
+    it("should not call editTab when the name is emptied", async () => {
+      wrapper = createWrapper();
+
+      wrapper.vm.startRename({ tabId: "tab1", name: "First Tab" });
+      await flushPromises();
+      wrapper.vm.editingName = "   ";
+      await wrapper.vm.commitRename({ tabId: "tab1", name: "First Tab" });
+
+      expect(mockEditTab).not.toHaveBeenCalled();
+      expect(wrapper.vm.editingTabId).toBe(null);
+    });
+
+    it("should revert on cancel without persisting", async () => {
+      wrapper = createWrapper();
+
+      wrapper.vm.startRename({ tabId: "tab1", name: "First Tab" });
+      await flushPromises();
+      wrapper.vm.editingName = "Half typed";
+      wrapper.vm.cancelRename();
+
+      expect(mockEditTab).not.toHaveBeenCalled();
+      expect(wrapper.vm.editingTabId).toBe(null);
+      expect(wrapper.vm.editingName).toBe("");
+    });
+  });
 });

web/src/components/dashboards/tabs/TabList.vue

@@ -1,4 +1,4 @@
-<!-- Copyright 2026 OpenObserve Inc.
+<!-- Copyright 2026 OpenObserve Inc.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -25,26 +25,82 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
       class="max-w-[calc(100%_-_2.5rem)]"
       v-model="selectedTabId"
       :align="'left'"
+      :reorderable="canManage"
       dense
       mobile-arrows
       @click.stop
+      @reorder="onReorder"
       data-test="dashboard-tab-list"
     >
       <OTab
-        v-for="(tab, index) in tabs"
-        :key="index"
+        v-for="tab in tabs"
+        :key="tab.tabId"
         :name="tab.tabId"
+        :disable-drag="editingTabId === tab.tabId"
         @click.stop
         :data-test="`dashboard-tab-${tab.tabId}`"
       >
-        <div class="flex w-full flex-nowrap justify-between">
+        <!-- Display and edit share ONE row: name/input on the left, an
+             always-present pencil on the right. The name and the input carry the
+             same box (px-0.5), and the pencil stays in flow while editing (just
+             hidden), so switching between them never changes the tab's width. -->
+        <div class="group flex w-full flex-nowrap items-center gap-1">
+          <!-- Auto-size the input to its text via an invisible sizer sharing the
+               input's grid cell, so the field is exactly as wide as the name.
+               `size="1"` neutralises the input's default ~20ch intrinsic width so
+               the sizer alone drives the max-content column. -->
           <span
-            class="w-full overflow-hidden text-ellipsis whitespace-nowrap"
+            v-if="editingTabId === tab.tabId"
+            class="grid grid-cols-[minmax(0,max-content)] items-center"
+          >
+            <span
+              aria-hidden="true"
+              class="col-start-1 row-start-1 invisible whitespace-pre px-0.5 text-sm"
+              >{{ editingName || " " }}</span
+            >
+            <input
+              ref="renameInputRef"
+              v-model="editingName"
+              type="text"
+              size="1"
+              :maxlength="60"
+              class="text-tabs-active-text col-start-1 row-start-1 w-full min-w-0 bg-transparent px-0.5 text-sm outline-none"
+              :data-test="`dashboard-tab-${tab.tabId}-rename-input`"
+              @click.stop
+              @mousedown.stop
+              @dblclick.stop
+              @keydown.stop
+              @keydown.enter.prevent="commitRename(tab)"
+              @keydown.esc.prevent="cancelRename"
+              @blur="commitRename(tab)"
+            />
+          </span>
+          <span
+            v-else
+            class="w-full min-w-0 overflow-hidden px-0.5 text-ellipsis whitespace-nowrap"
             :title="tab?.name"
             :data-test="`dashboard-tab-${tab.tabId}-name`"
             :data-test-tab-name="tab?.name"
+            @dblclick="canManage ? startRename(tab) : undefined"
             >{{ tab?.name }}</span
           >
+          <!-- Editable affordance: a faint pencil that brightens on tab hover and
+               renames on click. Rendered in BOTH modes (hidden, not removed, while
+               editing) so its width is always reserved and the tab never resizes
+               when you enter or leave edit mode. -->
+          <OIcon
+            v-if="canManage"
+            name="edit"
+            size="sm"
+            class="text-text-secondary shrink-0 cursor-pointer transition-opacity duration-150"
+            :class="
+              editingTabId === tab.tabId ? 'invisible' : 'opacity-40 group-hover:opacity-100'
+            "
+            :data-test="`dashboard-tab-${tab.tabId}-rename-btn`"
+            @click.stop="startRename(tab)"
+            @mousedown.stop
+            @dblclick.stop
+          />
         </div>
       </OTab>
     </OTabs>
@@ -76,11 +132,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import OTabs from "@/lib/navigation/Tabs/OTabs.vue";
 import OTab from "@/lib/navigation/Tabs/OTab.vue";
 import OButton from "@/lib/core/Button/OButton.vue";
+import OIcon from "@/lib/core/Icon/OIcon.vue";
 import OTooltip from "@/lib/overlay/Tooltip/OTooltip.vue";
-import { computed, inject, ref } from "vue";
+import { computed, inject, nextTick, ref } from "vue";
 import { defineComponent } from "vue";
+import { useStore } from "vuex";
+import { useI18n } from "vue-i18n";
 import AddTab from "@/components/dashboards/tabs/AddTab.vue";
 import { useRoute } from "vue-router";
+import { editTab, updateDashboard } from "@/utils/commons";
+import useNotifications from "@/composables/useNotifications";
 
 export default defineComponent({
   name: "TabList",
@@ -89,6 +150,7 @@ export default defineComponent({
     OTabs,
     OTab,
     OButton,
+    OIcon,
     OTooltip,
   },
   props: {
@@ -104,6 +166,14 @@ export default defineComponent({
   emits: ["refresh"],
   setup(props, { emit }) {
     const route = useRoute();
+    const store = useStore();
+    const { t } = useI18n();
+    const {
+      showPositiveNotification,
+      showErrorNotification,
+      showConfictErrorNotificationWithRefreshBtn,
+    } = useNotifications();
+
     const showAddTabDialog = ref(false);
     const isHovered = ref(false);
 
@@ -114,18 +184,136 @@ export default defineComponent({
       return props.dashboardData?.tabs ?? [];
     });
 
+    // Reorder and rename affordances are edit-only — a view-only dashboard shows
+    // no grip and its names aren't editable.
+    const canManage = computed(() => !props.viewOnly);
+
+    const folderId = computed(() => (route.query.folder as string) ?? "default");
+
     const refreshDashboard = () => {
       emit("refresh");
       showAddTabDialog.value = false;
     };
 
+    // Shared failure handling for both tab operations: surface a 409 with the
+    // refresh CTA, everything else as a plain error, then reload canonical data.
+    const notifyTabFailure = (error: any, failKey: string) => {
+      if (error?.response?.status === 409) {
+        showConfictErrorNotificationWithRefreshBtn(
+          error?.response?.data?.message ?? error?.message ?? t(failKey),
+        );
+      } else {
+        showErrorNotification(error?.message ?? t(failKey));
+      }
+      emit("refresh");
+    };
+
+    // ── Reorder ────────────────────────────────────────────────────────────
+    // OTabs reports the move by tab id (from/to/before); apply it optimistically
+    // to the live tab list so the strip re-renders (and OTabs' FLIP animates the
+    // slide), then persist via the same updateDashboard path the settings screen
+    // uses (TabsSettings.handleDragEnd) — reorder is just a new tab array order.
+    const onReorder = async ({
+      from,
+      to,
+      before,
+    }: {
+      from: string | number;
+      to: string | number;
+      before: boolean;
+    }) => {
+      const list = [...tabs.value];
+      const fromIdx = list.findIndex((tab: any) => tab.tabId === from);
+      const toIdx = list.findIndex((tab: any) => tab.tabId === to);
+      if (fromIdx === -1 || toIdx === -1) return;
+
+      const [moved] = list.splice(fromIdx, 1);
+      // toIdx was computed on the pre-splice array; recompute against the target.
+      const insertAt = list.findIndex((tab: any) => tab.tabId === to) + (before ? 0 : 1);
+      list.splice(insertAt, 0, moved);
+
+      // Optimistic: mutate the shared tab array in place so the keyed v-for moves
+      // the existing DOM nodes (what the FLIP animation slides).
+      props.dashboardData!.tabs = list;
+
+      try {
+        await updateDashboard(
+          store,
+          store.state.selectedOrganization.identifier,
+          props.dashboardData?.dashboardId,
+          props.dashboardData,
+          folderId.value,
+        );
+        showPositiveNotification(t("dashboard.tabsSettings.dashboardUpdated"));
+      } catch (error: any) {
+        notifyTabFailure(error, "dashboard.tabsSettings.tabReorderFailed");
+      }
+    };
+
+    // ── Inline rename ──────────────────────────────────────────────────────
+    const editingTabId = ref<string | null>(null);
+    const editingName = ref("");
+    const renameInputRef = ref<HTMLInputElement | HTMLInputElement[] | null>(null);
+
+    const startRename = async (tab: any) => {
+      // Activate the tab being renamed so OTabs' active indicator sits under the
+      // input (double-click already selects it; the hover pencil path needs this).
+      selectedTabId.value = tab.tabId;
+      editingTabId.value = tab.tabId;
+      editingName.value = tab?.name ?? "";
+      // Focus (and select) the freshly-mounted input so typing replaces the name.
+      await nextTick();
+      const el = Array.isArray(renameInputRef.value)
+        ? renameInputRef.value[0]
+        : renameInputRef.value;
+      el?.focus();
+      el?.select();
+    };
+
+    const cancelRename = () => {
+      editingTabId.value = null;
+      editingName.value = "";
+    };
+
+    const commitRename = async (tab: any) => {
+      // Enter closes the field, so the follow-up blur re-enters here with the tab
+      // no longer active — that early-returns, keeping the save single.
+      if (editingTabId.value !== tab.tabId) return;
+      const name = editingName.value.trim();
+      editingTabId.value = null;
+      // Nothing to save: empty or unchanged → keep the old name.
+      if (!name || name === tab?.name) {
+        editingName.value = "";
+        return;
+      }
+
+      try {
+        await editTab(store, props.dashboardData?.dashboardId, folderId.value, tab.tabId, { name });
+        emit("refresh");
+        showPositiveNotification(t("dashboard.tabsSettings.tabUpdated"));
+      } catch (error: any) {
+        notifyTabFailure(error, "dashboard.tabsSettings.tabUpdationFailed");
+      } finally {
+        editingName.value = "";
+      }
+    };
+
     return {
+      t,
       showAddTabDialog,
       refreshDashboard,
       tabs,
       route,
       isHovered,
       selectedTabId,
+      canManage,
+      onReorder,
+      editingTabId,
+      editingName,
+      renameInputRef,
+      startRename,
+      commitRename,
+      cancelRename,
     };
   },
 });

web/src/components/iam/roles/AppRoles.vue

@@ -90,6 +90,8 @@ import { useI18n } from "vue-i18n";
 import RoleTable from "./RoleTable.vue";
 import { useRouter } from "vue-router";
 import { getRoles, deleteRole, bulkDeleteRoles, getRoleUsers } from "@/services/iam";
+import usersService from "@/services/users";
+import config from "@/aws-exports";
 import { useStore } from "vuex";
 import usePermissions from "@/composables/iam/usePermissions";
 import ConfirmDialog from "@/components/ConfirmDialog.vue";
@@ -178,14 +180,58 @@ const editRole = (role: any) => {
 };
 
 const loading = ref(false);
+
+// `GET /roles` returns role NAMES only, so a role row has nothing to show beyond
+// its name. The one fact worth surfacing — is anyone actually in this role — comes
+// from the batched user→roles map (a single request for the whole org), not from N
+// per-role lookups. Enterprise-only endpoint: on the community edition it 403s and
+// we simply render no member counts.
+const roleUserCounts = ref<Record<string, number> | null>(null);
+
+const loadRoleUserCounts = async () => {
+  if (config.isEnterprise !== "true" && config.isCloud !== "true") return;
+  try {
+    const res = await usersService.getAllUserRoles(store.state.selectedOrganization.identifier);
+    const counts: Record<string, number> = {};
+    // Response is a map of user email -> role list.
+    Object.values(res.data ?? {}).forEach((roles: any) => {
+      (Array.isArray(roles) ? roles : []).forEach((role: any) => {
+        const key = String(role ?? "").trim();
+        if (!key) return;
+        counts[key] = (counts[key] ?? 0) + 1;
+      });
+    });
+    roleUserCounts.value = counts;
+  } catch {
+    // Silent: member counts are context. The list stays fully usable without them.
+    roleUserCounts.value = null;
+  }
+};
+
+// Patch the counts onto rows already on screen. null (not 0) while the map is
+// unavailable, so "unknown" and "nobody holds this role" stay distinguishable.
+const applyRoleUserCounts = () => {
+  const counts = roleUserCounts.value;
+  rolesState.roles = rolesState.roles.map((role: any) => ({
+    ...role,
+    user_count: counts ? (counts[role.role_name] ?? 0) : null,
+  }));
+  updateTable();
+};
+
 const setupRoles = async () => {
   loading.value = true;
   await getRoles(store.state.selectedOrganization.identifier)
     .then((res) => {
       rolesState.roles = res.data.map((role: string) => ({
         role_name: role,
+        user_count: null,
       }));
       updateTable();
+      // Fire-and-forget: the roles list renders immediately and the member counts
+      // (a second request) fill in when they land. Awaiting here would hold the
+      // whole table hostage to a secondary, enterprise-only endpoint.
+      void loadRoleUserCounts().then(applyRoleUserCounts);
     })
     .catch((err) => {
       console.log(err);

web/src/components/iam/roles/RoleTable.vue

@@ -1,8 +1,9 @@
 <!-- Copyright 2026 OpenObserve Inc. -->
 
 <script setup lang="ts">
+import { computed } from "vue";
 import OTable from "@/lib/core/Table/OTable.vue";
-import type { OTableColumnDef } from "@/lib/core/Table/OTable.types";
+import { COL, type OTableColumnDef } from "@/lib/core/Table/OTable.types";
 import OEmptyState from "@/lib/core/EmptyState/OEmptyState.vue";
 import OButton from "@/lib/core/Button/OButton.vue";
 import OIcon from "@/lib/core/Icon/OIcon.vue";
@@ -11,7 +12,7 @@ import { useI18n } from "vue-i18n";
 
 const { t } = useI18n();
 
-defineProps<{
+const props = defineProps<{
   data: any[];
   loading?: boolean;
   actionLoading?: boolean;
@@ -36,15 +37,37 @@ const onEmptyStateAction = (id?: string) => {
   if (id === "create") emit("create");
 };
 
-const columns: OTableColumnDef[] = [
-  {
-    id: "role_name",
-    header: t("iam.roleName"),
-    accessorKey: "role_name",
-    sortable: true,
-    meta: { align: "left", autoWidth: true, isName: true },
-  },
-  {
+// The Users column only exists when the caller could resolve member counts (the
+// batched user→roles map is enterprise-only) — an all-dash column would be noise.
+const hasUserCounts = computed(() =>
+  (props.data ?? []).some((row: any) => typeof row?.user_count === "number"),
+);
+
+const columns = computed<OTableColumnDef[]>(() => {
+  const cols: OTableColumnDef[] = [
+    {
+      id: "role_name",
+      header: t("iam.roleName"),
+      accessorKey: "role_name",
+      sortable: true,
+      meta: { align: "left", autoWidth: true, isName: true },
+    },
+  ];
+
+  if (hasUserCounts.value) {
+    cols.push({
+      id: "user_count",
+      header: t("iam.roleUsers"),
+      accessorKey: "user_count",
+      sortable: true,
+      resizable: true,
+      hideable: true,
+      size: COL.count,
+      meta: { align: "right" },
+    });
+  }
+
+  cols.push({
     id: "actions",
     header: t("common.actions"),
     isAction: true,
@@ -53,8 +76,15 @@ const columns: OTableColumnDef[] = [
     minSize: 64,
     maxSize: 100,
     meta: { align: "center", actionCount: 2 },
-  },
-];
+  });
+
+  return cols;
+});
+
+// A role nobody holds reads muted in the Users column — a cleanup candidate, not a
+// fault. No summary strip: the count is already in the footer and "unused" is just
+// this column sorted ascending, so a strip would restate what the rows already say.
+const isUnusedRole = (row: any): boolean => row?.user_count === 0;
 </script>
 
 <template>
@@ -90,6 +120,16 @@ const columns: OTableColumnDef[] = [
         />
       </div>
     </template>
+
+    <!-- Members: a role with nobody in it is muted, not coloured — it is a cleanup
+         candidate, not an error. -->
+    <template #cell-user_count="{ row }">
+      <span
+        class="tabular-nums"
+        :class="isUnusedRole(row) ? 'text-text-muted' : 'text-text-body'"
+        >{{ typeof row.user_count === "number" ? row.user_count : "—" }}</span
+      >
+    </template>
     <template #toolbar-trailing>
       <slot name="toolbar-trailing" />
     </template>

web/src/components/iam/serviceAccounts/ServiceAccountsList.vue

@@ -136,10 +136,25 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             />
           </template>
 
+          <!-- Relative age, not a raw timestamp: on a credentials list "3 days ago"
+               is the auditable fact, and recently minted accounts carry a dot so a
+               new key stands out without reading dates. -->
           <template #cell-created_at="{ row }">
-            <span :data-test="`service-accounts-created-${row.email}`" class="text-text-body">{{
-              formatCreatedAt(row.created_at)
-            }}</span>
+            <span
+              :data-test="`service-accounts-created-${row.email}`"
+              class="inline-flex min-w-0 items-center justify-end gap-1.5"
+            >
+              <span
+                v-if="isRecentlyCreated(row)"
+                class="bg-badge-teal-soft-text h-1.5 w-1.5 shrink-0 rounded-full"
+              />
+              <OTimeCell
+                :value="row.created_at"
+                unit="us"
+                mode="relative"
+                :timezone="store.state.timezone"
+              />
+            </span>
           </template>
 
           <template #cell-actions="{ row }">
@@ -435,6 +450,7 @@ import OTooltip from "@/lib/overlay/Tooltip/OTooltip.vue";
 import OTag from "@/lib/core/Badge/OTag.vue";
 import OCodeCell from "@/lib/core/Table/cells/OCodeCell.vue";
 import OUserCell from "@/lib/core/Table/cells/OUserCell.vue";
+import OTimeCell from "@/lib/core/Table/cells/OTimeCell.vue";
 import { useStore } from "vuex";
 import { useRouter } from "vue-router";
 import { useI18n } from "vue-i18n";
@@ -483,6 +499,7 @@ export default defineComponent({
     OTag,
     OCodeCell,
     OUserCell,
+    OTimeCell,
     OSearchInput,
     OTabs,
     OTab,
@@ -834,9 +851,18 @@ export default defineComponent({
       }
     };
 
-    // created_at arrives as epoch microseconds (chrono timestamp_micros on the
-    // backend). Render it the same way the Alerts list does: a readable
-    // "YYYY-MM-DD HH:mm:ss" string. Falsy/zero values show an em dash.
+    // ── Account age (created_at is epoch MICROseconds) ───────────────────────
+    // A key minted in the last week is the audit-relevant one, so it gets a dot in
+    // the Created column and its own tile in the strip.
+    const RECENT_ACCOUNT_MS = 7 * 24 * 60 * 60 * 1000;
+    const isRecentlyCreated = (row: any): boolean => {
+      const micros = Number(row?.created_at);
+      if (!micros || !Number.isFinite(micros) || micros <= 0) return false;
+      return Date.now() - micros / 1000 <= RECENT_ACCOUNT_MS;
+    };
+
+    // Kept for the tests and any caller that still wants the absolute string; the
+    // Created column itself now renders a relative OTimeCell.
     const formatCreatedAt = (createdAt: number): string => {
       if (!createdAt) return "—";
       const iso = new Date(createdAt / 1000).toISOString();
@@ -1128,6 +1154,7 @@ export default defineComponent({
       bulkDeleteServiceAccounts,
       redactToken,
       formatCreatedAt,
+      isRecentlyCreated,
       downloadTokenAsFile,
       isSystemAccount,
       isRowSelectable,

web/src/components/iam/users/User.vue

@@ -40,7 +40,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         <OTable
           :key="tableKey"
           :frame="false"
-          :data="rows"
+          :data="displayedRows"
           :columns="columns"
           row-key="email"
           :loading="loading"
@@ -62,6 +62,26 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
           table-id="iam-users-list"
           @update:selected-ids="handleSelectedIdsUpdate"
         >
+          <!-- Access screens have no health state, so the strip counts ROLE
+               MEMBERSHIP and doubles as the role facet: the org's biggest roles
+               first, the tail folded into one tile, Total last and never
+               highlighted. Deliberately no green/amber/red — a role is a category,
+               not a severity. -->
+          <template #subheader>
+            <div
+              class="px-page-edge border-table-row-divider border-b py-1.5"
+              data-test="user-list-summary"
+            >
+              <OStatStrip
+                :items="summaryStats"
+                :loading="loading"
+                selectable
+                :selected-key="roleFilter"
+                @select="onStatSelect"
+              />
+            </div>
+          </template>
+
           <template #toolbar>
             <div class="flex w-full items-center gap-2">
               <OSearchInput
@@ -92,8 +112,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             <OEmptyState
               size="hero"
               preset="no-users"
-              :filtered="!!filterQuery"
-              @action="(id) => (id === 'clear-filters' ? (filterQuery = '') : addRoutePush({}))"
+              :filtered="!!(filterQuery || roleFilter)"
+              @action="
+                (id) =>
+                  id === 'clear-filters'
+                    ? ((filterQuery = ''), (roleFilter = null))
+                    : addRoutePush({})
+              "
             />
           </template>
 
@@ -103,16 +128,22 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             <span v-else class="text-text-body">—</span>
           </template>
 
-          <!-- Roles badges — typed userRole tags for built-in roles, custom
-               roles keep their original casing via an untyped tag. -->
+          <!-- Roles badges — built-in roles carry the privilege colour from the
+               userRole group; a custom role is a category, not a privilege level, so
+               it gets one stable neutral chip rather than an untyped tag whose
+               colour would be derived from the role's spelling. The invited chip
+               rides along here too: pending users are otherwise indistinguishable
+               on this path. -->
           <template #cell-roles="{ row }">
             <div class="flex flex-wrap items-center gap-1">
               <OTag
                 v-for="(roleName, idx) in row.roles || []"
                 :key="`${roleName}-${idx}`"
                 :type="isBuiltinRole(roleName) ? 'userRole' : undefined"
+                :variant="isBuiltinRole(roleName) ? undefined : 'default-soft'"
                 :value="roleName"
               />
+              <OTag v-if="row.status === 'pending'" type="userStatus" value="invited" />
             </div>
           </template>
 
@@ -252,6 +283,9 @@ import { defineComponent, ref, onActivated, onBeforeMount, watch } from "vue";
 import OButton from "@/lib/core/Button/OButton.vue";
 import OTooltip from "@/lib/overlay/Tooltip/OTooltip.vue";
 import OTag from "@/lib/core/Badge/OTag.vue";
+import OStatStrip from "@/lib/data/StatStrip/OStatStrip.vue";
+import type { StatItem } from "@/lib/data/StatStrip/OStatStrip.types";
+import type { IconName } from "@/lib/core/Icon/OIcon.icons";
 import ODialog from "@/lib/overlay/Dialog/ODialog.vue";
 import OPageLayout from "@/lib/core/PageLayout/OPageLayout.vue";
 import OTable from "@/lib/core/Table/OTable.vue";
@@ -291,6 +325,7 @@ export default defineComponent({
     OButton,
     OTooltip,
     OTag,
+    OStatStrip,
     OIcon,
     ODialog,
     OEmptyState,
@@ -466,6 +501,188 @@ export default defineComponent({
       "serviceaccount",
     ]);
     const isBuiltinRole = (r: string) => BUILTIN_ROLES.has(String(r ?? "").toLowerCase());
+
+    // ── Role tiles — the page's primary signal ──────────────────────────────
+    // A user can hold SEVERAL roles (built-in + custom), so the strip counts role
+    // MEMBERSHIP, not buckets: one Admin+custom user is counted under both tiles.
+    // Tiles therefore do not sum to the user total — each is "how many users hold
+    // this role", and its bar is that share of all users, which stays meaningful
+    // under overlap. Roles come from the data, so custom roles appear as first-class
+    // tiles without being enumerated anywhere.
+    const PRIVILEGE_RANK: Record<string, number> = {
+      root: 5,
+      admin: 4,
+      editor: 3,
+      member: 3,
+      user: 2,
+      viewer: 2,
+      serviceaccount: 1,
+    };
+    // Roles are unbounded (an org can define dozens), and a strip is a fixed row of
+    // tiles — past this many the strip stops being scannable, so the tail collapses
+    // into one "Other roles" tile that still filters. Five leaves room for that
+    // tile plus Invited and Total without the strip wrapping on a laptop.
+    const MAX_ROLE_TILES = 5;
+
+    const userRoles = (row: any): string[] => {
+      const roles = Array.isArray(row?.roles) && row.roles.length ? row.roles : [row?.role];
+      const seen = new Set<string>();
+      // Dedupe per user: a role listed twice must not count twice.
+      return roles
+        .filter(Boolean)
+        .map((r: any) =>
+          String(r)
+            .replace(/\s*\(Invited\)\s*$/i, "")
+            .trim(),
+        )
+        .filter((r: string) => {
+          const key = r.toLowerCase();
+          if (!r || seen.has(key)) return false;
+          seen.add(key);
+          return true;
+        });
+    };
+    const hasRole = (row: any, roleKey: string): boolean =>
+      userRoles(row).some((r) => r.toLowerCase() === roleKey);
+    const isInvited = (row: any): boolean => row?.status === "pending";
+
+    // role key -> { label (original casing), count of users holding it }
+    const roleTally = computed(() => {
+      const tally = new Map<string, { label: string; count: number }>();
+      for (const row of rows.value || []) {
+        for (const role of userRoles(row)) {
+          const key = role.toLowerCase();
+          const entry = tally.get(key);
+          if (entry) entry.count += 1;
+          else tally.set(key, { label: role, count: 1 });
+        }
+      }
+      return tally;
+    });
+
+    // BIGGEST roles first, so the tiles are the org's actual top roles rather than
+    // whichever ones happen to outrank the others: a two-user built-in must not
+    // take a slot from a role half the org holds. Privilege breaks ties (Admin
+    // before Viewer at equal size), then the name for stability.
+    const rankedRoles = computed(() => {
+      return Array.from(roleTally.value.entries())
+        .map(([key, entry]) => ({ key, ...entry, rank: PRIVILEGE_RANK[key] ?? 0 }))
+        .sort((a, b) => b.count - a.count || b.rank - a.rank || a.key.localeCompare(b.key));
+    });
+    const visibleRoles = computed(() => rankedRoles.value.slice(0, MAX_ROLE_TILES));
+    const overflowRoles = computed(() => rankedRoles.value.slice(MAX_ROLE_TILES));
+    // Unique users holding ANY overflow role — summing the tail would double-count
+    // anyone holding two of them.
+    const overflowUserCount = computed(() => {
+      const keys = new Set(overflowRoles.value.map((r) => r.key));
+      if (!keys.size) return 0;
+      return (rows.value || []).filter((row: any) =>
+        userRoles(row).some((r) => keys.has(r.toLowerCase())),
+      ).length;
+    });
+
+    // Built-in roles keep the privilege ramp; every custom role reads teal so it is
+    // visibly "not a built-in" without each one inventing its own colour.
+    const roleTone = (key: string): StatItem["tone"] => {
+      if (key === "root" || key === "admin") return "orange";
+      if (key === "editor" || key === "member") return "blue";
+      if (key === "user" || key === "viewer") return "neutral";
+      if (key === "serviceaccount") return "purple";
+      return "teal";
+    };
+    const roleIcon = (key: string): IconName => {
+      if (key === "root" || key === "admin") return "admin-panel-settings";
+      if (key === "editor" || key === "member") return "edit";
+      if (key === "user" || key === "viewer") return "visibility";
+      if (key === "serviceaccount") return "key";
+      return "manage-accounts";
+    };
+
+    // ── Role facet + summary strip ──────────────────────────────────────────
+    // Counts run over the full row set (not the facet-filtered one) so the tiles
+    // keep their totals while a facet is active.
+    const OVERFLOW_KEY = "__other_roles__";
+    const INVITED_KEY = "__invited__";
+    const roleFilter = ref<string | null>(null);
+
+    const displayedRows = computed(() => {
+      const all = rows.value || [];
+      const f = roleFilter.value;
+      if (!f) return all;
+      if (f === INVITED_KEY) return all.filter((row: any) => isInvited(row));
+      if (f === OVERFLOW_KEY) {
+        const keys = new Set(overflowRoles.value.map((r) => r.key));
+        return all.filter((row: any) => userRoles(row).some((r) => keys.has(r.toLowerCase())));
+      }
+      return all.filter((row: any) => hasRole(row, f));
+    });
+
+    const onStatSelect = (key: string) => {
+      if (key === "total") {
+        roleFilter.value = null;
+        return;
+      }
+      roleFilter.value = roleFilter.value === key ? null : key;
+    };
+
+    const invitedCount = computed(
+      () => (rows.value || []).filter((row: any) => isInvited(row)).length,
+    );
+
+    const summaryStats = computed<StatItem[]>(() => {
+      const total = (rows.value || []).length;
+      const hasData = total > 0;
+      const v = (n: number): string | number => (hasData ? n : "—");
+      const share = hasData ? total : undefined;
+
+      const tiles: StatItem[] = visibleRoles.value.map((role) => ({
+        key: role.key,
+        // The label is the role name itself — data, not a translatable string.
+        label: role.label,
+        value: v(role.count),
+        icon: roleIcon(role.key),
+        tone: roleTone(role.key),
+        max: share,
+        dataTest: `user-summary-role-${role.key}`,
+      }));
+
+      if (overflowRoles.value.length) {
+        tiles.push({
+          key: OVERFLOW_KEY,
+          label: t("iam.summaryOtherRoles", { count: overflowRoles.value.length }),
+          value: v(overflowUserCount.value),
+          icon: "more-horiz",
+          tone: "teal",
+          max: share,
+          dataTest: "user-summary-other-roles",
+        });
+      }
+
+      // Invited is a lifecycle state, not a privilege, so it sits after the roles.
+      if (invitedCount.value > 0) {
+        tiles.push({
+          key: INVITED_KEY,
+          label: t("iam.summaryInvited"),
+          value: v(invitedCount.value),
+          icon: "person-add",
+          tone: "purple",
+          max: share,
+          dataTest: "user-summary-invited",
+        });
+      }
+
+      tiles.push({
+        key: "total",
+        label: t("iam.summaryTotalUsers"),
+        value: v(total),
+        icon: "group-work",
+        tone: "primary",
+        // Clickable (it clears the facet) but never shows the ring.
+        dataTest: "user-summary-total",
+      });
+      return tiles;
+    });
+
     const userEmail: any = ref("");
     const options = ref<{ label: string; value: string }[]>([]);
     const customRoles = ref<string[]>([]);
@@ -1235,20 +1452,13 @@ export default defineComponent({
       openBulkDeleteDialog,
       bulkDeleteUsers,
       rows,
+      displayedRows,
+      roleFilter,
+      onStatSelect,
+      summaryStats,
       tableKey,
       // showAddUserBtn,
     };
   },
 });
 </script>
-
-<style scoped>
-/* keep(lib-override): compact role chip styling (child OTag DOM) */
-:deep(.o2-role-chip) {
-  padding: 0.125rem 0.5rem;
-  font-size: var(--text-2xs);
-  font-weight: 600;
-  border-radius: 0.375rem;
-  line-height: 1.4;
-}
-</style>

web/src/components/synthetic-monitoring/MonitorTable.vue

@@ -33,9 +33,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     :empty-message="emptyMessage"
     :data-test="dataTest"
     :horizontal-scroll="true"
+    :row-class="monitorRowClass"
+    :get-row-style="monitorRowStyle"
     show-index
     @row-click="(row: any) => emit('row-click', row)"
   >
+    <!-- Summary strip supplied by the parent (it owns the status facet). -->
+    <template v-if="$slots.subheader" #subheader>
+      <slot name="subheader" />
+    </template>
+
     <!-- Toolbar slots (passthrough to parent) -->
     <template #toolbar>
       <slot name="toolbar" />
@@ -120,17 +127,17 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
     <!-- History sparkbars -->
     <template #cell-history="{ row }">
-      <div class="spark">
+      <div class="flex h-5 items-end gap-px">
         <span
           v-for="(tick, i) in (row as any).history"
           :key="i"
-          class="spark-bar"
+          class="rounded-default h-full w-1 shrink-0 cursor-pointer"
           :class="
             tick.status === 'up'
-              ? 'bg-[var(--color-success-500)]'
+              ? 'bg-success-500'
               : tick.status === 'down'
-                ? 'bg-[var(--color-error-500)]'
-                : 'bg-[var(--color-warning-500)]'
+                ? 'bg-error-500'
+                : 'bg-warning-500'
           "
           @mouseenter="showSparkTip($event, tick)"
           @mouseleave="hideSparkTip"
@@ -145,10 +152,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         class="font-mono text-sm font-semibold"
         :class="
           parseFloat((row as any).responseTime) < 300
-            ? 'text-[var(--color-success-600)]'
+            ? 'text-status-success-text'
             : parseFloat((row as any).responseTime) < 1000
-              ? 'text-[var(--color-warning-600)]'
-              : 'text-[var(--color-error-600)]'
+              ? 'text-status-warning-text'
+              : 'text-status-error-text'
         "
         >{{ (row as any).responseTime }}</span
       >
@@ -175,10 +182,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             :class="
               'min-w-11 text-right font-mono text-sm text-xs font-semibold ' +
               ((row as any).uptime >= 99
-                ? 'text-[var(--color-success-600)]'
+                ? 'text-status-success-text'
                 : (row as any).uptime >= 95
-                  ? 'text-[var(--color-warning-600)]'
-                  : 'text-[var(--color-error-600)]')
+                  ? 'text-status-warning-text'
+                  : 'text-status-error-text')
             "
             >{{ (row as any).uptime }}%</span
           >
@@ -201,7 +208,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
           }}</span>
           <span
             v-if="(row as any).locations.length > 1"
-            class="rounded-default shrink-0 bg-[var(--color-surface-subtle)] px-1 py-0.5 text-xs font-bold whitespace-nowrap"
+            class="rounded-default bg-surface-subtle shrink-0 px-1 py-0.5 text-xs font-bold whitespace-nowrap"
             >+{{ (row as any).locations.length - 1 }}</span
           >
         </div>
@@ -215,8 +222,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     </template>
 
     <!-- Last check -->
+    <!-- Relative age from the RAW microsecond timestamp — the same renderer every
+         other table uses, so "3m ago" reads identically across the app. -->
     <template #cell-lastCheck="{ row }">
-      <span class="truncate">{{ (row as any).lastCheck || "—" }}</span>
+      <OTimeCell :value="(row as any).lastCheckAt" unit="us" mode="relative" :timezone="timezone" />
     </template>
 
     <!-- Row actions -->
@@ -419,14 +428,25 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
   <Teleport to="body">
     <div
       v-if="sparkTip.show && sparkTip.tick"
-      class="spark-tooltip"
+      class="rounded-surface bg-surface-overlay border-border-default text-text-body pointer-events-auto fixed z-50 flex w-56 flex-col gap-2 border p-3 text-xs shadow-lg"
       :style="{ left: sparkTip.x + 'px', top: sparkTip.y + 'px' }"
       @mouseenter="keepSparkTip"
       @mouseleave="hideSparkTip"
     >
-      <div class="stt-header">
-        <span class="stt-time">{{ sparkTip.tick.hour }} – {{ sparkTip.tick.nextHour }}</span>
-        <span class="stt-badge" :class="'stt-badge--' + sparkTip.tick.status">
+      <div class="flex items-center justify-between gap-2">
+        <span class="text-text-secondary"
+          >{{ sparkTip.tick.hour }} – {{ sparkTip.tick.nextHour }}</span
+        >
+        <OBadge
+          size="sm"
+          :variant="
+            sparkTip.tick.status === 'up'
+              ? 'success-soft'
+              : sparkTip.tick.status === 'down'
+                ? 'error-soft'
+                : 'warning-soft'
+          "
+        >
           {{
             sparkTip.tick.status === "up"
               ? t("synthetics.table.statusUp")
@@ -434,22 +454,24 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                 ? t("synthetics.table.statusDown")
                 : t("synthetics.table.statusDegraded")
           }}
-        </span>
+        </OBadge>
       </div>
-      <div class="stt-divider" />
-      <div class="stt-checks">
-        <div v-for="c in sparkTip.tick.checks" :key="c.loc" class="stt-check">
-          <span class="stt-dot" :class="c.ok ? 'stt-dot--up' : 'stt-dot--down'" />
-          <span class="stt-loc">{{ c.loc }}</span>
-          <span class="stt-ms">{{
+      <div class="border-border-default border-t" />
+      <div class="flex flex-col gap-1">
+        <div v-for="c in sparkTip.tick.checks" :key="c.loc" class="flex items-center gap-1.5">
+          <span
+            class="h-1.5 w-1.5 shrink-0 rounded-full"
+            :class="c.ok ? 'bg-success-500' : 'bg-error-500'"
+          />
+          <span class="min-w-0 flex-1 truncate">{{ c.loc }}</span>
+          <span class="text-text-secondary shrink-0 tabular-nums">{{
             c.ms !== null ? c.ms + t("synthetics.table.ms") : t("synthetics.table.timeout")
           }}</span>
         </div>
       </div>
-      <div v-if="sparkTip.tick.avgMs !== null" class="stt-avg">
+      <div v-if="sparkTip.tick.avgMs !== null" class="text-text-secondary tabular-nums">
         {{ t("synthetics.table.avg") }} · {{ sparkTip.tick.avgMs }}{{ t("synthetics.table.ms") }}
       </div>
-      <div class="stt-arrow" />
     </div>
   </Teleport>
 </template>
@@ -471,14 +493,47 @@ import ODropdownSeparator from "@/lib/overlay/Dropdown/ODropdownSeparator.vue";
 import OTooltip from "@/lib/overlay/Tooltip/OTooltip.vue";
 import OEmptyState from "@/lib/core/EmptyState/OEmptyState.vue";
 import { resolveBadge } from "@/lib/core/Badge/badgeGroups";
+import OTimeCell from "@/lib/core/Table/cells/OTimeCell.vue";
 
 type Mode = "all" | "browser";
 
+// Extreme-left health rail — inset box-shadow so it paints regardless of
+// border-collapse; rem width + token colour keep it theme-aware. Same vocabulary
+// as Alerts and Pipelines: red = failing, amber = degrading, green = passing,
+// grey = never reported.
+// Full-row wash for the two rows that are not "just not green":
+//   failing  → light red, because a failing check is the reason this page exists
+//              and it is rare in a healthy estate;
+//   paused   → muted grey, de-emphasis rather than alarm.
+// A DEGRADED monitor keeps a clean row and reads from the amber rail: it is worth
+// noticing, not worth acting on this second, and washing it too would put most of
+// a wobbly estate in colour.
+const monitorRowClass = (row: any): string => {
+  if (String(row?.status ?? "").toLowerCase() === "failed") return "!bg-status-error-bg";
+  return row?.enabled === false ? "!bg-surface-panel" : "";
+};
+
+const monitorRowStyle = (row: any): Record<string, string> => {
+  const status = String(row?.status ?? "").toLowerCase();
+  const color =
+    status === "failed"
+      ? "var(--color-error-500)"
+      : status === "warning"
+        ? "var(--color-warning-500)"
+        : status === "passed"
+          ? "var(--color-success-500)"
+          : "var(--color-grey-400)";
+  return { boxShadow: `inset 0.25rem 0 0 0 ${color}` };
+};
+
 const props = withDefaults(
   defineProps<{
     mode: Mode;
     data: any[];
     loading?: boolean;
+    /** IANA zone for the Last Check tooltip. Passed in: this table is a leaf
+     *  component and must not reach into the store for it. */
+    timezone?: string;
     footerTitle?: string;
     emptyMessage?: string;
     dataTest?: string;

web/src/composables/iam/usePermissions.ts

@@ -23,7 +23,7 @@ const groups = {
 };
 
 const roles = {
-  roles: [],
+  roles: [] as any[],
 };
 
 const permissions = {

web/src/lib/navigation/Tabs/OTab.types.ts

@@ -12,6 +12,12 @@ export interface OTabProps {
   icon?: string;
   /** Prevents interaction with this tab */
   disable?: boolean;
+  /**
+   * Opt this single tab out of drag-to-reorder even while OTabs is reorderable
+   * (e.g. its label is being renamed inline). The grip stays visible but the tab
+   * is no longer draggable and shows a text cursor instead of grab.
+   */
+  disableDrag?: boolean;
   /** Tooltip shown on hover — especially useful when disable is true to explain why */
   tooltip?: string;
 }

web/src/lib/navigation/Tabs/OTab.vue

@@ -15,6 +15,7 @@ defineOptions({ inheritAttrs: false });
 
 const props = withDefaults(defineProps<OTabProps>(), {
   disable: false,
+  disableDrag: false,
 });
 
 defineSlots<OTabSlots>();
@@ -25,6 +26,8 @@ const isActive = computed<boolean>(() => context?.value.modelValue === props.nam
 const isDense = computed<boolean>(() => context?.value.dense ?? false);
 const isVertical = computed<boolean>(() => context?.value.isVertical ?? false);
 const isReorderable = computed<boolean>(() => context?.value.reorderable ?? false);
+/** Reorderable, and this tab hasn't opted out (e.g. while its label is edited). */
+const isDraggable = computed<boolean>(() => isReorderable.value && !props.disableDrag);
 /** This tab is the one being dragged → dim it. */
 const isDragging = computed<boolean>(
   () => isReorderable.value && context?.value.draggingName === props.name,
@@ -143,10 +146,11 @@ const heightClasses = computed<string>(() => {
         baseClasses,
         stateClasses,
         heightClasses,
-        isReorderable ? 'cursor-grab active:cursor-grabbing' : '',
+        isDraggable ? 'cursor-grab active:cursor-grabbing' : '',
+        isReorderable && disableDrag ? 'cursor-text' : '',
         isDragging ? 'opacity-40' : '',
       ]"
-      :draggable="isReorderable || undefined"
+      :draggable="isDraggable || undefined"
       :data-otab-name="name"
       v-bind="$attrs"
     >

web/src/lib/navigation/Tabs/OTabs.vue

@@ -99,11 +99,74 @@ function onTabDrop(e: DragEvent): void {
   const from = draggingName.value ?? e.dataTransfer?.getData("text/plain") ?? null;
   const to = dropTargetName.value;
   if (from != null && to != null && from !== to) {
+    // FLIP: sample the current tab positions BEFORE the parent applies the
+    // reorder, then slide each tab from its old slot to its new one once the
+    // DOM has patched — so the drop settles with motion instead of a jump.
+    captureFlipFirst();
     emit("reorder", { from, to, before: dropBefore.value });
+    nextTick(() => playFlip());
   }
   clearDrag();
 }
 
+// ── FLIP reorder animation ─────────────────────────────────────────────────
+// The parent owns the list, so OTabs can't reorder the DOM itself — it only
+// animates whatever move the parent applies in response to `reorder`. "First"
+// rects are captured just before we emit; "Last" rects after Vue patches the
+// DOM on the next tick. Each moved tab is offset back to its old x with no
+// transition, then released to slide to identity. Purely visual; if the parent
+// declines the move (no DOM change) every delta is 0 and this is a no-op.
+const flipFirst = new Map<string | number, number>();
+
+function tabButtons(): HTMLElement[] {
+  const list = tablistRef.value;
+  if (!list) return [];
+  return Array.from(list.querySelectorAll<HTMLElement>("[data-otab-name]"));
+}
+
+function captureFlipFirst(): void {
+  flipFirst.clear();
+  for (const el of tabButtons()) {
+    const name = el.dataset.otabName;
+    if (name != null) flipFirst.set(name, el.getBoundingClientRect().left);
+  }
+}
+
+function playFlip(): void {
+  if (flipFirst.size === 0) return;
+  const moved: HTMLElement[] = [];
+  for (const el of tabButtons()) {
+    const name = el.dataset.otabName;
+    const first = name != null ? flipFirst.get(name) : undefined;
+    if (first == null) continue;
+    const dx = first - el.getBoundingClientRect().left;
+    if (!dx) continue;
+    el.style.transition = "none";
+    el.style.transform = `translateX(${dx}px)`;
+    moved.push(el);
+  }
+  flipFirst.clear();
+  if (moved.length === 0) return;
+  // Commit the inverted starting transforms before releasing them.
+  void tablistRef.value?.offsetWidth;
+  requestAnimationFrame(() => {
+    for (const el of moved) {
+      el.style.transition = "transform 250ms cubic-bezier(0.2, 0, 0, 1)";
+      el.style.transform = "";
+    }
+  });
+}
+
+function onTabTransitionEnd(e: TransitionEvent): void {
+  // Clear the inline FLIP styles once the slide finishes so nothing lingers on
+  // the tab (and a later reorder starts from a clean transform).
+  if (e.propertyName !== "transform") return;
+  const el = (e.target as HTMLElement | null)?.closest<HTMLElement>("[data-otab-name]");
+  if (!el) return;
+  el.style.transition = "";
+  el.style.transform = "";
+}
+
 function onTabDragEnd(): void {
   clearDrag();
 }
@@ -283,6 +346,7 @@ const alignClasses: Record<NonNullable<OTabsProps["align"]>, string> = {
         @dragover="onTabDragOver"
         @drop="onTabDrop"
         @dragend="onTabDragEnd"
+        @transitionend="onTabTransitionEnd"
       >
         <slot />
       </div>
@@ -328,6 +392,7 @@ const alignClasses: Record<NonNullable<OTabsProps["align"]>, string> = {
             @dragover="onTabDragOver"
             @drop="onTabDrop"
             @dragend="onTabDragEnd"
+            @transitionend="onTabTransitionEnd"
           >
             <!-- Single shared underline — slides (translateX + width) to the
                  active tab instead of each tab drawing its own border. -->

web/src/locales/languages/en-US.json

@@ -1649,6 +1649,9 @@
     "addGroup": "New user group",
     "addRole": "New role",
     "roleName": "Role Name",
+    "roleUsers": "Users",
+    "summaryInvited": "Invited",
+    "summaryTotalUsers": "Users",
     "groupName": "Group Name",
     "permissionName": "Permission Name",
     "permission": "Permission",
@@ -1934,7 +1937,8 @@
     },
     "addRolePage": {
       "roleCreated": "Role \"{name}\" Created Successfully!"
-    }
+    },
+    "summaryOtherRoles": "+{count} more roles"
   },
   "ticket": {
     "header": "Support Tickets",
@@ -9832,7 +9836,6 @@
       "actionClick": "click",
       "actionType": "type",
       "all": "All",
-      "allStatuses": "All Statuses",
       "allBrowsers": "All browsers",
       "allDevices": "All devices",
       "allLocations": "All locations",
@@ -10451,7 +10454,7 @@
       "passRateByLocation": "Pass Rate by Location",
       "durationByLocation": "Duration by Location (Avg)",
       "flakyRate": "Flaky Rate",
-      "stepsWindowTruncated": "Showing the newest {count} executions ({from} \u2013 {to}); the selected range holds more.",
+      "stepsWindowTruncated": "Showing the newest {count} executions ({from} – {to}); the selected range holds more.",
       "retryRate": "Retry Rate",
       "statusTimeline": "Status Timeline",
       "tabOverview": "Overview",

web/src/plugins/logs/SearchResult.vue

@@ -1322,7 +1322,7 @@ export default defineComponent({
     const correlationDashboardProps = ref<any>(null);
     const correlationLoading = ref(false);
     const correlationError = ref<string | null>(null);
-    const detailTableInitialTab = ref<string>("json");
+    const detailTableInitialTab = ref<string>("table");
     const { findRelatedTelemetry, semanticGroups } = useServiceCorrelation();
 
     // Flag to prevent duplicate correlation API calls
@@ -1622,7 +1622,7 @@ export default defineComponent({
     const openLogDetails = (props: any, index: number) => {
       searchObj.meta.showDetailTab = true;
       searchObj.meta.resultGrid.navigation.currentRowIndex = index;
-      detailTableInitialTab.value = "json"; // Reset to default tab
+      detailTableInitialTab.value = "table"; // Reset to default tab (#13368: Table is the default log-detail view)
 
       // Prepare correlation context (but don't open panel automatically)
       const logData = searchObj.data.queryResults?.hits?.[index];

web/src/views/Dashboards/ScheduledDashboards.spec.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, vi } from "vitest";
-import { mount } from "@vue/test-utils";
+import { mount, flushPromises } from "@vue/test-utils";
 import { createI18n } from "vue-i18n";
 import ScheduledDashboards from "./ScheduledDashboards.vue";
 
@@ -334,14 +334,23 @@ describe("ScheduledDashboards", () => {
       },
     ];
 
-    it("should format reports correctly", () => {
+    it("should format reports correctly", async () => {
       const wrapper = createWrapper({ reports: mockReports });
+      await flushPromises();
 
-      // Component processes reports multiple times due to watchers, resulting in more items than input
-      expect(wrapper.props("reports")).toHaveLength(4);
+      // props are never mutated — formatReports builds a new array instead of
+      // pushing onto the aliased prop, so no duplication occurs
+      expect(wrapper.props("reports")).toHaveLength(2);
       expect(wrapper.props("reports")[0].name).toBe("Test Report 1");
       expect(wrapper.props("reports")[1].name).toBe("Test Report 2");
-      expect(wrapper.exists()).toBe(true);
+
+      // default "cached" tab shows only the report without destinations, formatted
+      const table = wrapper.findComponent({ name: "OTable" });
+      const rows = table.props("data") as any[];
+      expect(rows).toHaveLength(1);
+      expect(rows[0].name).toBe("Test Report 1");
+      expect(rows[0].tab).toBe("Test Tab");
+      expect(rows[0].isCached).toBe(true);
     });
 
     it("should handle frequency formatting for different types", () => {
@@ -388,12 +397,20 @@ describe("ScheduledDashboards", () => {
       },
     ];
 
-    it("should filter cached reports by default", () => {
+    it("should filter cached reports by default", async () => {
       const wrapper = createWrapper({ reports: mockReports });
+      await flushPromises();
 
       expect(wrapper.exists()).toBe(true);
-      // Component processes reports multiple times due to watchers, resulting in more items than input
-      expect(wrapper.props("reports")).toHaveLength(4);
+      // props are not mutated (no duplication)
+      expect(wrapper.props("reports")).toHaveLength(2);
+
+      // default "cached" tab shows only the report without destinations
+      const table = wrapper.findComponent({ name: "OTable" });
+      const rows = table.props("data") as any[];
+      expect(rows).toHaveLength(1);
+      expect(rows[0].name).toBe("Cached Report");
+      expect(rows[0].isCached).toBe(true);
     });
 
     it("should handle tab changes", () => {
@@ -414,7 +431,7 @@ describe("ScheduledDashboards", () => {
       expect(appTabs.exists()).toBe(true);
     });
 
-    it("should display correct reports count", () => {
+    it("should display correct reports count", async () => {
       const mockReports = [
         {
           name: "Report 1",
@@ -443,9 +460,14 @@ describe("ScheduledDashboards", () => {
       ];
 
       const wrapper = createWrapper({ reports: mockReports });
+      await flushPromises();
 
-      // Component processes reports multiple times due to watchers: 3 input -> 6 processed
-      expect(wrapper.props("reports")).toHaveLength(6);
+      // props are not mutated — 3 in, 3 out (no duplication)
+      expect(wrapper.props("reports")).toHaveLength(3);
+
+      // all three reports are cached, so all show under the default "cached" tab
+      const table = wrapper.findComponent({ name: "OTable" });
+      expect(table.props("data")).toHaveLength(3);
     });
   });
 

web/src/views/Dashboards/ScheduledDashboards.vue

@@ -214,23 +214,20 @@ onMounted(() => {
 });
 
 const formatReports = () => {
-  props.reports.length > 0 &&
-    props.reports.forEach((report: any) => {
-      scheduledReports.value.push({
-        name: report.name,
-        tab: getTabName(report.dashboards?.[0]?.tabs?.[0]),
-        time_range: getTimeRangeValue(report.dashboards?.[0]?.timerange),
-        frequency: getFrequencyValue(report.frequency),
-        last_triggered_at_raw: report.last_triggered_at || null,
-        last_triggered_at: report.last_triggered_at
-          ? convertUnixToDateFormat(report.last_triggered_at)
-          : "-",
-        created_at_raw: report.created_at || null,
-        created_at: convertUnixToDateFormat(report.created_at),
-        orgId: report.org_id,
-        isCached: !report?.destinations?.length,
-      });
-    });
+  scheduledReports.value = props.reports.map((report: any) => ({
+    name: report.name,
+    tab: getTabName(report.dashboards?.[0]?.tabs?.[0]),
+    time_range: getTimeRangeValue(report.dashboards?.[0]?.timerange),
+    frequency: getFrequencyValue(report.frequency),
+    last_triggered_at_raw: report.last_triggered_at || null,
+    last_triggered_at: report.last_triggered_at
+      ? convertUnixToDateFormat(report.last_triggered_at)
+      : "-",
+    created_at_raw: report.created_at || null,
+    created_at: convertUnixToDateFormat(report.created_at),
+    orgId: report.org_id,
+    isCached: !report?.destinations?.length,
+  }));
 
   filterReports();
 };

web/src/views/Dashboards/ViewDashboard.vue

@@ -1619,8 +1619,11 @@ export default defineComponent({
       scheduledReports.value = [];
       isLoadingReports.value = true;
 
+      // folder_id is intentionally omitted here: it filters by the REPORT's own
+      // folder, not the dashboard's folder, so passing the dashboard folder id
+      // would incorrectly exclude reports saved to a different report folder.
       reports
-        .list(store.state.selectedOrganization.identifier, folderId.value, dashboardId.value)
+        .list(store.state.selectedOrganization.identifier, "", dashboardId.value)
         .then((response) => {
           scheduledReports.value = response.data;
         })

web/src/views/SyntheticMonitoring.vue

@@ -93,6 +93,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
           :mode="monitorTableMode"
           :data="filteredMonitors"
           :loading="loading"
+          :timezone="store.state.timezone"
           :footer-title="footerTitle"
           :empty-message="emptyMessage"
           :selected-ids="selectedMonitorIds"
@@ -124,6 +125,24 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
           "
           @empty-action="onEmptyAction"
         >
+          <!-- Health strip: the status facet, attention-first (failing → degrading
+               → passing → unknown), with Total last. It replaces the status dropdown
+               that used to hide these same counts inside its option labels. -->
+          <template #subheader>
+            <div
+              class="px-page-edge border-table-row-divider border-b py-1.5"
+              data-test="synthetic-monitoring-summary"
+            >
+              <OStatStrip
+                :items="summaryStats"
+                :loading="loading"
+                selectable
+                :selected-key="statusFilter === 'all' ? null : statusFilter"
+                @select="onStatSelect"
+              />
+            </div>
+          </template>
+
           <!-- Toolbar content rendered inside OTable's toolbar bar -->
           <template #toolbar>
             <div class="flex min-w-0 flex-1 items-center gap-2">
@@ -184,8 +203,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                 </OInput>
               </div>
 
-              <!-- Status filter -->
-              <OSelect v-model="statusFilter" :options="statusOpts" size="md" class="w-35!" />
+              <!-- Status is faceted by the summary strip below, not by a dropdown -
+                   the counts belong on screen, not hidden inside option labels. -->
             </div>
           </template>
 
@@ -330,7 +349,6 @@ import { useStore } from "vuex";
 import OPageLayout from "@/lib/core/PageLayout/OPageLayout.vue";
 import OIcon from "@/lib/core/Icon/OIcon.vue";
 import OButton from "@/lib/core/Button/OButton.vue";
-import OSelect from "@/lib/forms/Select/OSelect.vue";
 import OInput from "@/lib/forms/Input/OInput.vue";
 import OTabs from "@/lib/navigation/Tabs/OTabs.vue";
 import OTab from "@/lib/navigation/Tabs/OTab.vue";
@@ -360,6 +378,8 @@ import {
 } from "@/types/synthetics";
 import { CHECK_TYPE_CARDS } from "@/constants/synthetics";
 import { useI18n } from "vue-i18n";
+import OStatStrip from "@/lib/data/StatStrip/OStatStrip.vue";
+import type { StatItem } from "@/lib/data/StatStrip/OStatStrip.types";
 import syntheticsService from "@/services/synthetics";
 import { locationDisplayLabel } from "@/utils/synthetics/format";
 import { getFoldersListByType } from "@/utils/commons";
@@ -440,6 +460,8 @@ function mapMonitor(m: ApiMonitor) {
     responseTime: m.last_response_ms !== null ? `${m.last_response_ms}ms` : null,
     locations: m.locations,
     lastCheck: m.last_check_at !== null ? formatTimeAgo(m.last_check_at) : "—",
+    // Raw microsecond epoch — drives the relative Last Check cell.
+    lastCheckAt: m.last_check_at,
     enabled: m.enabled,
     uptime: null as number | null,
     history: [] as unknown[],
@@ -813,39 +835,73 @@ const filteredStatusMonitors = computed(() =>
   ),
 );
 
-const statusTabs = computed(() => {
+// Attention-first order, matching every other strip in the app. Counts come from
+// `filteredStatusMonitors` (everything except the status facet itself), so the
+// tiles keep their totals while a status is selected.
+const summaryStats = computed<StatItem[]>(() => {
   const ms = filteredStatusMonitors.value;
-  const tabs = [
-    { filter: "all", label: t("synthetics.filters.allStatuses"), count: ms.length },
+  const total = ms.length;
+  const count = (status: string) => ms.filter((m) => m.status === status).length;
+  const v = (n: number): string | number => (total > 0 ? n : "—");
+  const share = total > 0 ? total : undefined;
+  const tiles: StatItem[] = [
     {
-      filter: "passed",
-      label: t("synthetics.filters.passed"),
-      count: ms.filter((m) => m.status === "passed").length,
-    },
-    {
-      filter: "warning",
-      label: t("synthetics.filters.warning"),
-      count: ms.filter((m) => m.status === "warning").length,
-    },
-    {
-      filter: "failed",
+      key: "failed",
       label: t("synthetics.filters.failed"),
-      count: ms.filter((m) => m.status === "failed").length,
+      value: v(count("failed")),
+      icon: "error-outline",
+      tone: "error",
+      max: share,
+      dataTest: "synthetics-summary-failed",
+    },
+    {
+      key: "warning",
+      label: t("synthetics.filters.warning"),
+      value: v(count("warning")),
+      icon: "warning-amber",
+      tone: "warning",
+      max: share,
+      dataTest: "synthetics-summary-warning",
+    },
+    {
+      key: "passed",
+      label: t("synthetics.filters.passed"),
+      value: v(count("passed")),
+      icon: "check-circle",
+      tone: "success",
+      max: share,
+      dataTest: "synthetics-summary-passed",
     },
   ];
-  const unknownCount = ms.filter((m) => m.status === "unknown").length;
-  if (unknownCount > 0) {
-    tabs.push({ filter: "unknown", label: t("synthetics.labels.unknown"), count: unknownCount });
+  // "Unknown" only means something once a monitor has never reported.
+  if (count("unknown") > 0) {
+    tiles.push({
+      key: "unknown",
+      label: t("synthetics.labels.unknown"),
+      value: v(count("unknown")),
+      icon: "help-outline",
+      tone: "neutral",
+      max: share,
+      dataTest: "synthetics-summary-unknown",
+    });
   }
-  return tabs;
+  tiles.push({
+    key: "all",
+    // "All", not "All Statuses": the tile sits in a row of statuses, so the noun is
+    // already established, and its job is simply "clear the facet".
+    label: t("synthetics.filters.all"),
+    value: v(total),
+    icon: "monitor-heart",
+    tone: "primary",
+    dataTest: "synthetics-summary-total",
+  });
+  return tiles;
 });
 
-const statusOpts = computed(() =>
-  statusTabs.value.map((s) => ({
-    label: `${s.label} (${s.count})`,
-    value: s.filter,
-  })),
-);
+// Selecting the active tile clears back to "all", like every other strip.
+const onStatSelect = (key: string) => {
+  statusFilter.value = key === "all" || statusFilter.value === key ? "all" : key;
+};
 
 const filteredMonitors = computed(() =>
   enrichedMonitors.value.filter(