1 line
3.4 MiB
1 line
3.4 MiB
[{"_timestamp":1771331712217000,"log":"[2026-02-17T13:34:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7639494272991316000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e0tdzzhmsh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712218000,"log":"[2026-02-17T14:22:42Z DEBUG sqlx::query] SELECT FROM streams returned 127 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ff3bzjmmeg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712219000,"log":"[2026-02-17T14:02:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 98ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ujxfn92mss","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712220000,"log":"[2026-02-17T14:16:23Z DEBUG sqlx::query] INSERT FROM users returned 322 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ay2iqhsnh8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712221000,"log":"[2026-02-17T14:39:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=h093maid","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iz95ghqya8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712222000,"log":"[2026-02-17T13:58:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 386ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g35i58g0an","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712223000,"log":"[2026-02-17T12:45:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oxvmjamm1p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712224000,"log":"[2026-02-17T13:52:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 362ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jlhd8dvd5z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712225000,"log":"[2026-02-17T13:43:25Z INFO actix_web::middleware::logger] GET /api/_search 400 216ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vyh4vmtzz7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712226000,"log":"[2026-02-17T13:57:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 50ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gmrlj4kiuf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712227000,"log":"[2026-02-17T15:20:44Z TRACE cache::redis] cache get key=stream:schema:9151","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-guk0vmyoms","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712228000,"log":"[2026-02-17T13:50:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 343ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6yrjtl2zeu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712229000,"log":"[2026-02-17T13:43:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 400ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x164a4wcvd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712230000,"log":"[2026-02-17T13:25:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2776482209162039000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wlrw44dfe5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712231000,"log":"[2026-02-17T14:02:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 64ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t2gtih9p9v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712232000,"log":"[2026-02-17T14:30:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 554 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8lci3vhlh9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712233000,"log":"[2026-02-17T14:42:14Z INFO tonic::transport::server] grpc router request_received client_id=e953g6xz","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aqi6xm0op4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712234000,"log":"[2026-02-17T13:38:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5236413592580624000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5d6kd4jqhq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712235000,"log":"[2026-02-17T14:16:55Z DEBUG sqlx::query] DELETE FROM alerts returned 480 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tgkmjyc6h1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712236000,"log":"[2026-02-17T13:00:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gte7gb61le","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712237000,"log":"[2026-02-17T15:21:42Z TRACE cache::redis] cache hit key=user:session:3477","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4yz9ysne8p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712238000,"log":"[2026-02-17T12:44:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0iza3dqwiz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712239000,"log":"[2026-02-17T13:21:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9981674074521143000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3ow0sg13wn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712240000,"log":"[2026-02-17T14:42:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=rl20sdyu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zdtfte76vs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712241000,"log":"[2026-02-17T14:40:02Z INFO tonic::transport::server] grpc ingester connected client_id=f8sudhvn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7a6645dn22","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712242000,"log":"[2026-02-17T14:39:12Z INFO tonic::transport::server] grpc ingester connected client_id=t0v7ox5k","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q01im4qqk0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712243000,"log":"[2026-02-17T15:14:09Z TRACE cache::redis] cache set key=stream:schema:2085","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l5vjllt2kf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712244000,"log":"[2026-02-17T14:56:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l5sm7azbdu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712245000,"log":"[2026-02-17T13:34:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3303883226546849000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9p1o4j4jal","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712246000,"log":"[2026-02-17T13:09:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1317860390088425500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fonipqxz5u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712247000,"log":"[2026-02-17T15:10:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1518","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-unrq9fj0n6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712248000,"log":"[2026-02-17T13:00:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3g7bxspvaa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712249000,"log":"[2026-02-17T15:11:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=2469","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fv0rzyuj60","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712250000,"log":"[2026-02-17T15:01:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-72t7va5czy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712251000,"log":"[2026-02-17T13:59:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 202ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mxkcm94ci8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712252000,"log":"[2026-02-17T13:09:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/917578382055728100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5ov3k6cktb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712253000,"log":"[2026-02-17T14:46:42Z INFO tonic::transport::server] grpc ingester connected client_id=6e8zne3f","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e5n1ojjhor","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712254000,"log":"[2026-02-17T13:39:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8193517759939175000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ogskpetg7r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712255000,"log":"[2026-02-17T13:19:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5984951578032402000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lwr01nd5bc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712256000,"log":"[2026-02-17T14:12:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 939 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s0yc6brdjt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712257000,"log":"[2026-02-17T12:39:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ldyuw179we","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712258000,"log":"[2026-02-17T13:33:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2143294690844237300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e2riyeidhy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712259000,"log":"[2026-02-17T13:15:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3836561708776782300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vit6dvtl5l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712260000,"log":"[2026-02-17T14:02:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 115ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-554zid9jcm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712261000,"log":"[2026-02-17T14:16:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 195 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-igoapwbdv5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712262000,"log":"[2026-02-17T13:38:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9517092621777630000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-esl83x5vdl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712263000,"log":"[2026-02-17T13:54:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 150ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-23ihdbr1ri","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712264000,"log":"[2026-02-17T14:48:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qkriy49ji8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712265000,"log":"[2026-02-17T14:51:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s127wab811","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712266000,"log":"[2026-02-17T13:38:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6923219720158804000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ssilc2j5zd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712267000,"log":"[2026-02-17T12:55:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-omnng2y8ta","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712268000,"log":"[2026-02-17T15:01:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rc741xfmtf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712269000,"log":"[2026-02-17T13:51:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 406ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tydp3qpm6m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712270000,"log":"[2026-02-17T13:50:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 191ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-khny8and14","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712271000,"log":"[2026-02-17T14:22:53Z DEBUG sqlx::query] INSERT FROM users returned 94 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rdbsmxn87x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712272000,"log":"[2026-02-17T14:20:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 204 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sn3vj8fvsu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712273000,"log":"[2026-02-17T14:47:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=rp6mdjxj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kx08onckvc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712274000,"log":"[2026-02-17T12:47:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e1p3j03n2k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712275000,"log":"[2026-02-17T14:58:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-908wch27nq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712276000,"log":"[2026-02-17T13:29:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5000275616136905000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vo7o0k61j9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712277000,"log":"[2026-02-17T14:52:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2jyf65sc51","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712278000,"log":"[2026-02-17T14:20:55Z DEBUG sqlx::query] DELETE FROM alerts returned 810 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ncnxf16tan","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712279000,"log":"[2026-02-17T12:48:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-98yt24ef5t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712280000,"log":"[2026-02-17T14:34:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=lrrfqx8p","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7pw6wtce9k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712281000,"log":"[2026-02-17T14:42:53Z INFO tonic::transport::server] grpc querier disconnected client_id=b9z18pda","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y2egf13lmf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712282000,"log":"[2026-02-17T12:58:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-48bcz05zfg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712283000,"log":"[2026-02-17T13:42:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 410ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a4japt6vpb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712284000,"log":"[2026-02-17T13:59:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 122ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kv7mmiyu0l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712285000,"log":"[2026-02-17T13:50:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 210ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6j7wz65kan","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712286000,"log":"[2026-02-17T13:20:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3100800192742319000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w6kiyx19oh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712287000,"log":"[2026-02-17T12:58:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n38vdalh8r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712288000,"log":"[2026-02-17T13:34:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9721982529276211000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r7av3imbi9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712289000,"log":"[2026-02-17T14:09:02Z DEBUG sqlx::query] SELECT FROM streams returned 296 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zcbgudypyl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712290000,"log":"[2026-02-17T13:35:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8863335268107017000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a90fey12xn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712291000,"log":"[2026-02-17T14:57:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kee75wbs6f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712292000,"log":"[2026-02-17T13:34:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1993750701215000300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ei7ekvqdyr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712293000,"log":"[2026-02-17T15:17:20Z TRACE cache::redis] cache get key=query:result:8730","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wp0awwsmzz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712294000,"log":"[2026-02-17T13:49:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 134ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9re772hc6u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712295000,"log":"[2026-02-17T12:55:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9m0jwjk8d4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712296000,"log":"[2026-02-17T13:03:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tprinjv9ar","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712297000,"log":"[2026-02-17T14:10:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 349 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6z3rk6bskz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712298000,"log":"[2026-02-17T12:50:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-64vestnlrw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712299000,"log":"[2026-02-17T12:43:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fvciqf9mwc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712300000,"log":"[2026-02-17T13:52:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 181ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vex1tgzyv8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712301000,"log":"[2026-02-17T14:40:12Z INFO tonic::transport::server] grpc ingester connected client_id=mx4ueqw6","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k4ntpbq72x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712302000,"log":"[2026-02-17T13:51:40Z INFO actix_web::middleware::logger] POST /api/_search 400 195ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-inpqu1sg09","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712303000,"log":"[2026-02-17T12:54:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zg2lwo9h1b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712304000,"log":"[2026-02-17T13:22:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3029119193560478700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tsb4gvii68","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712305000,"log":"[2026-02-17T13:33:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8061602335315201000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qob8togn3o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712306000,"log":"[2026-02-17T12:44:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-swhri7b43z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712307000,"log":"[2026-02-17T13:45:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 257ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e3f9d55gej","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712308000,"log":"[2026-02-17T14:12:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 948 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rdxmw979pn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712309000,"log":"[2026-02-17T12:51:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0wb72n7rf4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712310000,"log":"[2026-02-17T15:01:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hkmq1xs0i2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712311000,"log":"[2026-02-17T13:42:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 257ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6jb5ivdze5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712312000,"log":"[2026-02-17T13:20:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2311792969704151600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bbqdu8n9e3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712313000,"log":"[2026-02-17T14:47:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=c814et1g","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6bfjtw145s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712314000,"log":"[2026-02-17T14:29:35Z DEBUG sqlx::query] DELETE FROM alerts returned 687 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bgv29lu744","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712315000,"log":"[2026-02-17T15:16:04Z TRACE cache::redis] cache delete key=stream:schema:1764","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mamw98sj9j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712316000,"log":"[2026-02-17T14:29:53Z DEBUG sqlx::query] INSERT FROM users returned 692 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x27v7ajok9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712317000,"log":"[2026-02-17T14:23:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 843 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ha8glzb3t6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712318000,"log":"[2026-02-17T13:26:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5022294304855111000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4trseqowkl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712319000,"log":"[2026-02-17T15:16:25Z TRACE cache::redis] cache miss key=query:result:9019","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hzcaqcn29w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712320000,"log":"[2026-02-17T14:54:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1qdcx1zu58","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712321000,"log":"[2026-02-17T13:09:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/100517087932285870.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iaonddq42u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712322000,"log":"[2026-02-17T14:45:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=vrljnihw","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yrkinrxmna","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712323000,"log":"[2026-02-17T14:31:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=1gq314rj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uujkz9kx8s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712324000,"log":"[2026-02-17T14:06:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 457ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gsc4ytswm9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712325000,"log":"[2026-02-17T13:30:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9336737989918413000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3683uue7g7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712326000,"log":"[2026-02-17T13:32:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5723355005921400000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9l5l6hp0u3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712327000,"log":"[2026-02-17T12:53:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4a9qycjp3i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712328000,"log":"[2026-02-17T14:31:40Z DEBUG sqlx::query] DELETE FROM alerts returned 26 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z4p4urs94r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712329000,"log":"[2026-02-17T12:50:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s8mmad66is","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712330000,"log":"[2026-02-17T12:52:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b58dbja08v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712331000,"log":"[2026-02-17T13:46:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 4ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9fgxj90bif","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712332000,"log":"[2026-02-17T15:15:31Z TRACE cache::redis] cache miss key=auth:token:5575","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jcwr0zkauc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712333000,"log":"[2026-02-17T13:46:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 492ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ff21gvjyqm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712334000,"log":"[2026-02-17T15:19:37Z TRACE cache::redis] cache miss key=user:session:3905","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mdyymsd150","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712335000,"log":"[2026-02-17T12:43:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v296b7m3p2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712336000,"log":"[2026-02-17T15:09:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6783","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2vxul5nd0m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712337000,"log":"[2026-02-17T15:05:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=7071","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q5ddqfwcp1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712338000,"log":"[2026-02-17T12:56:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2awsr2ktps","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712339000,"log":"[2026-02-17T13:10:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5699688935215581000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1jkqoo25ho","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712340000,"log":"[2026-02-17T13:54:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 160ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8rv0idrl65","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712341000,"log":"[2026-02-17T13:12:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/495291545560201400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gguhaxx2y2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712342000,"log":"[2026-02-17T15:09:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=6837","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h66byaza11","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712343000,"log":"[2026-02-17T14:15:40Z DEBUG sqlx::query] DELETE FROM alerts returned 273 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lyffkz7cre","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712344000,"log":"[2026-02-17T14:58:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4v1b96myvf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712345000,"log":"[2026-02-17T15:13:38Z TRACE cache::redis] cache get key=org:config:4024","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1ccc7gr9s3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712346000,"log":"[2026-02-17T14:03:45Z INFO actix_web::middleware::logger] GET /api/_search 503 243ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uk47lh3umd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712347000,"log":"[2026-02-17T13:10:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8535970757651980000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ghd9e0ade6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712348000,"log":"[2026-02-17T13:44:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o3x8pfxwyi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712349000,"log":"[2026-02-17T14:48:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rqjqeq1ylc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712350000,"log":"[2026-02-17T13:51:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 212ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vss7sntysm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712351000,"log":"[2026-02-17T14:01:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 97ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7j8z6wxvdl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712352000,"log":"[2026-02-17T13:41:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1100947046058367000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nh4xwp6irc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712353000,"log":"[2026-02-17T15:08:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=576","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e9qawgfmf9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712354000,"log":"[2026-02-17T13:23:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8055542916536565000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sig4hj5fzi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712355000,"log":"[2026-02-17T14:03:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 476ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cevyigo5ug","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712356000,"log":"[2026-02-17T14:13:00Z DEBUG sqlx::query] DELETE FROM alerts returned 645 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1wzljak1eb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712357000,"log":"[2026-02-17T13:44:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 60ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-unkoe8hkbp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712358000,"log":"[2026-02-17T14:32:47Z INFO tonic::transport::server] grpc ingester connected client_id=sw4m73ft","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uzgoqs41ny","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712359000,"log":"[2026-02-17T15:00:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3h3rizm5p0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712360000,"log":"[2026-02-17T14:56:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-95o8fo1i99","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712361000,"log":"[2026-02-17T15:00:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u0e2po5yy8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712362000,"log":"[2026-02-17T14:58:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b6zev5c2kn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712363000,"log":"[2026-02-17T14:12:40Z DEBUG sqlx::query] DELETE FROM alerts returned 223 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ynfelopl5c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712364000,"log":"[2026-02-17T15:01:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vae21l7moh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712365000,"log":"[2026-02-17T13:37:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9709513674054304000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ium601sviz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712366000,"log":"[2026-02-17T13:26:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7510183194794162000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9h7cy2a9y1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712367000,"log":"[2026-02-17T13:34:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/998686022893913200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t1q2yg6rga","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712368000,"log":"[2026-02-17T15:17:42Z TRACE cache::redis] cache hit key=user:session:7402","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6atxg9e5oj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712369000,"log":"[2026-02-17T13:37:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1531036441615891700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1gunzxxdof","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712370000,"log":"[2026-02-17T14:38:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=56dhxwpn","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vur36vn58d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712371000,"log":"[2026-02-17T14:32:43Z INFO tonic::transport::server] grpc querier disconnected client_id=ruwdo1vn","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bqbhia1v2y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712372000,"log":"[2026-02-17T12:43:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oltd1oyaz8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712373000,"log":"[2026-02-17T13:34:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8968924823463815000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-branc0lhy1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712374000,"log":"[2026-02-17T14:45:08Z INFO tonic::transport::server] grpc querier disconnected client_id=4draa3ae","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7x50z7vllk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712375000,"log":"[2026-02-17T14:30:48Z DEBUG sqlx::query] INSERT FROM users returned 719 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nfw56ubp17","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712376000,"log":"[2026-02-17T15:20:58Z TRACE cache::redis] cache delete key=org:config:4879","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t5cxymtcdy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712377000,"log":"[2026-02-17T14:37:57Z INFO tonic::transport::server] grpc ingester connected client_id=3z24u0ed","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2hofz4x6q1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712378000,"log":"[2026-02-17T14:36:34Z INFO tonic::transport::server] grpc router request_received client_id=m1l1eb85","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5wpch3u9bi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712379000,"log":"[2026-02-17T14:35:54Z INFO tonic::transport::server] grpc router request_received client_id=wsklt7ow","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fyv5o813a2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712380000,"log":"[2026-02-17T14:11:12Z DEBUG sqlx::query] SELECT FROM streams returned 838 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4den5qy5ab","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712381000,"log":"[2026-02-17T14:55:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j6cib13q88","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712382000,"log":"[2026-02-17T13:37:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9615729202422491000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b9hl9vqv51","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712383000,"log":"[2026-02-17T15:05:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l5xdmezkj3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712384000,"log":"[2026-02-17T13:22:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8617197015344791000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ky822th6dc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712385000,"log":"[2026-02-17T13:10:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/847214836138507900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zpft43ayup","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712386000,"log":"[2026-02-17T14:41:54Z INFO tonic::transport::server] grpc router request_received client_id=xbbuo46f","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ghqa5pdbfo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712387000,"log":"[2026-02-17T14:42:38Z INFO tonic::transport::server] grpc querier disconnected client_id=51c6jwgw","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8ss41cq8h9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712388000,"log":"[2026-02-17T14:06:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 299ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jwjb8i6ut5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712389000,"log":"[2026-02-17T15:05:20Z DEBUG metrics::reporter] recorded metric active_connections value=9707","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j1ydxpr0j0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712390000,"log":"[2026-02-17T12:37:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1cz3f9u3t0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712391000,"log":"[2026-02-17T15:08:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8710","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5p65r6mvr7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712392000,"log":"[2026-02-17T14:48:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yl7yr0p31r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712393000,"log":"[2026-02-17T13:40:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6870724897670111000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d6k0gbrhir","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712394000,"log":"[2026-02-17T13:44:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 424ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o76xsrgrla","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712395000,"log":"[2026-02-17T14:50:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tdy0rx01v1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712396000,"log":"[2026-02-17T14:18:52Z DEBUG sqlx::query] SELECT FROM streams returned 490 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n474e1gxgg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712397000,"log":"[2026-02-17T14:08:58Z DEBUG sqlx::query] INSERT FROM users returned 439 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-62dxhsxsxf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712398000,"log":"[2026-02-17T14:28:20Z DEBUG sqlx::query] DELETE FROM alerts returned 710 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-smllbmr9fz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712399000,"log":"[2026-02-17T13:36:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5220685762921009000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ku7g0p5h5d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712400000,"log":"[2026-02-17T15:02:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7wu6snm77y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712401000,"log":"[2026-02-17T13:11:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5368768812103186000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8sulfvma63","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712402000,"log":"[2026-02-17T15:15:42Z TRACE cache::redis] cache hit key=user:session:2406","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9r6n4po2q0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712403000,"log":"[2026-02-17T13:06:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5y45h2og1q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712404000,"log":"[2026-02-17T14:50:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3k869ihv08","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712405000,"log":"[2026-02-17T13:01:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ild6d71pq8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712406000,"log":"[2026-02-17T14:53:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k5v96dzfiy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712407000,"log":"[2026-02-17T14:07:20Z DEBUG sqlx::query] DELETE FROM alerts returned 759 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8nbr02jj6c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712408000,"log":"[2026-02-17T13:41:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7601093718110170000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-90yc3dnb0f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712409000,"log":"[2026-02-17T12:58:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gmso3mwaao","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712410000,"log":"[2026-02-17T12:46:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-id58m8rqd1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712411000,"log":"[2026-02-17T13:12:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5275473767118529000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p750n63f41","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712412000,"log":"[2026-02-17T13:35:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/309172719035342140.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hae7knh35c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712413000,"log":"[2026-02-17T13:23:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6758724585590943000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q4pn3kor8d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712414000,"log":"[2026-02-17T14:38:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=j7bzbwwj","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-179l744tvm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712415000,"log":"[2026-02-17T13:01:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-blg8jwtzop","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712416000,"log":"[2026-02-17T13:44:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 367ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2c3lho5vm5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712417000,"log":"[2026-02-17T14:43:58Z INFO tonic::transport::server] grpc querier disconnected client_id=g5bfz9vt","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jylbthtf02","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712418000,"log":"[2026-02-17T12:49:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l1bnsylw8v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712419000,"log":"[2026-02-17T13:24:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8793188622919630000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-06dmerp1oi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712420000,"log":"[2026-02-17T15:16:30Z TRACE cache::redis] cache hit key=query:result:7265","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-adk3zyl9qq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712421000,"log":"[2026-02-17T13:57:25Z INFO actix_web::middleware::logger] GET /api/_search 404 2ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h86kzn1mby","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712422000,"log":"[2026-02-17T14:26:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 772 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uiatbevlc4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712423000,"log":"[2026-02-17T14:59:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mxhdcf12i2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712424000,"log":"[2026-02-17T15:20:56Z TRACE cache::redis] cache get key=auth:token:5748","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yi0bgzq78u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712425000,"log":"[2026-02-17T13:16:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2484754260422999000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jt2qzgsrmh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712426000,"log":"[2026-02-17T14:59:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gt1rgra1kf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712427000,"log":"[2026-02-17T12:39:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vubxvajw4j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712428000,"log":"[2026-02-17T13:53:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 82ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2doivjmmo1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712429000,"log":"[2026-02-17T15:19:46Z TRACE cache::redis] cache delete key=auth:token:2490","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vake7mq78x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712430000,"log":"[2026-02-17T12:55:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iowim408vu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712431000,"log":"[2026-02-17T13:02:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xmyef96c2o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712432000,"log":"[2026-02-17T13:20:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4054579579912813600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lhvdcyypbc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712433000,"log":"[2026-02-17T12:45:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zfjnsi3by4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712434000,"log":"[2026-02-17T14:38:53Z INFO tonic::transport::server] grpc querier disconnected client_id=an33pqy7","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xa98b1i8vo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712435000,"log":"[2026-02-17T14:59:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5lcuj0mipi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712436000,"log":"[2026-02-17T13:23:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/179320715522166560.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gp88tay6cq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712437000,"log":"[2026-02-17T13:14:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5765517176094954000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vafcizeg49","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712438000,"log":"[2026-02-17T14:03:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 356ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6213r8n9e7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712439000,"log":"[2026-02-17T15:16:02Z TRACE cache::redis] cache get key=user:session:8087","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5rhafzbmui","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712440000,"log":"[2026-02-17T13:03:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mj9ber9sl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712441000,"log":"[2026-02-17T15:02:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pantr6wksn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712442000,"log":"[2026-02-17T12:58:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sb0h6t40nn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712443000,"log":"[2026-02-17T14:41:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=hfcvg4c7","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7zw77ozths","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712444000,"log":"[2026-02-17T12:40:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xyw6w9ps0v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712445000,"log":"[2026-02-17T13:45:40Z INFO actix_web::middleware::logger] POST /api/_search 400 243ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qoekc4ilyo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712446000,"log":"[2026-02-17T13:52:40Z INFO actix_web::middleware::logger] POST /api/_search 200 101ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z9q4tgil25","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712447000,"log":"[2026-02-17T14:23:47Z DEBUG sqlx::query] SELECT FROM streams returned 41 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-txxzh2eynr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712448000,"log":"[2026-02-17T14:48:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q9pj4ulfwr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712449000,"log":"[2026-02-17T14:09:30Z DEBUG sqlx::query] DELETE FROM alerts returned 85 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lb9f5uipwl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712450000,"log":"[2026-02-17T15:19:43Z TRACE cache::redis] cache miss key=org:config:8967","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-grca87f0wq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712451000,"log":"[2026-02-17T13:24:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6001944806162298000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kb0w5zrqp5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712452000,"log":"[2026-02-17T12:50:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-79t6ps0u2q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712453000,"log":"[2026-02-17T13:56:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 382ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qh26htkwxe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712454000,"log":"[2026-02-17T15:21:07Z TRACE cache::redis] cache miss key=user:session:2708","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6e4jygh60v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712455000,"log":"[2026-02-17T14:27:28Z DEBUG sqlx::query] INSERT FROM users returned 426 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iha5tetvaf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712456000,"log":"[2026-02-17T14:37:42Z INFO tonic::transport::server] grpc ingester connected client_id=61rlj4z2","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wjybeqdte5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712457000,"log":"[2026-02-17T15:12:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=3446","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qisb37uocf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712458000,"log":"[2026-02-17T12:58:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zborrvf98f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712459000,"log":"[2026-02-17T14:21:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 937 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yq66x51d6w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712460000,"log":"[2026-02-17T14:48:19Z INFO tonic::transport::server] grpc router request_received client_id=di6d6t15","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9frqh60fiy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712461000,"log":"[2026-02-17T14:51:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ddcn4wcwd0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712462000,"log":"[2026-02-17T13:19:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6433510489449818000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qn0hrj8izv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712463000,"log":"[2026-02-17T14:26:12Z DEBUG sqlx::query] SELECT FROM streams returned 668 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zua5zuqvcf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712464000,"log":"[2026-02-17T13:14:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9014453460387533000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w0h086r7jh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712465000,"log":"[2026-02-17T14:09:48Z DEBUG sqlx::query] INSERT FROM users returned 90 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-timvvhu0jw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712466000,"log":"[2026-02-17T13:19:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1104662475590891800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wxph8ajikr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712467000,"log":"[2026-02-17T15:07:55Z DEBUG metrics::reporter] recorded metric active_connections value=2754","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p1311cjmuq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712468000,"log":"[2026-02-17T14:21:30Z DEBUG sqlx::query] DELETE FROM alerts returned 403 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1r6c25wtu6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712469000,"log":"[2026-02-17T12:37:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-haz2fi28pt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712470000,"log":"[2026-02-17T13:41:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7918614926202692000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fujx1dtn4b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712471000,"log":"[2026-02-17T15:01:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-46c50jfegv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712472000,"log":"[2026-02-17T13:42:05Z INFO actix_web::middleware::logger] GET /api/_search 401 248ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7ngq1ef8yj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712473000,"log":"[2026-02-17T14:44:44Z INFO tonic::transport::server] grpc router request_received client_id=awnx7vhx","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3o3oyhd6o5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712474000,"log":"[2026-02-17T13:44:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 9ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yzdogfagu6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712475000,"log":"[2026-02-17T13:38:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1820431692221331500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1yln2rru2f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712476000,"log":"[2026-02-17T14:56:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lzfpdxjgqz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712477000,"log":"[2026-02-17T14:07:53Z DEBUG sqlx::query] INSERT FROM users returned 517 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gtb47hzmlg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712478000,"log":"[2026-02-17T13:35:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/150262342991340770.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rp0lldayo1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712479000,"log":"[2026-02-17T15:06:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1454","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b4x20e2akc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712480000,"log":"[2026-02-17T13:33:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9320604473070397000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mz7e35asn6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712481000,"log":"[2026-02-17T14:18:02Z DEBUG sqlx::query] SELECT FROM streams returned 552 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3eu2v77hdx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712482000,"log":"[2026-02-17T13:44:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 321ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p0vwvyz97n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712483000,"log":"[2026-02-17T12:44:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ucwugy0py0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712484000,"log":"[2026-02-17T15:20:25Z TRACE cache::redis] cache miss key=query:result:5328","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vxwa76oczd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712485000,"log":"[2026-02-17T13:21:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6304746557511216000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-09x832p6nv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712486000,"log":"[2026-02-17T14:11:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 3 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-adnoc45vyd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712487000,"log":"[2026-02-17T13:18:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1698396892002782500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qcxby3nloe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712488000,"log":"[2026-02-17T14:46:57Z INFO tonic::transport::server] grpc ingester connected client_id=vd620nus","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-auxhzs9eo3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712489000,"log":"[2026-02-17T13:33:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/953158535776854700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z1ez4csksv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712490000,"log":"[2026-02-17T12:43:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j0hab8yzeg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712491000,"log":"[2026-02-17T14:05:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 310ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ly1b1uz6rx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712492000,"log":"[2026-02-17T12:45:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qze6i2uynb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712493000,"log":"[2026-02-17T13:12:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3479224962941123000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1g11krz7gt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712494000,"log":"[2026-02-17T13:33:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2685592639523051000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-be6kqiz8s0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712495000,"log":"[2026-02-17T15:11:05Z DEBUG metrics::reporter] recorded metric active_connections value=4496","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ocga6o5m7t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712496000,"log":"[2026-02-17T14:27:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 385 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aa1437zjv1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712497000,"log":"[2026-02-17T13:18:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6203952341121965000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mp0g54pxwo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712498000,"log":"[2026-02-17T14:21:12Z DEBUG sqlx::query] SELECT FROM streams returned 34 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x0vxm3g03w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712499000,"log":"[2026-02-17T13:08:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/204901267371144770.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u4r57frxx1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712500000,"log":"[2026-02-17T15:11:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6635","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-daaw4fdvb5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712501000,"log":"[2026-02-17T14:11:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 131 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z4g3lh366i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712502000,"log":"[2026-02-17T14:44:12Z INFO tonic::transport::server] grpc ingester connected client_id=bu0xvr9w","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3cphodaqz7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712503000,"log":"[2026-02-17T14:09:07Z DEBUG sqlx::query] SELECT FROM streams returned 920 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2i5ljibvk6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712504000,"log":"[2026-02-17T14:36:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=g2g7tryg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p0xfuegsk1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712505000,"log":"[2026-02-17T14:29:42Z DEBUG sqlx::query] SELECT FROM streams returned 415 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d1zwuml1tz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712506000,"log":"[2026-02-17T14:57:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wa8kffgso6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712507000,"log":"[2026-02-17T12:52:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-quj63r0pw8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712508000,"log":"[2026-02-17T14:51:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-opgyy25jgp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712509000,"log":"[2026-02-17T12:38:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4skkl1xykx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712510000,"log":"[2026-02-17T13:11:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7810045156007352000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9x0b9xsy83","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712511000,"log":"[2026-02-17T12:54:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-efm9mmcdg0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712512000,"log":"[2026-02-17T14:40:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=svushypj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-frdb17hf7a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712513000,"log":"[2026-02-17T13:56:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bq7vyew756","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712514000,"log":"[2026-02-17T14:09:00Z DEBUG sqlx::query] DELETE FROM alerts returned 894 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3bmu1cea6i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712515000,"log":"[2026-02-17T14:31:03Z DEBUG sqlx::query] INSERT FROM users returned 819 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lz6xdbrls9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712516000,"log":"[2026-02-17T15:07:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=3171","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4sjfigah7y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712517000,"log":"[2026-02-17T13:20:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2761622018193568300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-btrluwc8g9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712518000,"log":"[2026-02-17T13:13:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8531774757584229000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-47engrln1r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712519000,"log":"[2026-02-17T14:11:57Z DEBUG sqlx::query] SELECT FROM streams returned 181 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o46ll811p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712520000,"log":"[2026-02-17T13:45:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 318ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4qq8xoubl1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712521000,"log":"[2026-02-17T14:17:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 804 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-133z5xamq1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712522000,"log":"[2026-02-17T12:58:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n36zeu605z","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712523000,"log":"[2026-02-17T13:41:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8544741634823305000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4k2sn4nmhg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712524000,"log":"[2026-02-17T15:12:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1006","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xbb4vxqp9v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712525000,"log":"[2026-02-17T15:15:54Z TRACE cache::redis] cache hit key=stream:schema:4098","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1nvd2b1aby","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712526000,"log":"[2026-02-17T12:41:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1pf9heq2ip","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712527000,"log":"[2026-02-17T12:45:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9f5dxgugdx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712528000,"log":"[2026-02-17T14:08:05Z DEBUG sqlx::query] DELETE FROM alerts returned 402 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aobvhbczw9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712529000,"log":"[2026-02-17T14:26:10Z DEBUG sqlx::query] DELETE FROM alerts returned 927 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ovn9gu6yrs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712530000,"log":"[2026-02-17T13:37:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1181016667095045000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-almkm40j26","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712531000,"log":"[2026-02-17T15:14:39Z TRACE cache::redis] cache set key=stream:schema:7588","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8lapd3saen","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712532000,"log":"[2026-02-17T13:40:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3626510944036114400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hmoie55ixn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712533000,"log":"[2026-02-17T14:18:28Z DEBUG sqlx::query] INSERT FROM users returned 318 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q6h9rbgorx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712534000,"log":"[2026-02-17T15:13:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=2408","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ps34jsi9x5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712535000,"log":"[2026-02-17T12:48:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gk0bb07kvx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712536000,"log":"[2026-02-17T13:02:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-at2qt42gkc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712537000,"log":"[2026-02-17T14:33:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=cbytwpr4","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6dcpf519zy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712538000,"log":"[2026-02-17T15:16:59Z TRACE cache::redis] cache expire key=stream:schema:9744","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fcqk2kre88","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712539000,"log":"[2026-02-17T13:52:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 21ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-84agdq8esk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712540000,"log":"[2026-02-17T13:17:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4149404131883389400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ypni9rc5pb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712541000,"log":"[2026-02-17T14:26:40Z DEBUG sqlx::query] DELETE FROM alerts returned 211 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-toi7m2pkmo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712542000,"log":"[2026-02-17T12:36:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cwzcz65k0k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712543000,"log":"[2026-02-17T12:48:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pkq5kzvo7o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712544000,"log":"[2026-02-17T14:03:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 294ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z7s49n98oc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712545000,"log":"[2026-02-17T13:45:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 345ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-davhucoozy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712546000,"log":"[2026-02-17T13:14:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3998263658624184000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ik85qekmg5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712547000,"log":"[2026-02-17T12:45:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cztph9smdq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712548000,"log":"[2026-02-17T13:01:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x8o6q0591m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712549000,"log":"[2026-02-17T14:45:07Z INFO tonic::transport::server] grpc ingester connected client_id=pvhd67w0","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t4d4ny7ya5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712550000,"log":"[2026-02-17T14:23:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 295 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3i02u7avpb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712551000,"log":"[2026-02-17T14:18:38Z DEBUG sqlx::query] INSERT FROM users returned 5 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9t3s5p20xx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712552000,"log":"[2026-02-17T13:03:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1g7hkxi3vl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712553000,"log":"[2026-02-17T13:05:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tas46do34","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712554000,"log":"[2026-02-17T15:06:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=510","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oxiidqtwd7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712555000,"log":"[2026-02-17T13:21:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4574633255755660300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wdors3k23m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712556000,"log":"[2026-02-17T13:19:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/257172514181469570.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-42qzan5a0c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712557000,"log":"[2026-02-17T12:41:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2iufldy9yf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712558000,"log":"[2026-02-17T12:39:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jl90d40m61","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712559000,"log":"[2026-02-17T15:14:12Z TRACE cache::redis] cache hit key=user:session:8546","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f4r7t2sf69","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712560000,"log":"[2026-02-17T15:02:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tjxo4rkdps","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712561000,"log":"[2026-02-17T14:23:50Z DEBUG sqlx::query] DELETE FROM alerts returned 947 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wa6mwe1k11","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712562000,"log":"[2026-02-17T13:13:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2005567618153771800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-70d7u6shry","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712563000,"log":"[2026-02-17T14:02:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 36ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9cq145j0x4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712564000,"log":"[2026-02-17T12:58:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9llabksqfd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712565000,"log":"[2026-02-17T13:58:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 53ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8sg2krsjqb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712566000,"log":"[2026-02-17T12:49:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vczg2r1hqy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712567000,"log":"[2026-02-17T13:25:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/162582662761703300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mvracnj2tx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712568000,"log":"[2026-02-17T15:20:37Z TRACE cache::redis] cache miss key=user:session:6674","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b7hx25fpmn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712569000,"log":"[2026-02-17T14:37:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=hzxgwvy2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sw1x80crb6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712570000,"log":"[2026-02-17T14:45:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pbhlz1ac","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j9ch62bikk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712571000,"log":"[2026-02-17T13:28:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7522669449453164000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3kd73z0qw4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712572000,"log":"[2026-02-17T13:29:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1863656352881682000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zv30jpjipd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712573000,"log":"[2026-02-17T13:36:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9887680923222096000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-psbmtwbes6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712574000,"log":"[2026-02-17T14:56:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-96dudm1fat","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712575000,"log":"[2026-02-17T12:55:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f8n0kvi3um","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712576000,"log":"[2026-02-17T13:22:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5915854221802068000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2w0vrl2xc5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712577000,"log":"[2026-02-17T13:04:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zwds5fmm4y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712578000,"log":"[2026-02-17T13:53:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 45ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t424hku15i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712579000,"log":"[2026-02-17T12:38:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p4ffp0oyjh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712580000,"log":"[2026-02-17T13:16:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4212856326212097500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2o74qn90du","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712581000,"log":"[2026-02-17T13:06:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uds5ent2l1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712582000,"log":"[2026-02-17T14:05:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 257ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gu038ykws1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712583000,"log":"[2026-02-17T13:49:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 416ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4xv1e7luuy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712584000,"log":"[2026-02-17T13:04:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8qdkve9l61","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712585000,"log":"[2026-02-17T15:01:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kd4zao14qv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712586000,"log":"[2026-02-17T15:03:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lotcez9lwp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712587000,"log":"[2026-02-17T12:40:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gx4slpjckw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712588000,"log":"[2026-02-17T14:51:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2zllia0kfp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712589000,"log":"[2026-02-17T13:32:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6824465207917438000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2nay80lcq5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712590000,"log":"[2026-02-17T13:33:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8446393710375648000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c8e2gleqxw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712591000,"log":"[2026-02-17T14:15:23Z DEBUG sqlx::query] INSERT FROM users returned 587 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lm1qg7o3sa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712592000,"log":"[2026-02-17T13:41:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6178621100766097000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7vy3tk96mm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712593000,"log":"[2026-02-17T12:44:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yqop8763ij","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712594000,"log":"[2026-02-17T15:18:39Z TRACE cache::redis] cache set key=stream:schema:1106","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6bzpl7iyju","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712595000,"log":"[2026-02-17T13:48:45Z INFO actix_web::middleware::logger] GET /api/_search 503 126ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7dh4t4c56n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712596000,"log":"[2026-02-17T14:33:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=6eu802a7","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jwefbgf8p1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712597000,"log":"[2026-02-17T13:39:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2565905392025187300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wz0vp0fv4w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712598000,"log":"[2026-02-17T13:54:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 417ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nc0qx4t4ov","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712599000,"log":"[2026-02-17T13:45:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 412ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bjxjeeo4bf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712600000,"log":"[2026-02-17T13:03:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q6ttdchzgf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712601000,"log":"[2026-02-17T12:55:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fw051jfzyi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712602000,"log":"[2026-02-17T13:04:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ue8wz8t1tt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712603000,"log":"[2026-02-17T15:14:19Z TRACE cache::redis] cache miss key=stream:schema:4693","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a019frsnf4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712604000,"log":"[2026-02-17T14:42:12Z INFO tonic::transport::server] grpc ingester connected client_id=4hl5fj6b","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xslyrhqhee","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712605000,"log":"[2026-02-17T13:05:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mfqk22170d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712606000,"log":"[2026-02-17T12:47:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ub9x3evjkp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712607000,"log":"[2026-02-17T13:57:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 75ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9gl044csv6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712608000,"log":"[2026-02-17T12:46:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-no43dxjoks","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712609000,"log":"[2026-02-17T13:57:20Z INFO actix_web::middleware::logger] POST /api/_search 201 337ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-882r7hvja9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712610000,"log":"[2026-02-17T12:39:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4t4ok0frjb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712611000,"log":"[2026-02-17T14:46:53Z INFO tonic::transport::server] grpc querier disconnected client_id=nqhmqfka","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lap9sp50bn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712612000,"log":"[2026-02-17T13:30:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2539489327861512700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tnu9k8zibv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712613000,"log":"[2026-02-17T13:53:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 228ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xzibaja5o7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712614000,"log":"[2026-02-17T13:43:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 416ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0auyzocv8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712615000,"log":"[2026-02-17T14:35:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=tlrkc2no","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ev8krxxldl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712616000,"log":"[2026-02-17T15:12:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=7062","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bf0t4s7qoo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712617000,"log":"[2026-02-17T13:27:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6952693044166738000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a76w7ygqzy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712618000,"log":"[2026-02-17T14:12:50Z DEBUG sqlx::query] DELETE FROM alerts returned 566 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fe59y07yxb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712619000,"log":"[2026-02-17T13:35:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2851044755300507000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mnr9jmkc8w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712620000,"log":"[2026-02-17T15:14:36Z TRACE cache::redis] cache hit key=auth:token:2479","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i74adhpusq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712621000,"log":"[2026-02-17T13:56:45Z INFO actix_web::middleware::logger] GET /api/_search 204 144ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ylbsqc4ebq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712622000,"log":"[2026-02-17T13:37:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6449447366022014000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q2o11lq5ai","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712623000,"log":"[2026-02-17T13:14:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6968376880856904000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tdnh4s9b3s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712624000,"log":"[2026-02-17T12:35:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xnedyy9t5f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712625000,"log":"[2026-02-17T13:54:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 401ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ullogpmk25","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712626000,"log":"[2026-02-17T12:44:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dvm0edc06y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712627000,"log":"[2026-02-17T14:04:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 403ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8ythfcu1gj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712628000,"log":"[2026-02-17T15:06:20Z DEBUG metrics::reporter] recorded metric active_connections value=1530","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dgaozta6fj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712629000,"log":"[2026-02-17T13:01:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-67t0q5qsgy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712630000,"log":"[2026-02-17T13:55:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 341ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vu6occ1ijn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712631000,"log":"[2026-02-17T13:29:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7402439337495075000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5u1iojjd1l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712632000,"log":"[2026-02-17T14:09:37Z DEBUG sqlx::query] SELECT FROM streams returned 349 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x329ayiwrj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712633000,"log":"[2026-02-17T13:10:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9250115626337411000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yj6vjc71w6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712634000,"log":"[2026-02-17T13:54:20Z INFO actix_web::middleware::logger] POST /api/_search 201 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fcfe9hz8n4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712635000,"log":"[2026-02-17T14:48:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0fj2ot8i3p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712636000,"log":"[2026-02-17T14:46:19Z INFO tonic::transport::server] grpc router request_received client_id=4r5dtxdx","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8yrc9y2vbe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712637000,"log":"[2026-02-17T13:34:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1233003767461083000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0wfh1o5fnm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712638000,"log":"[2026-02-17T13:21:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2836847239086535000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-upwkqagfcw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712639000,"log":"[2026-02-17T12:59:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dgwm2y0s8z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712640000,"log":"[2026-02-17T14:00:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 190ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0pqdmexa4d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712641000,"log":"[2026-02-17T14:23:53Z DEBUG sqlx::query] INSERT FROM users returned 835 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fzmfuzn73k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712642000,"log":"[2026-02-17T13:57:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 24ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-57swkuzr3c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712643000,"log":"[2026-02-17T14:35:17Z INFO tonic::transport::server] grpc ingester connected client_id=um8xfi1h","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kvk0fju1fq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712644000,"log":"[2026-02-17T14:53:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oab5s8sljo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712645000,"log":"[2026-02-17T13:07:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z7xspgpzll","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712646000,"log":"[2026-02-17T13:46:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 130ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0cilch27gu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712647000,"log":"[2026-02-17T13:17:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8123380667106206000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7fnrzuchrx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712648000,"log":"[2026-02-17T14:04:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 217ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ugn08ff4e8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712649000,"log":"[2026-02-17T14:37:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=lghquyhe","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pszr6m06pj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712650000,"log":"[2026-02-17T14:48:23Z INFO tonic::transport::server] grpc querier disconnected client_id=sj9cs3u0","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0dzmvzrqid","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712651000,"log":"[2026-02-17T12:55:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0hf8k5u0tb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712652000,"log":"[2026-02-17T12:35:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-er55kqunc2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712653000,"log":"[2026-02-17T14:25:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 997 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5s3eytf2lo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712654000,"log":"[2026-02-17T14:52:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f0slzobmi8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712655000,"log":"[2026-02-17T12:45:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-49d22gr6xt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712656000,"log":"[2026-02-17T14:29:43Z DEBUG sqlx::query] INSERT FROM users returned 983 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9f2ymxq9gt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712657000,"log":"[2026-02-17T13:40:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8429660599988898000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n0um7gfrgz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712658000,"log":"[2026-02-17T14:44:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=zyc2eyis","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dxqst47gpa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712659000,"log":"[2026-02-17T15:04:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yopb4xsngv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712660000,"log":"[2026-02-17T15:12:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8558","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-90ajzul6em","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712661000,"log":"[2026-02-17T12:55:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-91zacdopqu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712662000,"log":"[2026-02-17T14:41:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=m7a2m9cn","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h7pj9fpf48","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712663000,"log":"[2026-02-17T12:53:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-egs98i2m7p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712664000,"log":"[2026-02-17T13:25:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1298661727998715600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-56uvgilf6u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712665000,"log":"[2026-02-17T14:37:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=yrx1svmb","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dgekch57y9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712666000,"log":"[2026-02-17T13:51:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 440ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4df955gg7m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712667000,"log":"[2026-02-17T15:10:20Z DEBUG metrics::reporter] recorded metric active_connections value=7267","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fqn0ry20zz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712668000,"log":"[2026-02-17T14:44:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=xdqb3loz","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nbkhxvkb4p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712669000,"log":"[2026-02-17T13:53:05Z INFO actix_web::middleware::logger] GET /api/_search 500 490ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s64qrsmary","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712670000,"log":"[2026-02-17T14:20:58Z DEBUG sqlx::query] INSERT FROM users returned 607 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vr7u4r2zt5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712671000,"log":"[2026-02-17T13:58:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 274ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hhasouij7n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712672000,"log":"[2026-02-17T13:29:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4928092818027734000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n7vvqn46xs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712673000,"log":"[2026-02-17T13:25:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8017618284374357000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cjco71sjei","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712674000,"log":"[2026-02-17T14:32:38Z INFO tonic::transport::server] grpc querier disconnected client_id=8clwjrq4","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6bqk1c47qc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712675000,"log":"[2026-02-17T13:50:40Z INFO actix_web::middleware::logger] POST /api/_search 404 425ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h74d9m00zi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712676000,"log":"[2026-02-17T14:44:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=pi5zhj1w","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3kk7gpcybu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712677000,"log":"[2026-02-17T14:08:50Z DEBUG sqlx::query] DELETE FROM alerts returned 349 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-thpk3jrfi1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712678000,"log":"[2026-02-17T14:34:23Z INFO tonic::transport::server] grpc querier disconnected client_id=8113cj3n","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4pda6qifxc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712679000,"log":"[2026-02-17T14:55:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sdl925lxig","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712680000,"log":"[2026-02-17T13:42:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2f8eb91hgk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712681000,"log":"[2026-02-17T13:27:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5123612977821097000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-knq2fhxnb1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712682000,"log":"[2026-02-17T14:56:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qyhz93fjs3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712683000,"log":"[2026-02-17T14:16:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 468 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6j7yfp2wgb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712684000,"log":"[2026-02-17T13:28:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7222104466589126000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-97d0s1of1w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712685000,"log":"[2026-02-17T15:11:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6u1oi7cq0l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712686000,"log":"[2026-02-17T12:56:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ylml38jvl2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712687000,"log":"[2026-02-17T12:48:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f30e3qde3o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712688000,"log":"[2026-02-17T13:57:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 180ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jeikxao6dv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712689000,"log":"[2026-02-17T13:25:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6007979726831789000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dair9x21v7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712690000,"log":"[2026-02-17T13:30:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4925700144634746000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-toswk53jgw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712691000,"log":"[2026-02-17T13:21:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/106335040552401060.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iiv5k5vabn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712692000,"log":"[2026-02-17T12:58:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-97d2qichnb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712693000,"log":"[2026-02-17T15:03:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8jrccme2g8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712694000,"log":"[2026-02-17T15:02:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q3octmjen8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712695000,"log":"[2026-02-17T13:55:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 287ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pyq88hnx9d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712696000,"log":"[2026-02-17T14:28:27Z DEBUG sqlx::query] SELECT FROM streams returned 195 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y5zfj9n6ve","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712697000,"log":"[2026-02-17T14:54:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tcgyddgaq4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712698000,"log":"[2026-02-17T15:03:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6oyga804oz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712699000,"log":"[2026-02-17T15:07:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=7478","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v5sg89m7pj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712700000,"log":"[2026-02-17T14:16:10Z DEBUG sqlx::query] DELETE FROM alerts returned 205 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ef4mjka7l6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712701000,"log":"[2026-02-17T14:35:24Z INFO tonic::transport::server] grpc router request_received client_id=an869f94","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uz7f7a4vlo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712702000,"log":"[2026-02-17T13:02:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0m6a6shstu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712703000,"log":"[2026-02-17T14:18:58Z DEBUG sqlx::query] INSERT FROM users returned 289 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4jw7u7y4t8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712704000,"log":"[2026-02-17T14:50:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uctmjefd2z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712705000,"log":"[2026-02-17T14:53:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4olkv9g450","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712706000,"log":"[2026-02-17T14:40:18Z INFO tonic::transport::server] grpc querier disconnected client_id=1ngqic2y","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qr1jxtsd6p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712707000,"log":"[2026-02-17T15:02:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-is61hfwe42","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712708000,"log":"[2026-02-17T13:04:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l2nuoc26ay","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712709000,"log":"[2026-02-17T15:18:05Z TRACE cache::redis] cache expire key=query:result:8675","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5h09v5ujwa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712710000,"log":"[2026-02-17T14:38:14Z INFO tonic::transport::server] grpc router request_received client_id=3jgzjhjg","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-edfad0lrpy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712711000,"log":"[2026-02-17T15:02:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jy2t4tfjyn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712712000,"log":"[2026-02-17T12:46:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3xzx9vbri8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712713000,"log":"[2026-02-17T13:09:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9768966655597263000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-osvsn5ruw8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712714000,"log":"[2026-02-17T12:47:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q4lpiev09n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712715000,"log":"[2026-02-17T13:47:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 25ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-me627jacoc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712716000,"log":"[2026-02-17T13:42:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 167ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-chd71kmatn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712717000,"log":"[2026-02-17T15:07:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=2322","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ktuigzthx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712718000,"log":"[2026-02-17T13:40:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1194560438262601700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3gvnkxjd23","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712719000,"log":"[2026-02-17T13:37:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1770007180134252000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n4akpy1fpf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712720000,"log":"[2026-02-17T13:24:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5605040965608330000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g08nfs4z94","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712721000,"log":"[2026-02-17T15:12:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=6206","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-as8pw5g25c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712722000,"log":"[2026-02-17T12:39:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gyszdvz7h5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712723000,"log":"[2026-02-17T12:46:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-igrogyn9gf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712724000,"log":"[2026-02-17T12:38:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y5u3fq7hfk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712725000,"log":"[2026-02-17T13:33:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3742689395114871000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yu2n3ksdsb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712726000,"log":"[2026-02-17T13:33:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/88439013623538020.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6s9rnvxetq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712727000,"log":"[2026-02-17T15:00:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-facyz1pd1m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712728000,"log":"[2026-02-17T13:29:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5707420932839363000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-clwrryw7c0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712729000,"log":"[2026-02-17T14:47:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=9yfp2ukk","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lfpgh7348s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712730000,"log":"[2026-02-17T14:09:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 743 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-idwt2j2jmk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712731000,"log":"[2026-02-17T14:20:57Z DEBUG sqlx::query] SELECT FROM streams returned 160 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vdbzimbgc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712732000,"log":"[2026-02-17T13:36:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9469913275131636000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b7wzuspo28","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712733000,"log":"[2026-02-17T13:12:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7517731819813663000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rdqdujq9zi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712734000,"log":"[2026-02-17T13:05:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-amfoypoo7b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712735000,"log":"[2026-02-17T12:41:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vahd7f8kik","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712736000,"log":"[2026-02-17T13:26:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6599641727453749000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pqluvtaime","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712737000,"log":"[2026-02-17T12:49:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4e1ihyd9oi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712738000,"log":"[2026-02-17T14:44:14Z INFO tonic::transport::server] grpc router request_received client_id=u50k7rty","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-40z92ew978","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712739000,"log":"[2026-02-17T13:06:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zr1jf2z5kq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712740000,"log":"[2026-02-17T13:31:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1227471491161518000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sbbkx4hsb5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712741000,"log":"[2026-02-17T14:08:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 156 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-akl9ztvcdd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712742000,"log":"[2026-02-17T13:21:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/10231637329243260.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ymo0yk4bnb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712743000,"log":"[2026-02-17T14:13:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 923 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qctxfii0rl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712744000,"log":"[2026-02-17T12:50:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-180zur1nfg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712745000,"log":"[2026-02-17T13:13:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9397620376197446000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rarlnkjxww","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712746000,"log":"[2026-02-17T13:32:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3811250877272880000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5fswptw7u7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712747000,"log":"[2026-02-17T14:27:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 540 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nzir8z9x4y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712748000,"log":"[2026-02-17T14:03:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 179ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yrs3varzcy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712749000,"log":"[2026-02-17T15:03:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-121q5znoe9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712750000,"log":"[2026-02-17T13:31:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/574987160398738900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kn99rijrqj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712751000,"log":"[2026-02-17T13:08:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jw986210re","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712752000,"log":"[2026-02-17T13:28:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5594592408664354000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7dslrpwuha","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712753000,"log":"[2026-02-17T15:21:24Z TRACE cache::redis] cache hit key=stream:schema:7205","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y14v79ssuw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712754000,"log":"[2026-02-17T12:56:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-anxkoobscb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712755000,"log":"[2026-02-17T14:50:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lgl6iss2iu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712756000,"log":"[2026-02-17T14:59:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-el8go8wwvn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712757000,"log":"[2026-02-17T15:10:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=9601","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-migthlz6ek","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712758000,"log":"[2026-02-17T14:42:19Z INFO tonic::transport::server] grpc router request_received client_id=k2wgsby1","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qys6a4g5x9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712759000,"log":"[2026-02-17T13:27:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7897645067069183000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c8q9n0nc9x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712760000,"log":"[2026-02-17T15:20:53Z TRACE cache::redis] cache expire key=org:config:8799","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z0ob7cqmhg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712761000,"log":"[2026-02-17T14:18:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 290 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yg14kwz6jg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712762000,"log":"[2026-02-17T13:14:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4194742309506056700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bf2hrycc3t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712763000,"log":"[2026-02-17T13:12:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5777379954075255000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z0iz4rnjcn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712764000,"log":"[2026-02-17T13:57:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 62ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zmf6da0ioc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712765000,"log":"[2026-02-17T12:36:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wqbv4bfqs3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712766000,"log":"[2026-02-17T15:14:10Z TRACE cache::redis] cache delete key=query:result:5209","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yv5m2q8t55","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712767000,"log":"[2026-02-17T13:03:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i8aw5v75q8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712768000,"log":"[2026-02-17T13:46:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 181ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vn7wis1l1r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712769000,"log":"[2026-02-17T14:04:45Z INFO actix_web::middleware::logger] GET /api/_search 403 65ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ey2isivdmt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712770000,"log":"[2026-02-17T13:46:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 379ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wxb50nugca","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712771000,"log":"[2026-02-17T14:37:18Z INFO tonic::transport::server] grpc querier disconnected client_id=wxmkkfl7","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m1zi4rcqxd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712772000,"log":"[2026-02-17T13:34:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4235593596369100300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9dnjjd9l1x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712773000,"log":"[2026-02-17T12:43:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3165w6jvxh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712774000,"log":"[2026-02-17T13:34:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1612191036825905200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pos7g7cb48","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712775000,"log":"[2026-02-17T15:17:37Z TRACE cache::redis] cache miss key=user:session:9229","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a6drgoxz2k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712776000,"log":"[2026-02-17T13:04:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lbq03bwm82","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712777000,"log":"[2026-02-17T14:04:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 390ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-84rjv8c1mw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712778000,"log":"[2026-02-17T13:34:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6279369357875683000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ykstgakk6h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712779000,"log":"[2026-02-17T13:48:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 326ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p3phmg1opl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712780000,"log":"[2026-02-17T13:49:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 130ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a5vu28cfmy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712781000,"log":"[2026-02-17T15:03:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3ugrv8issq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712782000,"log":"[2026-02-17T13:53:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 338ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v2ukpmv2fe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712783000,"log":"[2026-02-17T14:30:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 42 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ru73cwurr2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712784000,"log":"[2026-02-17T15:15:02Z TRACE cache::redis] cache get key=user:session:2792","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lzk0ye36qi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712785000,"log":"[2026-02-17T13:53:00Z INFO actix_web::middleware::logger] POST /api/_search 204 159ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nh193opimx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712786000,"log":"[2026-02-17T14:24:08Z DEBUG sqlx::query] INSERT FROM users returned 754 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-aynlt9sjsu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712787000,"log":"[2026-02-17T12:36:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-facajedb55","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712788000,"log":"[2026-02-17T14:07:33Z DEBUG sqlx::query] INSERT FROM users returned 718 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-45900tmw5v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712789000,"log":"[2026-02-17T12:57:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3xjnse3ehe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712790000,"log":"[2026-02-17T13:55:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 308ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1std3dy67h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712791000,"log":"[2026-02-17T14:10:55Z DEBUG sqlx::query] DELETE FROM alerts returned 136 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-metz44t1r6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712792000,"log":"[2026-02-17T15:00:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uj37bmi0w6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712793000,"log":"[2026-02-17T14:43:54Z INFO tonic::transport::server] grpc router request_received client_id=louji8bi","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5z7o19kras","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712794000,"log":"[2026-02-17T13:39:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/213323534105291940.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-396uq0wbht","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712795000,"log":"[2026-02-17T12:52:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x8c871mlme","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712796000,"log":"[2026-02-17T14:13:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 917 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gs441x5u10","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712797000,"log":"[2026-02-17T13:42:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 17ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hsqgo6jimr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712798000,"log":"[2026-02-17T15:15:36Z TRACE cache::redis] cache hit key=auth:token:3425","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z07kiflbe9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712799000,"log":"[2026-02-17T13:05:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dgys7eurn1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712800000,"log":"[2026-02-17T14:14:23Z DEBUG sqlx::query] INSERT FROM users returned 468 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fyu1g2a1fy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712801000,"log":"[2026-02-17T12:45:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0834pzmef7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712802000,"log":"[2026-02-17T12:43:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-27r9nni7pj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712803000,"log":"[2026-02-17T14:30:45Z DEBUG sqlx::query] DELETE FROM alerts returned 966 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5kfo40in2l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712804000,"log":"[2026-02-17T13:40:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/526692186361565900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-byidzv9j4m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712805000,"log":"[2026-02-17T12:50:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-taqjfyrgql","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712806000,"log":"[2026-02-17T12:38:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fwjry2i3td","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712807000,"log":"[2026-02-17T15:19:39Z TRACE cache::redis] cache set key=stream:schema:7627","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i7bfqeqig9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712808000,"log":"[2026-02-17T12:39:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-br7h106skm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712809000,"log":"[2026-02-17T13:19:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2177909063531571200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9k3nl4ya0b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712810000,"log":"[2026-02-17T14:45:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=jmoknsqj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e4qcq1g0u4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712811000,"log":"[2026-02-17T14:20:42Z DEBUG sqlx::query] SELECT FROM streams returned 607 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ur3l0bla3y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712812000,"log":"[2026-02-17T12:41:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8r9wlubp8i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712813000,"log":"[2026-02-17T13:22:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5728172742761464000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9taoq9uoi2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712814000,"log":"[2026-02-17T14:23:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 260 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pl8pjsrk13","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712815000,"log":"[2026-02-17T15:08:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=5148","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-viq8jzgm42","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712816000,"log":"[2026-02-17T14:49:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xmncuxct34","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712817000,"log":"[2026-02-17T12:44:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l5tmx5lns","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712818000,"log":"[2026-02-17T12:51:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-23duaw1ya9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712819000,"log":"[2026-02-17T15:15:58Z TRACE cache::redis] cache delete key=org:config:2191","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o77zt1jlhp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712820000,"log":"[2026-02-17T14:04:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 274ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5sq81zn63m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712821000,"log":"[2026-02-17T14:26:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 173 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k7ojxrgobq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712822000,"log":"[2026-02-17T15:19:17Z TRACE cache::redis] cache expire key=user:session:2474","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fptk8vwmaq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712823000,"log":"[2026-02-17T13:47:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 421ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3uc14mh3dx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712824000,"log":"[2026-02-17T13:48:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 251ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vk0nkwj6qg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712825000,"log":"[2026-02-17T13:43:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 468ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2usdtzs7j1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712826000,"log":"[2026-02-17T13:31:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6373464362656300000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-biplgcy25j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712827000,"log":"[2026-02-17T14:57:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-20act713bn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712828000,"log":"[2026-02-17T13:59:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 305ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-59v2vslvf6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712829000,"log":"[2026-02-17T13:31:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2874314222967313400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ekl2aje9ih","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712830000,"log":"[2026-02-17T13:50:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 314ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mmgkkcvk5v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712831000,"log":"[2026-02-17T13:37:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3296639753636774400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-frk9wmhfbi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712832000,"log":"[2026-02-17T12:38:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tjdzxkdiwd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712833000,"log":"[2026-02-17T15:14:46Z TRACE cache::redis] cache delete key=auth:token:6206","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nurgn54ton","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712834000,"log":"[2026-02-17T13:33:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5394810135940678000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4tw0t4ruqz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712835000,"log":"[2026-02-17T14:43:42Z INFO tonic::transport::server] grpc ingester connected client_id=yixpruhw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wjksm3lkhx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712836000,"log":"[2026-02-17T14:21:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 733 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p9aeimvzhl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712837000,"log":"[2026-02-17T14:10:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 419 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h6rvfs72r2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712838000,"log":"[2026-02-17T12:41:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cz723wd2uh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712839000,"log":"[2026-02-17T14:07:57Z DEBUG sqlx::query] SELECT FROM streams returned 935 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mp63bc5pns","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712840000,"log":"[2026-02-17T15:12:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6072","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uyz41tef7k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712841000,"log":"[2026-02-17T15:05:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lvnthp9iij","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712842000,"log":"[2026-02-17T13:58:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 332ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p41ykeyrbs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712843000,"log":"[2026-02-17T12:50:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-37ccdafiu7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712844000,"log":"[2026-02-17T14:47:57Z INFO tonic::transport::server] grpc ingester connected client_id=j57f5gk8","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nu56vng7l9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712845000,"log":"[2026-02-17T14:05:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 250ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1nasu6k6uw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712846000,"log":"[2026-02-17T14:17:07Z DEBUG sqlx::query] SELECT FROM streams returned 336 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cw2qwg4zwl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712847000,"log":"[2026-02-17T14:20:37Z DEBUG sqlx::query] SELECT FROM streams returned 122 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mqv484qc8d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712848000,"log":"[2026-02-17T13:29:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8742456879248761000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kl1knyob8i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712849000,"log":"[2026-02-17T12:52:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ezbumrjm42","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712850000,"log":"[2026-02-17T14:49:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-srseyfukbr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712851000,"log":"[2026-02-17T14:40:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=bjz4rg59","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p0zny83jps","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712852000,"log":"[2026-02-17T14:44:54Z INFO tonic::transport::server] grpc router request_received client_id=q1rg783h","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7skgr5s0du","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712853000,"log":"[2026-02-17T13:01:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kfhlksfyji","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712854000,"log":"[2026-02-17T14:34:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=4ty3j3as","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oefdhwug22","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712855000,"log":"[2026-02-17T14:30:28Z DEBUG sqlx::query] INSERT FROM users returned 836 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j5zg5k0ld2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712856000,"log":"[2026-02-17T13:06:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fgsmfskomh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712857000,"log":"[2026-02-17T13:39:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8653476625709640000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j7a4nybi7t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712858000,"log":"[2026-02-17T14:10:42Z DEBUG sqlx::query] SELECT FROM streams returned 512 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hcybsndyq3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712859000,"log":"[2026-02-17T14:56:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7otwhymikx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712860000,"log":"[2026-02-17T14:22:08Z DEBUG sqlx::query] INSERT FROM users returned 245 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ye63w2h628","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712861000,"log":"[2026-02-17T14:19:27Z DEBUG sqlx::query] SELECT FROM streams returned 456 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7aa4ft5l21","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712862000,"log":"[2026-02-17T14:59:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dtfbdawgcy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712863000,"log":"[2026-02-17T13:37:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2316431396971889700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xabspabqf5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712864000,"log":"[2026-02-17T13:13:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5192083015261557000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q4frbctfhu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712865000,"log":"[2026-02-17T12:57:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1iil1fsx10","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712866000,"log":"[2026-02-17T14:21:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 611 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e8gz6s30p2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712867000,"log":"[2026-02-17T14:03:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 2ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7zp1owj3vk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712868000,"log":"[2026-02-17T14:23:57Z DEBUG sqlx::query] SELECT FROM streams returned 397 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lyezs53ome","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712869000,"log":"[2026-02-17T13:39:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9354905500549280000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pf3ddkxeo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712870000,"log":"[2026-02-17T15:10:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4522","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2msaa8l9po","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712871000,"log":"[2026-02-17T14:30:00Z DEBUG sqlx::query] DELETE FROM alerts returned 163 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t64bhfsdvd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712872000,"log":"[2026-02-17T14:47:48Z INFO tonic::transport::server] grpc querier disconnected client_id=imcchzx6","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nrshvh9u8i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712873000,"log":"[2026-02-17T15:06:50Z DEBUG metrics::reporter] recorded metric active_connections value=2725","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-izmlwtdhln","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712874000,"log":"[2026-02-17T13:59:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 70ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ew9u348sbk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712875000,"log":"[2026-02-17T12:52:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h1pcxip5od","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712876000,"log":"[2026-02-17T13:35:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7177274278906469000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gsqbo1sskf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712877000,"log":"[2026-02-17T13:48:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 104ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pqeib8usl6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712878000,"log":"[2026-02-17T12:49:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2p3qql42og","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712879000,"log":"[2026-02-17T14:04:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 7ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5mv9gp4kg0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712880000,"log":"[2026-02-17T14:54:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i2bjv164ag","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712881000,"log":"[2026-02-17T15:08:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5900","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mevvxz2cfc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712882000,"log":"[2026-02-17T13:24:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9698656857236451000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-elmomymo9h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712883000,"log":"[2026-02-17T13:55:05Z INFO actix_web::middleware::logger] GET /api/_search 201 200ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xyfocqgmda","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712884000,"log":"[2026-02-17T13:59:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 141ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oz8obd3ynu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712885000,"log":"[2026-02-17T13:41:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8823953207327524000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s4a08rwv2u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712886000,"log":"[2026-02-17T14:56:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ue0nqd3u9i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712887000,"log":"[2026-02-17T13:14:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4771729903837299000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-otf5o8xhfg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712888000,"log":"[2026-02-17T14:09:05Z DEBUG sqlx::query] DELETE FROM alerts returned 643 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-imuh0su5jn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712889000,"log":"[2026-02-17T14:51:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qv0z8e7ydx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712890000,"log":"[2026-02-17T13:22:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2940796356249755000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w6yl8cwar8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712891000,"log":"[2026-02-17T13:50:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 265ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-01abtc0kyd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712892000,"log":"[2026-02-17T14:04:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 31ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ipszbz7muq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712893000,"log":"[2026-02-17T12:55:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b05hj1hiyu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712894000,"log":"[2026-02-17T15:01:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hczarngyrf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712895000,"log":"[2026-02-17T14:43:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=5dq6unr9","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zwzyxl45ml","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712896000,"log":"[2026-02-17T12:52:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kqvtv75mi1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712897000,"log":"[2026-02-17T13:54:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 289ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-91y1s5zc29","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712898000,"log":"[2026-02-17T14:59:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2nk2sp1oyn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712899000,"log":"[2026-02-17T15:14:26Z TRACE cache::redis] cache get key=auth:token:1254","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v7nit12l8l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712900000,"log":"[2026-02-17T12:49:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7ftqsya94a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712901000,"log":"[2026-02-17T13:28:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5726007869269604000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8ixp2pvhu2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712902000,"log":"[2026-02-17T12:55:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qttu77os3f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712903000,"log":"[2026-02-17T12:45:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6zwy3tok5s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712904000,"log":"[2026-02-17T13:09:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2053528422779096300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fk6wj0ouwv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712905000,"log":"[2026-02-17T13:44:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 22ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-top3kch97a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712906000,"log":"[2026-02-17T14:00:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 251ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fh3mun2hha","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712907000,"log":"[2026-02-17T13:40:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6460327380611177000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ljbjxacjyf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712908000,"log":"[2026-02-17T14:36:58Z INFO tonic::transport::server] grpc querier disconnected client_id=ycf2x42s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bp05amkmbf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712909000,"log":"[2026-02-17T13:16:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/129288441625801100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wqjf2h28wg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712910000,"log":"[2026-02-17T12:57:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zfyiirdkaz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712911000,"log":"[2026-02-17T14:56:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w1q4zwfw0g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712912000,"log":"[2026-02-17T14:09:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 546 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lq93kax4fu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712913000,"log":"[2026-02-17T12:37:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p6da9k8b5k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712914000,"log":"[2026-02-17T14:56:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-znyv3ng3p1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712915000,"log":"[2026-02-17T13:53:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 388ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u852tt7bbn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712916000,"log":"[2026-02-17T13:54:25Z INFO actix_web::middleware::logger] GET /api/_search 404 383ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0qs2fj9xqk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712917000,"log":"[2026-02-17T12:36:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nrdr29cdwu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712918000,"log":"[2026-02-17T14:45:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=m2y1ygsc","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-meocjnn7zr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712919000,"log":"[2026-02-17T13:15:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4566530518454646000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rzg1brpynu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712920000,"log":"[2026-02-17T14:36:22Z INFO tonic::transport::server] grpc ingester connected client_id=90ui7hxv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o0oes3ts28","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712921000,"log":"[2026-02-17T13:55:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 46ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rtpyqnt156","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712922000,"log":"[2026-02-17T14:00:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 233ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-flg33fy4ca","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712923000,"log":"[2026-02-17T13:03:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4lv67996fs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712924000,"log":"[2026-02-17T15:15:25Z TRACE cache::redis] cache miss key=query:result:5538","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a58fwu0wi5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712925000,"log":"[2026-02-17T14:58:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tyzqolr6h8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712926000,"log":"[2026-02-17T15:16:17Z TRACE cache::redis] cache expire key=user:session:6439","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b5ll3ozxxy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712927000,"log":"[2026-02-17T12:48:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b7u8m9jb1y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712928000,"log":"[2026-02-17T14:52:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-46oe1j1352","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712929000,"log":"[2026-02-17T14:54:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-osxvkcbvwi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712930000,"log":"[2026-02-17T13:58:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ecr45lgs85","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712931000,"log":"[2026-02-17T15:06:35Z DEBUG metrics::reporter] recorded metric active_connections value=8913","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9ajt4u7tmx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712932000,"log":"[2026-02-17T13:27:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4146320720385074700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1fplomh6yo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712933000,"log":"[2026-02-17T13:24:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8376577835535457000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a3tpwrse9d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712934000,"log":"[2026-02-17T12:57:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i6cc4ijkfn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712935000,"log":"[2026-02-17T14:15:57Z DEBUG sqlx::query] SELECT FROM streams returned 405 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3tao8fpvxl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712936000,"log":"[2026-02-17T14:39:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=b9vv9g2x","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9o5mchvd4c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712937000,"log":"[2026-02-17T13:04:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-znows9uh3s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712938000,"log":"[2026-02-17T15:11:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9871","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kvmdjnqc32","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712939000,"log":"[2026-02-17T15:18:17Z TRACE cache::redis] cache expire key=user:session:5638","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vzwcymp5yb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712940000,"log":"[2026-02-17T15:16:42Z TRACE cache::redis] cache hit key=user:session:2896","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yva7g5fk3s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712941000,"log":"[2026-02-17T14:08:32Z DEBUG sqlx::query] SELECT FROM streams returned 940 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-retmlix68m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712942000,"log":"[2026-02-17T13:28:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3155524190444963000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zehrrvu1ih","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712943000,"log":"[2026-02-17T13:47:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 266ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xfqr3t9mxb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712944000,"log":"[2026-02-17T15:12:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4539","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rnkgprp33t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712945000,"log":"[2026-02-17T14:42:37Z INFO tonic::transport::server] grpc ingester connected client_id=mmyjm9pm","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xiyeetksk2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712946000,"log":"[2026-02-17T13:34:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9671469252905670000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ih7q1nbwij","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712947000,"log":"[2026-02-17T15:01:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5q83sq60mj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712948000,"log":"[2026-02-17T15:04:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0sr4zcxisb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712949000,"log":"[2026-02-17T13:44:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 379ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q7oudyiae5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712950000,"log":"[2026-02-17T14:11:25Z DEBUG sqlx::query] DELETE FROM alerts returned 692 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ais3fx43k4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712951000,"log":"[2026-02-17T14:53:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pjxb5l8dy2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712952000,"log":"[2026-02-17T13:34:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6436947220033715000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0dd9tp4njq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712953000,"log":"[2026-02-17T15:18:45Z TRACE cache::redis] cache set key=query:result:897","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w3jdvt7tdc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712954000,"log":"[2026-02-17T15:05:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8251","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-raraqsoi8z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712955000,"log":"[2026-02-17T14:08:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 812 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vdnvx5g4oo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712956000,"log":"[2026-02-17T14:05:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 425ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a9bn4z4qw2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712957000,"log":"[2026-02-17T13:45:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 335ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ornsu8i7ei","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712958000,"log":"[2026-02-17T13:17:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6368058059834727000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u47zau9zuw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712959000,"log":"[2026-02-17T14:32:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=evilswlu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fl0zdudq7f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712960000,"log":"[2026-02-17T14:08:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 239 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qmhbyq6sqa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712961000,"log":"[2026-02-17T15:04:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aynsi0hp2h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712962000,"log":"[2026-02-17T14:27:53Z DEBUG sqlx::query] INSERT FROM users returned 505 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lp8p7i4olm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712963000,"log":"[2026-02-17T13:02:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uhcg298slj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712964000,"log":"[2026-02-17T13:01:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gz2vxhypio","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712965000,"log":"[2026-02-17T14:36:12Z INFO tonic::transport::server] grpc ingester connected client_id=4narx9hr","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4yzuinpy88","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712966000,"log":"[2026-02-17T14:16:48Z DEBUG sqlx::query] INSERT FROM users returned 230 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9v8me7cp29","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712967000,"log":"[2026-02-17T14:25:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 319 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wm6qy6ojza","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712968000,"log":"[2026-02-17T13:49:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 332ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lkftunlzpc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712969000,"log":"[2026-02-17T14:52:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a3dqvqfndi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712970000,"log":"[2026-02-17T13:37:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8232848883302011000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a3wpm6hnxl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712971000,"log":"[2026-02-17T13:22:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/155837529227114140.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iv2qmotib7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712972000,"log":"[2026-02-17T14:51:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-siuhph14gz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712973000,"log":"[2026-02-17T15:11:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=6838","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1wrj27jic7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712974000,"log":"[2026-02-17T14:40:53Z INFO tonic::transport::server] grpc querier disconnected client_id=1lehcps7","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n6xh7fjwyd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712975000,"log":"[2026-02-17T13:00:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sa15kaflyn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712976000,"log":"[2026-02-17T13:20:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2314189008524403000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-esnh0yy2i2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712977000,"log":"[2026-02-17T14:48:28Z INFO tonic::transport::server] grpc querier disconnected client_id=psc72w4d","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pziqrgp7j8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712978000,"log":"[2026-02-17T14:43:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=sev7d6oz","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8qsa1ei3bc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712979000,"log":"[2026-02-17T12:55:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5elbuss9ac","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712980000,"log":"[2026-02-17T12:51:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o3hwtsh2ak","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712981000,"log":"[2026-02-17T12:48:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mxsidee8pe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712982000,"log":"[2026-02-17T14:15:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 306 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6jxkudtxnr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712983000,"log":"[2026-02-17T14:29:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 423 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-le87nxc2y9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712984000,"log":"[2026-02-17T15:10:50Z DEBUG metrics::reporter] recorded metric active_connections value=4990","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xi99r1371u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712985000,"log":"[2026-02-17T13:54:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 240ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bc9pm2kedc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712986000,"log":"[2026-02-17T13:06:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lbr5yl8qj8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712987000,"log":"[2026-02-17T13:19:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4413249265930452500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jk6ew0mtjl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712988000,"log":"[2026-02-17T14:46:48Z INFO tonic::transport::server] grpc querier disconnected client_id=9ek6igvv","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qxoxya4tcv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712989000,"log":"[2026-02-17T15:06:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7177","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iz4egw55is","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712990000,"log":"[2026-02-17T13:18:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8464947161558769000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-857yjyelqv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712991000,"log":"[2026-02-17T14:13:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 652 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f7b37tn4il","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712992000,"log":"[2026-02-17T13:45:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 457ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nlhnn8p2iv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712993000,"log":"[2026-02-17T13:19:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8176510665596541000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rse06xo948","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331712994000,"log":"[2026-02-17T13:31:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9759372630681795000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-922irt8kdf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712995000,"log":"[2026-02-17T15:16:20Z TRACE cache::redis] cache get key=query:result:3209","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oy669j1gwo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712996000,"log":"[2026-02-17T12:36:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i8t8sxnp9b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331712997000,"log":"[2026-02-17T14:04:25Z INFO actix_web::middleware::logger] GET /api/_search 400 496ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vf0qyhpuax","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712998000,"log":"[2026-02-17T13:27:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5146210798188455000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xelw2udq52","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331712999000,"log":"[2026-02-17T14:19:02Z DEBUG sqlx::query] SELECT FROM streams returned 890 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j03dtkzjl5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713000000,"log":"[2026-02-17T13:13:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8335156528640390000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w983mqdfp7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713001000,"log":"[2026-02-17T12:44:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-adxnlak6sa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713002000,"log":"[2026-02-17T14:47:07Z INFO tonic::transport::server] grpc ingester connected client_id=37ehzbc4","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gb8ey4jco8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713003000,"log":"[2026-02-17T12:48:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-js2ia9gnr2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713004000,"log":"[2026-02-17T13:16:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6691957990354913000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-77q7fgpu8t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713005000,"log":"[2026-02-17T12:57:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c7xrhss82w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713006000,"log":"[2026-02-17T15:03:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lakjpobpoe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713007000,"log":"[2026-02-17T13:33:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/111391771821403020.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w5s1ixaatg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713008000,"log":"[2026-02-17T14:02:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 213ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-osm8pe11xi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713009000,"log":"[2026-02-17T12:36:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-afxoqrjxbi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713010000,"log":"[2026-02-17T14:55:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rxx0wy3tk2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713011000,"log":"[2026-02-17T13:40:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8029827182474596000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3xz7tftun0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713012000,"log":"[2026-02-17T13:03:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bf27it37ri","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713013000,"log":"[2026-02-17T13:34:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/705615433078239400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d2v3ddecf2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713014000,"log":"[2026-02-17T15:18:12Z TRACE cache::redis] cache hit key=user:session:5524","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bptaia23kn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713015000,"log":"[2026-02-17T13:13:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3851849269617604000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pe8bprxr09","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713016000,"log":"[2026-02-17T12:46:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gjbx1lcfzp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713017000,"log":"[2026-02-17T13:01:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xxwafxeeo4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713018000,"log":"[2026-02-17T13:56:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 245ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6pqtbsoojx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713019000,"log":"[2026-02-17T14:40:34Z INFO tonic::transport::server] grpc router request_received client_id=205azdph","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o2n3u6v7bm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713020000,"log":"[2026-02-17T13:08:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-muw6wlf4ax","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713021000,"log":"[2026-02-17T13:27:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4950486772287992000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bagw0exwsj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713022000,"log":"[2026-02-17T14:22:00Z DEBUG sqlx::query] DELETE FROM alerts returned 146 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s3d8y2ctg1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713023000,"log":"[2026-02-17T13:53:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 105ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wzsyhm22hd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713024000,"log":"[2026-02-17T12:56:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jgh8swigek","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713025000,"log":"[2026-02-17T14:19:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 943 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-84509tagcu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713026000,"log":"[2026-02-17T14:57:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ep2saep4xs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713027000,"log":"[2026-02-17T15:09:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4976","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8326xv9ruu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713028000,"log":"[2026-02-17T13:31:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9923217158280829000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-07ls771u5x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713029000,"log":"[2026-02-17T14:51:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4qxszxdyxf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713030000,"log":"[2026-02-17T13:05:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m650zb20i5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713031000,"log":"[2026-02-17T14:56:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2cga2fatc9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713032000,"log":"[2026-02-17T14:41:09Z INFO tonic::transport::server] grpc router request_received client_id=ac5elnl3","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4xgvt9usdb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713033000,"log":"[2026-02-17T14:55:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-47jhi1ckeb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713034000,"log":"[2026-02-17T13:18:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1584109489514902300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ago0nl3p85","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713035000,"log":"[2026-02-17T13:25:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6485168259153930000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qkj6f34d71","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713036000,"log":"[2026-02-17T14:48:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f4scrmmv25","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713037000,"log":"[2026-02-17T14:13:48Z DEBUG sqlx::query] INSERT FROM users returned 139 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cy65l1imm9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713038000,"log":"[2026-02-17T14:05:00Z INFO actix_web::middleware::logger] POST /api/_search 204 224ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ywikcnpiit","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713039000,"log":"[2026-02-17T14:16:25Z DEBUG sqlx::query] DELETE FROM alerts returned 94 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f6eox3sbhu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713040000,"log":"[2026-02-17T13:15:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1681885314493461000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ovshwfd7lv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713041000,"log":"[2026-02-17T14:42:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rqzaqsnt","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8awmtqr2lw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713042000,"log":"[2026-02-17T14:28:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 642 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5cy6ehn9l7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713043000,"log":"[2026-02-17T14:49:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ge2nmx3f7m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713044000,"log":"[2026-02-17T12:45:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k31cob3w4w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713045000,"log":"[2026-02-17T14:01:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 286ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wzp3t0z1ws","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713046000,"log":"[2026-02-17T13:15:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9154240934519544000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pp7l2lf1o8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713047000,"log":"[2026-02-17T14:52:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c502y5oj4h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713048000,"log":"[2026-02-17T12:52:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-86r6uacpxj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713049000,"log":"[2026-02-17T13:26:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4086844662674098000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dd1ytuze33","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713050000,"log":"[2026-02-17T14:14:22Z DEBUG sqlx::query] SELECT FROM streams returned 987 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nax61eydvm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713051000,"log":"[2026-02-17T13:49:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 483ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z67fktfa63","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713052000,"log":"[2026-02-17T13:07:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-knqy00uezl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713053000,"log":"[2026-02-17T14:59:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y1jyoapo0c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713054000,"log":"[2026-02-17T12:37:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6nbnn0q7gc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713055000,"log":"[2026-02-17T12:56:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3t5iu6irbd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713056000,"log":"[2026-02-17T14:21:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 351 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-93vwhrslkf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713057000,"log":"[2026-02-17T13:12:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/220152131634936420.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8gfo463bbe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713058000,"log":"[2026-02-17T14:12:45Z DEBUG sqlx::query] DELETE FROM alerts returned 355 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-928li31e4d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713059000,"log":"[2026-02-17T12:35:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b2z0sqikjq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713060000,"log":"[2026-02-17T13:56:25Z INFO actix_web::middleware::logger] GET /api/_search 200 163ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lmzqp7blng","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713061000,"log":"[2026-02-17T14:32:58Z INFO tonic::transport::server] grpc querier disconnected client_id=pz3zedce","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t4k3bh5t27","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713062000,"log":"[2026-02-17T14:10:33Z DEBUG sqlx::query] INSERT FROM users returned 365 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ydnjn3ddvu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713063000,"log":"[2026-02-17T13:11:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6298140757808443000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zaxmcbb0p9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713064000,"log":"[2026-02-17T14:44:52Z INFO tonic::transport::server] grpc ingester connected client_id=jeyvyxgv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0deirq0kie","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713065000,"log":"[2026-02-17T12:49:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2902czshki","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713066000,"log":"[2026-02-17T13:45:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 317ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s2xejlh1y3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713067000,"log":"[2026-02-17T14:50:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pdd1676ans","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713068000,"log":"[2026-02-17T15:11:00Z DEBUG metrics::reporter] recorded metric active_connections value=7926","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-52scs202ja","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713069000,"log":"[2026-02-17T14:18:05Z DEBUG sqlx::query] DELETE FROM alerts returned 885 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c4ayexpf8q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713070000,"log":"[2026-02-17T14:58:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k66usepd13","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713071000,"log":"[2026-02-17T14:39:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=56b3x1c5","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z2cspi7ejx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713072000,"log":"[2026-02-17T14:59:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-83oencu46r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713073000,"log":"[2026-02-17T13:57:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 80ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eo54qpehh0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713074000,"log":"[2026-02-17T14:07:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 151 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0l0k7ypok8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713075000,"log":"[2026-02-17T15:04:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kv0ywkg3jv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713076000,"log":"[2026-02-17T12:59:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0y4bda0xhy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713077000,"log":"[2026-02-17T12:50:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2bckw1egry","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713078000,"log":"[2026-02-17T15:14:29Z TRACE cache::redis] cache expire key=stream:schema:8824","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w8les8j6b9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713079000,"log":"[2026-02-17T13:36:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1725669895991966000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1kpn8ddtfk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713080000,"log":"[2026-02-17T14:19:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 310 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2kw83ekm7i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713081000,"log":"[2026-02-17T13:15:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5174665506740061000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p8hpk6nklg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713082000,"log":"[2026-02-17T13:53:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 35ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wyd0ycwmtq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713083000,"log":"[2026-02-17T13:16:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6551112529757925000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wutjhufy7m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713084000,"log":"[2026-02-17T12:52:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fi02slu6z6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713085000,"log":"[2026-02-17T13:55:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 120ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6fztx8dbbo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713086000,"log":"[2026-02-17T15:19:19Z TRACE cache::redis] cache miss key=stream:schema:51","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bqs45oo2ov","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713087000,"log":"[2026-02-17T13:40:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4456724760499992600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-03uq52z8op","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713088000,"log":"[2026-02-17T13:45:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 321ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4dqrymy5sx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713089000,"log":"[2026-02-17T14:39:02Z INFO tonic::transport::server] grpc ingester connected client_id=hll4pi7o","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-olcfja45hi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713090000,"log":"[2026-02-17T12:38:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3evvcx4qxo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713091000,"log":"[2026-02-17T13:56:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 373ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9gl2e109rf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713092000,"log":"[2026-02-17T13:21:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7866646833427453000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pz4cvibeyd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713093000,"log":"[2026-02-17T15:08:55Z DEBUG metrics::reporter] recorded metric active_connections value=560","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pri7dnce7b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713094000,"log":"[2026-02-17T14:14:45Z DEBUG sqlx::query] DELETE FROM alerts returned 837 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b0sot55vih","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713095000,"log":"[2026-02-17T13:59:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 225ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m3eukm47am","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713096000,"log":"[2026-02-17T15:11:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=5250","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g32l4vi988","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713097000,"log":"[2026-02-17T13:33:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1556815335577452800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iuwk2zaizm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713098000,"log":"[2026-02-17T14:20:08Z DEBUG sqlx::query] INSERT FROM users returned 135 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qoggwoixk0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713099000,"log":"[2026-02-17T13:40:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1832125184530992600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-akt7c5q013","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713100000,"log":"[2026-02-17T14:45:59Z INFO tonic::transport::server] grpc router request_received client_id=8uspzb8z","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8khe5yurrw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713101000,"log":"[2026-02-17T13:53:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 392ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t9xbjelpcb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713102000,"log":"[2026-02-17T13:12:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6222774865952756000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0zfekjuo5x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713103000,"log":"[2026-02-17T14:28:07Z DEBUG sqlx::query] SELECT FROM streams returned 911 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-danei824lb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713104000,"log":"[2026-02-17T14:10:57Z DEBUG sqlx::query] SELECT FROM streams returned 572 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tc9yyfprca","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713105000,"log":"[2026-02-17T13:34:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/397551211124738100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u5bw02p98z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713106000,"log":"[2026-02-17T15:11:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2017","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rbhoivlx68","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713107000,"log":"[2026-02-17T14:39:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=yse2ynu3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hxm7j3cqw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713108000,"log":"[2026-02-17T14:35:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=ndt73e1g","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bbdvuopn4h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713109000,"log":"[2026-02-17T13:12:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8453363781614081000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v6odv8rw4d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713110000,"log":"[2026-02-17T13:34:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2337168739281712000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ks2kegbnpu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713111000,"log":"[2026-02-17T12:52:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nowib1wyyc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713112000,"log":"[2026-02-17T14:17:47Z DEBUG sqlx::query] SELECT FROM streams returned 659 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-el60iak0sp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713113000,"log":"[2026-02-17T13:31:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7601266857615002000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eoofvjja3t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713114000,"log":"[2026-02-17T13:26:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9692938786710254000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a3849ifeeu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713115000,"log":"[2026-02-17T15:12:05Z DEBUG metrics::reporter] recorded metric active_connections value=9746","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b90zgenzv5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713116000,"log":"[2026-02-17T15:09:15Z DEBUG metrics::reporter] recorded metric active_connections value=1707","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-81gjv4105t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713117000,"log":"[2026-02-17T14:56:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-24dugffsm8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713118000,"log":"[2026-02-17T12:54:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ymqual4ea8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713119000,"log":"[2026-02-17T13:58:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 467ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1dqrn7q33v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713120000,"log":"[2026-02-17T14:54:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wy012zz3d2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713121000,"log":"[2026-02-17T14:41:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=unalvr76","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-882slhan40","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713122000,"log":"[2026-02-17T12:53:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mtle0guj6a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713123000,"log":"[2026-02-17T15:21:13Z TRACE cache::redis] cache miss key=org:config:5001","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qh480ur8dn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713124000,"log":"[2026-02-17T13:00:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cqt9414bmc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713125000,"log":"[2026-02-17T14:25:40Z DEBUG sqlx::query] DELETE FROM alerts returned 299 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-liiaf0egfk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713126000,"log":"[2026-02-17T13:10:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1366617557168654600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n3g0lkrt7x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713127000,"log":"[2026-02-17T13:10:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9180935535839887000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0yokzuyj94","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713128000,"log":"[2026-02-17T12:50:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u9pjpl9r3b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713129000,"log":"[2026-02-17T14:16:00Z DEBUG sqlx::query] DELETE FROM alerts returned 587 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pu42x055ij","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713130000,"log":"[2026-02-17T14:26:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 104 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j14w6g26ac","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713131000,"log":"[2026-02-17T12:53:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-joawhztglo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713132000,"log":"[2026-02-17T14:53:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kwpsadhfo1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713133000,"log":"[2026-02-17T14:05:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 152ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jzn685oudp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713134000,"log":"[2026-02-17T12:49:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ore33rl5nk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713135000,"log":"[2026-02-17T15:14:15Z TRACE cache::redis] cache set key=query:result:2875","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-77vpu8hai4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713136000,"log":"[2026-02-17T13:25:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8132598895936325000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lvfsbp6e77","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713137000,"log":"[2026-02-17T14:01:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 268ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l1z7jaufxj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713138000,"log":"[2026-02-17T13:23:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8966717019586652000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rtl5fzclap","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713139000,"log":"[2026-02-17T15:18:40Z TRACE cache::redis] cache delete key=query:result:8285","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4xy0h6rzys","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713140000,"log":"[2026-02-17T12:39:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b6kxb13ul8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713141000,"log":"[2026-02-17T13:38:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5941729662522659000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vbgihv7a6s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713142000,"log":"[2026-02-17T14:35:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=e0ytf8o7","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xbl71934lb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713143000,"log":"[2026-02-17T13:35:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/613282358247502200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3xufwyqpxl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713144000,"log":"[2026-02-17T13:30:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3269836285847778300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u99u79ylst","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713145000,"log":"[2026-02-17T13:39:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6243287617764241000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4q3u0kowzu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713146000,"log":"[2026-02-17T14:57:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c9ioyvvi34","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713147000,"log":"[2026-02-17T14:37:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=r6jqxwy2","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ky97tf5ggm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713148000,"log":"[2026-02-17T14:25:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 440 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7g1mybka5p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713149000,"log":"[2026-02-17T13:36:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6088486598148441000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-myp18i6vnq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713150000,"log":"[2026-02-17T14:23:55Z DEBUG sqlx::query] DELETE FROM alerts returned 257 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rcqjd4eg44","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713151000,"log":"[2026-02-17T12:49:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o6l7pw6rfc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713152000,"log":"[2026-02-17T12:53:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vphqcli3re","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713153000,"log":"[2026-02-17T14:35:19Z INFO tonic::transport::server] grpc router request_received client_id=h8xo76cd","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-594jahtwis","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713154000,"log":"[2026-02-17T13:35:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4997646641755444000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lrxbad5ujl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713155000,"log":"[2026-02-17T14:27:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 288 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zfgw2wp4l4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713156000,"log":"[2026-02-17T13:27:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4463653671917198300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sq5ghgt9e9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713157000,"log":"[2026-02-17T14:02:00Z INFO actix_web::middleware::logger] POST /api/_search 204 2ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h2ggrahhpx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713158000,"log":"[2026-02-17T13:27:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/928835823940590300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mqy4itai2x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713159000,"log":"[2026-02-17T13:26:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7987546212800904000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-06s0hikrmh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713160000,"log":"[2026-02-17T13:25:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/363484072198498900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ybzyzvsupe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713161000,"log":"[2026-02-17T13:43:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 21ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zi6og013qi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713162000,"log":"[2026-02-17T13:51:20Z INFO actix_web::middleware::logger] POST /api/_search 201 270ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-llcx08px2k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713163000,"log":"[2026-02-17T14:36:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=4ej5068x","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jwi7d0v3n1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713164000,"log":"[2026-02-17T14:01:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 80ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8c3sd5ynj3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713165000,"log":"[2026-02-17T15:18:47Z TRACE cache::redis] cache expire key=user:session:140","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xndc250vxr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713166000,"log":"[2026-02-17T12:40:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-inl578pasi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713167000,"log":"[2026-02-17T14:00:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 405ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g5ida13r7c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713168000,"log":"[2026-02-17T14:23:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 859 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pi5w99osiu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713169000,"log":"[2026-02-17T14:43:12Z INFO tonic::transport::server] grpc ingester connected client_id=1hsskfn3","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0gi71siaze","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713170000,"log":"[2026-02-17T13:23:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6535041690213552000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y1c736m9bq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713171000,"log":"[2026-02-17T13:41:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6120962467617921000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-09nrb1w5wx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713172000,"log":"[2026-02-17T15:08:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4944","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jawb1s4y9k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713173000,"log":"[2026-02-17T14:49:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y9j5sicbor","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713174000,"log":"[2026-02-17T14:21:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 24 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ouxpby4pck","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713175000,"log":"[2026-02-17T12:35:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h7cdpecj8o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713176000,"log":"[2026-02-17T13:06:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3h54z0a15l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713177000,"log":"[2026-02-17T12:39:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-30wkusol9v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713178000,"log":"[2026-02-17T12:46:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gbpgywya5m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713179000,"log":"[2026-02-17T13:51:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 367ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-24xdoxj5ol","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713180000,"log":"[2026-02-17T13:17:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6194088977680501000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lyjdyr3e00","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713181000,"log":"[2026-02-17T15:16:00Z TRACE cache::redis] cache hit key=query:result:5354","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pf6jsbb9fl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713182000,"log":"[2026-02-17T12:48:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-50o8b5yqxb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713183000,"log":"[2026-02-17T15:10:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=199","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0g7i5wqwd0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713184000,"log":"[2026-02-17T13:36:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6927896040362312000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-grr3hqw2a8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713185000,"log":"[2026-02-17T14:46:47Z INFO tonic::transport::server] grpc ingester connected client_id=0t2qqryx","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dx1jcqmfnb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713186000,"log":"[2026-02-17T13:04:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9f9986xc6t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713187000,"log":"[2026-02-17T13:20:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6106940218073347000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-582y2p02lu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713188000,"log":"[2026-02-17T12:50:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bra4t02aad","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713189000,"log":"[2026-02-17T14:54:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2bucjk8xes","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713190000,"log":"[2026-02-17T14:14:53Z DEBUG sqlx::query] INSERT FROM users returned 933 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7yqndiadrc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713191000,"log":"[2026-02-17T14:21:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 610 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gvj32qaov3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713192000,"log":"[2026-02-17T15:02:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rcu41dd2tg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713193000,"log":"[2026-02-17T15:02:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k6uije1gqe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713194000,"log":"[2026-02-17T15:05:30Z DEBUG metrics::reporter] recorded metric active_connections value=7890","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-anxclgnuu1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713195000,"log":"[2026-02-17T14:14:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 329 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h3ycqnu1vy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713196000,"log":"[2026-02-17T13:10:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6598522007459187000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4gtse62mhf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713197000,"log":"[2026-02-17T13:38:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8641890993225383000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qphs8wiy3n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713198000,"log":"[2026-02-17T14:43:38Z INFO tonic::transport::server] grpc querier disconnected client_id=cfrgvub0","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-caxf4d23ng","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713199000,"log":"[2026-02-17T13:00:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ghvv9ce6zt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713200000,"log":"[2026-02-17T12:51:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f7c2k8lzpe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713201000,"log":"[2026-02-17T12:43:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-40z7iy5ki2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713202000,"log":"[2026-02-17T13:11:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6150861878167777000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lvw57swdql","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713203000,"log":"[2026-02-17T15:07:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=1910","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o258lutdrg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713204000,"log":"[2026-02-17T12:44:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w4ol69v4if","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713205000,"log":"[2026-02-17T14:29:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 698 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zjddgckuxk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713206000,"log":"[2026-02-17T12:43:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6q3if6zxqc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713207000,"log":"[2026-02-17T14:26:03Z DEBUG sqlx::query] INSERT FROM users returned 120 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0u32pvnwd2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713208000,"log":"[2026-02-17T14:21:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 312 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6y7mt4mum4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713209000,"log":"[2026-02-17T14:17:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 252 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-97tuw47cit","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713210000,"log":"[2026-02-17T13:13:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1367306440321813800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-87461rbv6k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713211000,"log":"[2026-02-17T14:29:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 546 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vdzi3i81z4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713212000,"log":"[2026-02-17T14:24:58Z DEBUG sqlx::query] INSERT FROM users returned 924 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gjp7eeisvk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713213000,"log":"[2026-02-17T15:13:40Z TRACE cache::redis] cache delete key=query:result:1854","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wz2870tjpp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713214000,"log":"[2026-02-17T14:23:23Z DEBUG sqlx::query] INSERT FROM users returned 278 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qkmwrnfmfi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713215000,"log":"[2026-02-17T12:51:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pgvo9bd5pa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713216000,"log":"[2026-02-17T14:39:32Z INFO tonic::transport::server] grpc ingester connected client_id=94dbkz90","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iwfe485ek4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713217000,"log":"[2026-02-17T14:57:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tddnkf69zh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713218000,"log":"[2026-02-17T12:37:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5xs7ku2htw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713219000,"log":"[2026-02-17T12:40:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l4d8zhre0v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713220000,"log":"[2026-02-17T13:30:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3542326194127223300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9mvu1q9jmk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713221000,"log":"[2026-02-17T14:07:42Z DEBUG sqlx::query] SELECT FROM streams returned 83 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-guwkjprigu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713222000,"log":"[2026-02-17T14:49:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xsi2iu9jz3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713223000,"log":"[2026-02-17T14:02:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 171ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3zmqohatna","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713224000,"log":"[2026-02-17T15:19:40Z TRACE cache::redis] cache delete key=query:result:8121","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xz4wfwwr2x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713225000,"log":"[2026-02-17T13:05:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vsdeiwfiin","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713226000,"log":"[2026-02-17T13:04:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3opfbbxp33","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713227000,"log":"[2026-02-17T15:08:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1241","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hijdboratn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713228000,"log":"[2026-02-17T14:22:35Z DEBUG sqlx::query] DELETE FROM alerts returned 368 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lzj8xl7byj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713229000,"log":"[2026-02-17T13:55:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 225ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r6hu5o5wm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713230000,"log":"[2026-02-17T12:35:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pq7p9ukj8a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713231000,"log":"[2026-02-17T13:02:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-exkhwndgee","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713232000,"log":"[2026-02-17T13:37:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2003178251154782500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uo9fei5fyh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713233000,"log":"[2026-02-17T15:04:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kak8g1xmqm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713234000,"log":"[2026-02-17T15:08:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8746","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7xv9hiz9p2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713235000,"log":"[2026-02-17T14:34:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=lijzqu3s","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xmkjnfw4fg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713236000,"log":"[2026-02-17T14:18:25Z DEBUG sqlx::query] DELETE FROM alerts returned 553 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f63vfkrqlj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713237000,"log":"[2026-02-17T14:06:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 482ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cdnw5tvpbh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713238000,"log":"[2026-02-17T13:49:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 187ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-62m462gywx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713239000,"log":"[2026-02-17T13:17:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7586591005908538000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2pq2ojkyqk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713240000,"log":"[2026-02-17T14:43:37Z INFO tonic::transport::server] grpc ingester connected client_id=et1x0zmo","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-neb4tl9dir","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713241000,"log":"[2026-02-17T13:24:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6251792636091353000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6t5vjhuphh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713242000,"log":"[2026-02-17T12:49:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f6a3tt7kr8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713243000,"log":"[2026-02-17T14:04:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 336ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-11ruw656ry","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713244000,"log":"[2026-02-17T15:03:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-797ywfm2ii","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713245000,"log":"[2026-02-17T12:55:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zqswao617t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713246000,"log":"[2026-02-17T13:31:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9700816050589270000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0plsti95if","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713247000,"log":"[2026-02-17T13:42:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 434ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qsjui8dm1v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713248000,"log":"[2026-02-17T12:43:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c6if0sb9uy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713249000,"log":"[2026-02-17T13:27:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/284099798509180860.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-926h0lqucd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713250000,"log":"[2026-02-17T14:44:42Z INFO tonic::transport::server] grpc ingester connected client_id=80no0dww","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qqn4138wp5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713251000,"log":"[2026-02-17T13:45:05Z INFO actix_web::middleware::logger] GET /api/_search 401 88ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1kvrr42b0t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713252000,"log":"[2026-02-17T15:03:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tqa2b5q0k9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713253000,"log":"[2026-02-17T15:04:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9vw6ww8e6f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713254000,"log":"[2026-02-17T13:45:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 338ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gfumx4xfkv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713255000,"log":"[2026-02-17T14:42:48Z INFO tonic::transport::server] grpc querier disconnected client_id=wq24k5ap","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ufrce5osxr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713256000,"log":"[2026-02-17T14:11:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 833 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2y51axuod9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713257000,"log":"[2026-02-17T13:41:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1812335396496072000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h5jf64hg1v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713258000,"log":"[2026-02-17T13:53:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 432ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2qws89qc1w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713259000,"log":"[2026-02-17T12:46:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4h39o1gouf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713260000,"log":"[2026-02-17T14:40:47Z INFO tonic::transport::server] grpc ingester connected client_id=722p9yvz","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-soazkh9dsn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713261000,"log":"[2026-02-17T14:13:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 175 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-soa4u6o60h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713262000,"log":"[2026-02-17T14:36:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=49zzm5wc","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3dm59zuqqk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713263000,"log":"[2026-02-17T14:38:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=79rm2cl7","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0n52uhvh1a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713264000,"log":"[2026-02-17T13:01:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lgmokavzbm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713265000,"log":"[2026-02-17T14:32:08Z INFO tonic::transport::server] grpc querier disconnected client_id=ln5rqpj3","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s3nh960omz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713266000,"log":"[2026-02-17T14:45:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=qw7bolz7","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-74vxn60brg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713267000,"log":"[2026-02-17T15:20:20Z TRACE cache::redis] cache get key=query:result:8308","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c383zifbf5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713268000,"log":"[2026-02-17T14:48:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5r752pyv8y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713269000,"log":"[2026-02-17T14:46:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=peu3vov6","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xkr4a64b6h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713270000,"log":"[2026-02-17T14:07:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 124 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m3xsqgbjct","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713271000,"log":"[2026-02-17T13:04:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vdhca3wktd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713272000,"log":"[2026-02-17T14:25:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 942 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9ptltr51lo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713273000,"log":"[2026-02-17T12:54:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wfquc9euph","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713274000,"log":"[2026-02-17T13:37:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4367171947620593700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jlpsw885e8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713275000,"log":"[2026-02-17T12:46:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g64cbltoqz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713276000,"log":"[2026-02-17T13:31:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4543516360632336000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qsi5u7bfhw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713277000,"log":"[2026-02-17T12:58:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w9tz4zwr7j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713278000,"log":"[2026-02-17T14:26:30Z DEBUG sqlx::query] DELETE FROM alerts returned 173 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y0yrre6him","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713279000,"log":"[2026-02-17T14:20:48Z DEBUG sqlx::query] INSERT FROM users returned 599 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gz0d1ej2ky","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713280000,"log":"[2026-02-17T12:45:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u5jx104k63","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713281000,"log":"[2026-02-17T14:45:42Z INFO tonic::transport::server] grpc ingester connected client_id=m3gjvcft","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ro0k9chglt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713282000,"log":"[2026-02-17T13:41:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 386ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l49d427kfa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713283000,"log":"[2026-02-17T14:57:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sriz90f9q4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713284000,"log":"[2026-02-17T12:46:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fgidfh6f0j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713285000,"log":"[2026-02-17T14:59:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mipfp88hi6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713286000,"log":"[2026-02-17T13:15:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2096032231145834500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qmuect35ds","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713287000,"log":"[2026-02-17T14:19:12Z DEBUG sqlx::query] SELECT FROM streams returned 516 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jw62ujle1q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713288000,"log":"[2026-02-17T12:47:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4n0tv6jlqj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713289000,"log":"[2026-02-17T13:06:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2js9l9pi8s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713290000,"log":"[2026-02-17T13:10:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4232254947678133000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5tbmcrwt53","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713291000,"log":"[2026-02-17T12:37:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tedtqfloj7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713292000,"log":"[2026-02-17T14:10:35Z DEBUG sqlx::query] DELETE FROM alerts returned 562 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1pvqgkr9fj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713293000,"log":"[2026-02-17T12:39:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8zhzukc6ky","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713294000,"log":"[2026-02-17T14:45:27Z INFO tonic::transport::server] grpc ingester connected client_id=lwyx0msa","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r67q97leyv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713295000,"log":"[2026-02-17T15:17:14Z TRACE cache::redis] cache get key=stream:schema:4683","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aqnayw40tr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713296000,"log":"[2026-02-17T15:16:35Z TRACE cache::redis] cache expire key=query:result:6548","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-guc1t97x4a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713297000,"log":"[2026-02-17T14:53:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bivea8mke3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713298000,"log":"[2026-02-17T12:43:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-crgr8kg0ws","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713299000,"log":"[2026-02-17T12:38:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r56m8s949","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713300000,"log":"[2026-02-17T14:01:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 464ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-so88m95gyi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713301000,"log":"[2026-02-17T13:48:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 10ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vyt2qmx0z3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713302000,"log":"[2026-02-17T12:45:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3doxb9k1a3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713303000,"log":"[2026-02-17T13:53:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 451ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9gl2b63743","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713304000,"log":"[2026-02-17T13:57:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 107ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9tr9ixuc7a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713305000,"log":"[2026-02-17T15:03:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lbvdfltud3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713306000,"log":"[2026-02-17T15:16:27Z TRACE cache::redis] cache set key=user:session:8434","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hiynvuhfp7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713307000,"log":"[2026-02-17T14:04:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 6ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-609dq9k3ko","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713308000,"log":"[2026-02-17T14:28:45Z DEBUG sqlx::query] DELETE FROM alerts returned 746 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ssg2g363gm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713309000,"log":"[2026-02-17T12:36:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jmgwfld3ui","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713310000,"log":"[2026-02-17T12:48:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-025buym5y4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713311000,"log":"[2026-02-17T13:23:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2954325786034280000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x2zxpnq9zg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713312000,"log":"[2026-02-17T14:34:27Z INFO tonic::transport::server] grpc ingester connected client_id=kk9bq4nq","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2mtpwg4mgb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713313000,"log":"[2026-02-17T13:23:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5410646712198306000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kou3iuufj2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713314000,"log":"[2026-02-17T13:32:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3218248490119093000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dn8dymiz4u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713315000,"log":"[2026-02-17T13:30:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3417641388844978700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v0mi9grrsx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713316000,"log":"[2026-02-17T14:49:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hox5dpeaqb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713317000,"log":"[2026-02-17T14:55:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fjinytzohs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713318000,"log":"[2026-02-17T14:06:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 142ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-899wzbv2w4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713319000,"log":"[2026-02-17T13:48:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 271ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tcfxe8ppha","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713320000,"log":"[2026-02-17T14:16:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 807 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uenh23xb2a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713321000,"log":"[2026-02-17T12:42:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k5nfze906e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713322000,"log":"[2026-02-17T13:54:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 263ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0wivhbq5ex","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713323000,"log":"[2026-02-17T14:01:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 217ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s2fwkuiy49","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713324000,"log":"[2026-02-17T14:33:42Z INFO tonic::transport::server] grpc ingester connected client_id=ythvnlp6","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cfei4g2x4i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713325000,"log":"[2026-02-17T13:05:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rlcz7vyj1w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713326000,"log":"[2026-02-17T12:47:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u701kvifzq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713327000,"log":"[2026-02-17T14:21:50Z DEBUG sqlx::query] DELETE FROM alerts returned 352 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sisha1rw59","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713328000,"log":"[2026-02-17T14:14:48Z DEBUG sqlx::query] INSERT FROM users returned 776 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-onoldzbsy9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713329000,"log":"[2026-02-17T13:31:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1137796237984565200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d9ku8b4mlw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713330000,"log":"[2026-02-17T15:18:16Z TRACE cache::redis] cache delete key=auth:token:4830","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jwwj64uzn6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713331000,"log":"[2026-02-17T13:52:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 128ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mbc87y6dgu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713332000,"log":"[2026-02-17T13:04:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fmfeyz5vii","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713333000,"log":"[2026-02-17T14:03:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 85ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f1t573khtl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713334000,"log":"[2026-02-17T13:39:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2414726591029776400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0xeh39rhht","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713335000,"log":"[2026-02-17T13:39:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4560746818556149000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e3uo5uyr1b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713336000,"log":"[2026-02-17T13:38:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/54716598062729550.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o02ylt8y5x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713337000,"log":"[2026-02-17T15:01:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e99h8emv0b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713338000,"log":"[2026-02-17T12:50:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jhw9ofpfak","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713339000,"log":"[2026-02-17T15:10:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2976","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-411801o2d6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713340000,"log":"[2026-02-17T13:15:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5690151058294180000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tu3q66h5h2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713341000,"log":"[2026-02-17T14:10:05Z DEBUG sqlx::query] DELETE FROM alerts returned 492 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j8y91esmfp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713342000,"log":"[2026-02-17T14:42:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=x6zypl1z","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kwlph1xd8a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713343000,"log":"[2026-02-17T14:32:28Z INFO tonic::transport::server] grpc querier disconnected client_id=9x59yqff","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0gumjyre68","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713344000,"log":"[2026-02-17T15:12:15Z DEBUG metrics::reporter] recorded metric active_connections value=207","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p7tqvv3s9q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713345000,"log":"[2026-02-17T14:28:30Z DEBUG sqlx::query] DELETE FROM alerts returned 675 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-egodddy908","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713346000,"log":"[2026-02-17T13:59:20Z INFO actix_web::middleware::logger] POST /api/_search 401 94ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ezy9nxn9xy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713347000,"log":"[2026-02-17T13:13:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8731066188518073000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-htyxym61v4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713348000,"log":"[2026-02-17T15:05:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ixmbwuutfe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713349000,"log":"[2026-02-17T13:38:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6102962163366312000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7w8yj2v180","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713350000,"log":"[2026-02-17T14:26:27Z DEBUG sqlx::query] SELECT FROM streams returned 495 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j6zepiaex8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713351000,"log":"[2026-02-17T13:12:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/656796647514232800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jwchjmaw8c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713352000,"log":"[2026-02-17T15:19:11Z TRACE cache::redis] cache expire key=auth:token:1615","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7535gqxkxl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713353000,"log":"[2026-02-17T12:59:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ksnfb9e75u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713354000,"log":"[2026-02-17T14:04:00Z INFO actix_web::middleware::logger] POST /api/_search 403 355ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qffiuws3ak","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713355000,"log":"[2026-02-17T12:46:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o1i0fgf1x1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713356000,"log":"[2026-02-17T14:49:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4hbkjpvekr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713357000,"log":"[2026-02-17T14:00:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 311ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2jnjn4gfiv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713358000,"log":"[2026-02-17T14:02:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 280ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1w66nd74ok","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713359000,"log":"[2026-02-17T13:05:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4tu5lstkwa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713360000,"log":"[2026-02-17T15:02:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ckkvpgzxyc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713361000,"log":"[2026-02-17T12:42:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oelgjr022d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713362000,"log":"[2026-02-17T13:22:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4932629713283731000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ifze07sbzz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713363000,"log":"[2026-02-17T13:54:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 171ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dxjjsf6g52","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713364000,"log":"[2026-02-17T13:33:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6023523466663345000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5wvu9vei14","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713365000,"log":"[2026-02-17T14:29:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 805 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ozwy8487dh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713366000,"log":"[2026-02-17T14:09:55Z DEBUG sqlx::query] DELETE FROM alerts returned 933 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2x931u9fyu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713367000,"log":"[2026-02-17T13:37:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8583904654604700000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4guf7uev3s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713368000,"log":"[2026-02-17T12:40:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o2k2u4ejtt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713369000,"log":"[2026-02-17T13:20:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4374094572911084500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r5csacoi0e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713370000,"log":"[2026-02-17T14:00:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 455ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tdz2xc0nrh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713371000,"log":"[2026-02-17T14:15:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 673 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ihdon35tmm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713372000,"log":"[2026-02-17T13:37:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7435857400434049000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e3xqh562t9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713373000,"log":"[2026-02-17T12:38:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dtfpp9ry1t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713374000,"log":"[2026-02-17T14:35:39Z INFO tonic::transport::server] grpc router request_received client_id=8jthuk4v","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uor9ltjoh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713375000,"log":"[2026-02-17T13:41:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2946910616604090400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0wwzt8ymdj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713376000,"log":"[2026-02-17T15:21:43Z TRACE cache::redis] cache miss key=org:config:4045","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cxnb2ohcxr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713377000,"log":"[2026-02-17T13:19:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9204737422085587000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pjymffgl5z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713378000,"log":"[2026-02-17T14:03:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 223ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e11gyjs5xh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713379000,"log":"[2026-02-17T14:11:50Z DEBUG sqlx::query] DELETE FROM alerts returned 233 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bo4ewrpp18","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713380000,"log":"[2026-02-17T14:17:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 569 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-900nas4guw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713381000,"log":"[2026-02-17T12:57:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-30t3l5jnbx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713382000,"log":"[2026-02-17T13:44:45Z INFO actix_web::middleware::logger] GET /api/_search 204 388ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ggwliz2m0z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713383000,"log":"[2026-02-17T13:58:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 216ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bel3iwdd8h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713384000,"log":"[2026-02-17T12:57:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wq3pir0mhi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713385000,"log":"[2026-02-17T13:03:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ktq4l1s5cf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713386000,"log":"[2026-02-17T12:37:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ah3j8vnm7u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713387000,"log":"[2026-02-17T13:20:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1442262735021415700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jnvjf1jbvd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713388000,"log":"[2026-02-17T12:38:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rnhes1m2mx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713389000,"log":"[2026-02-17T13:30:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3599917719167696000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vquaat3dh9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713390000,"log":"[2026-02-17T14:23:12Z DEBUG sqlx::query] SELECT FROM streams returned 513 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w339bqx5kw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713391000,"log":"[2026-02-17T15:15:09Z TRACE cache::redis] cache set key=stream:schema:5625","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7vni2pt12a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713392000,"log":"[2026-02-17T13:08:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-164o637wsc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713393000,"log":"[2026-02-17T13:20:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6910733503601876000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6vr0motycv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713394000,"log":"[2026-02-17T13:00:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0tybm7c3t2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713395000,"log":"[2026-02-17T13:31:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4773683240284801000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zdf76lm7tw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713396000,"log":"[2026-02-17T12:59:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p1s6dbxxss","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713397000,"log":"[2026-02-17T14:29:22Z DEBUG sqlx::query] SELECT FROM streams returned 493 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dhzkpv4k6r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713398000,"log":"[2026-02-17T14:50:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i3ibx2rqvh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713399000,"log":"[2026-02-17T15:13:37Z TRACE cache::redis] cache miss key=user:session:9875","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uio73o5ec8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713400000,"log":"[2026-02-17T14:43:23Z INFO tonic::transport::server] grpc querier disconnected client_id=31xaj189","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nj633y893t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713401000,"log":"[2026-02-17T13:14:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3348633007205001700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1raxv8b9gu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713402000,"log":"[2026-02-17T13:35:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3033342258964120000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pa3uj99xue","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713403000,"log":"[2026-02-17T13:51:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 281ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1mr5qvc26e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713404000,"log":"[2026-02-17T15:07:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3838","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sx0n9gm72i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713405000,"log":"[2026-02-17T15:03:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iqiqvbo0sy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713406000,"log":"[2026-02-17T15:09:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9316","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1a2sum4jro","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713407000,"log":"[2026-02-17T13:53:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 357ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rhrs2wwbnh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713408000,"log":"[2026-02-17T14:36:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=5kpv06s3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-faugb8f4pm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713409000,"log":"[2026-02-17T15:07:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3301","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-st5dk222nn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713410000,"log":"[2026-02-17T15:12:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4763","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pfossc5nia","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713411000,"log":"[2026-02-17T14:11:43Z DEBUG sqlx::query] INSERT FROM users returned 857 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eu51p5uv2i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713412000,"log":"[2026-02-17T13:07:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sft4ni8j4z","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713413000,"log":"[2026-02-17T12:43:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u1wa784dbl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713414000,"log":"[2026-02-17T13:33:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9864468669137338000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uvsed4wqmy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713415000,"log":"[2026-02-17T13:59:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 311ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qv0kte9mqz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713416000,"log":"[2026-02-17T15:16:43Z TRACE cache::redis] cache miss key=org:config:3146","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ackszsot92","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713417000,"log":"[2026-02-17T13:51:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 98ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2825d976rz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713418000,"log":"[2026-02-17T13:48:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 164ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bqr3ivgj44","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713419000,"log":"[2026-02-17T12:48:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nzrm670fuw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713420000,"log":"[2026-02-17T13:55:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 459ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dp2pzsqvdt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713421000,"log":"[2026-02-17T14:36:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pz3kxxr2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ij7b86t6hs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713422000,"log":"[2026-02-17T12:43:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6r4g1ar1zf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713423000,"log":"[2026-02-17T14:34:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=0u5dktgg","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v3m0u4ht4x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713424000,"log":"[2026-02-17T15:00:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7isi2b2gm7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713425000,"log":"[2026-02-17T13:02:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ac62iwq6wd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713426000,"log":"[2026-02-17T12:58:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3oss96bauu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713427000,"log":"[2026-02-17T12:41:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m42kcdoca1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713428000,"log":"[2026-02-17T14:31:57Z INFO tonic::transport::server] grpc ingester connected client_id=ukfxl8mp","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yjicxm9q75","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713429000,"log":"[2026-02-17T13:47:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 500ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6j1iikz0i6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713430000,"log":"[2026-02-17T13:22:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8887926774817603000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2xsbv9ya85","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713431000,"log":"[2026-02-17T13:52:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 316ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t4xhfwf9vq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713432000,"log":"[2026-02-17T14:16:43Z DEBUG sqlx::query] INSERT FROM users returned 863 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tzdrhrt534","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713433000,"log":"[2026-02-17T14:30:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 975 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hww6xr9u7b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713434000,"log":"[2026-02-17T14:07:10Z DEBUG sqlx::query] DELETE FROM alerts returned 722 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g5bll7dwkj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713435000,"log":"[2026-02-17T13:08:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ovgkw0p9kf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713436000,"log":"[2026-02-17T14:16:20Z DEBUG sqlx::query] DELETE FROM alerts returned 831 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3d7mm0a44y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713437000,"log":"[2026-02-17T14:09:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 854 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hogyx2o00l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713438000,"log":"[2026-02-17T13:44:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 35ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r7jk4fcfqr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713439000,"log":"[2026-02-17T14:19:33Z DEBUG sqlx::query] INSERT FROM users returned 57 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xheq163dif","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713440000,"log":"[2026-02-17T13:13:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5756241209693853000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kmzgg8bqwt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713441000,"log":"[2026-02-17T13:23:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7648026105671606000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b6jbjfk7kc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713442000,"log":"[2026-02-17T14:33:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2olkju5n","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s4y5o6pl20","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713443000,"log":"[2026-02-17T15:20:00Z TRACE cache::redis] cache hit key=query:result:4560","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jbv7s045r8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713444000,"log":"[2026-02-17T14:35:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=ok7xvlyp","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xodtohp8qc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713445000,"log":"[2026-02-17T14:41:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=9pdval2o","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-166g5ianep","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713446000,"log":"[2026-02-17T12:40:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3ob3dk2q1f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713447000,"log":"[2026-02-17T15:17:47Z TRACE cache::redis] cache expire key=user:session:4728","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m85zomne0b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713448000,"log":"[2026-02-17T12:53:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qtvqcucgxi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713449000,"log":"[2026-02-17T14:08:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 397 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oxhoe4hmty","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713450000,"log":"[2026-02-17T13:47:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 269ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-992orrvvm6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713451000,"log":"[2026-02-17T13:40:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2591114287268145700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-625ivv886u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713452000,"log":"[2026-02-17T14:03:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 391ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ndxu37s9gy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713453000,"log":"[2026-02-17T14:02:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 135ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u7no0kad1q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713454000,"log":"[2026-02-17T14:33:14Z INFO tonic::transport::server] grpc router request_received client_id=7we7s76v","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-20b04d6ksk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713455000,"log":"[2026-02-17T13:01:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zadtlkwmxi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713456000,"log":"[2026-02-17T13:46:40Z INFO actix_web::middleware::logger] POST /api/_search 200 80ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8pkurcobgj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713457000,"log":"[2026-02-17T13:15:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3303264335190459.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gmfcoi60mc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713458000,"log":"[2026-02-17T13:35:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1646569546218248400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e0qctfbk3s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713459000,"log":"[2026-02-17T13:48:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 95ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bc6zpumr0o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713460000,"log":"[2026-02-17T15:13:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=548","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-okort7zvw3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713461000,"log":"[2026-02-17T14:08:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 11 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xfezsabawo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713462000,"log":"[2026-02-17T13:31:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3564420902758383000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nnnhtv6b4v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713463000,"log":"[2026-02-17T12:44:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dm7d52fliw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713464000,"log":"[2026-02-17T14:26:47Z DEBUG sqlx::query] SELECT FROM streams returned 527 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sum76q8fj3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713465000,"log":"[2026-02-17T15:21:03Z TRACE cache::redis] cache set key=org:config:4812","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wpi2txquhn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713466000,"log":"[2026-02-17T15:19:18Z TRACE cache::redis] cache hit key=org:config:3332","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j1g1vttuis","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713467000,"log":"[2026-02-17T13:00:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o9mb9dg6c5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713468000,"log":"[2026-02-17T12:56:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b77vt66rno","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713469000,"log":"[2026-02-17T14:19:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 160 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k9y0ytbp23","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713470000,"log":"[2026-02-17T13:00:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e19g06c3vd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713471000,"log":"[2026-02-17T13:03:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y68rkc00cq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713472000,"log":"[2026-02-17T14:55:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4lnaysmm0j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713473000,"log":"[2026-02-17T12:51:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vkbzwnc8q4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713474000,"log":"[2026-02-17T13:13:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5861529416443039000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-erct3lrtkk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713475000,"log":"[2026-02-17T14:28:03Z DEBUG sqlx::query] INSERT FROM users returned 896 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1nd1gv6ku4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713476000,"log":"[2026-02-17T13:25:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1766940178491667200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z1cahj70rf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713477000,"log":"[2026-02-17T15:06:30Z DEBUG metrics::reporter] recorded metric active_connections value=1837","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fac8wk41b5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713478000,"log":"[2026-02-17T13:06:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lsf435o4fe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713479000,"log":"[2026-02-17T12:36:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uan45z3js1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713480000,"log":"[2026-02-17T14:35:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=in9byatu","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h705fh0852","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713481000,"log":"[2026-02-17T13:48:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 330ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l6mzsd4g9h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713482000,"log":"[2026-02-17T13:49:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 442ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3jsvnirbfh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713483000,"log":"[2026-02-17T15:04:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2lhla5ez7h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713484000,"log":"[2026-02-17T14:48:24Z INFO tonic::transport::server] grpc router request_received client_id=j4q9l34l","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q4je92p68b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713485000,"log":"[2026-02-17T14:29:33Z DEBUG sqlx::query] INSERT FROM users returned 324 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k3qlw0zlcf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713486000,"log":"[2026-02-17T14:38:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=poezjou6","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eh42hpompp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713487000,"log":"[2026-02-17T15:03:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ymrqtvofge","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713488000,"log":"[2026-02-17T15:12:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=6827","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8ybcvs7mjg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713489000,"log":"[2026-02-17T15:04:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tkry9y6muq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713490000,"log":"[2026-02-17T12:35:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nv1iwjv3cn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713491000,"log":"[2026-02-17T14:26:02Z DEBUG sqlx::query] SELECT FROM streams returned 977 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n784epk2t3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713492000,"log":"[2026-02-17T13:32:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7185029440652435000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-prae66l716","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713493000,"log":"[2026-02-17T13:15:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/610018156955842700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4qj445w4tt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713494000,"log":"[2026-02-17T13:33:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7094916400314222000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-626uscakev","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713495000,"log":"[2026-02-17T13:19:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6064540816198529000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pivgxf4nx6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713496000,"log":"[2026-02-17T14:24:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 267 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cjn3ey0wwn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713497000,"log":"[2026-02-17T13:49:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 230ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xwl0defd8g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713498000,"log":"[2026-02-17T13:19:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6838811560909719000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qbov9l6p7b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713499000,"log":"[2026-02-17T14:32:53Z INFO tonic::transport::server] grpc querier disconnected client_id=w06k6sn1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cgo71dhqg5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713500000,"log":"[2026-02-17T15:20:39Z TRACE cache::redis] cache set key=stream:schema:5759","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e8vuhs1day","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713501000,"log":"[2026-02-17T13:51:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 85ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vwqatkufgt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713502000,"log":"[2026-02-17T14:28:35Z DEBUG sqlx::query] DELETE FROM alerts returned 901 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pl246p2ga3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713503000,"log":"[2026-02-17T15:19:27Z TRACE cache::redis] cache set key=user:session:4701","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uhxes2gtc6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713504000,"log":"[2026-02-17T13:18:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2895877503349102600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k6v06kd1w8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713505000,"log":"[2026-02-17T13:27:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3226660591058683400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rkqb9r293i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713506000,"log":"[2026-02-17T12:35:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mjn6jb04gq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713507000,"log":"[2026-02-17T14:26:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 13 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a7as3zhvdf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713508000,"log":"[2026-02-17T14:45:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=mi20gcxc","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7h67um7ny0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713509000,"log":"[2026-02-17T13:02:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eyvbhsaodd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713510000,"log":"[2026-02-17T14:05:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 337ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0a0itnkcyc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713511000,"log":"[2026-02-17T13:01:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kv6os2jrhq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713512000,"log":"[2026-02-17T12:58:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-glafvmdasn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713513000,"log":"[2026-02-17T15:12:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=1037","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pvxmu4g57c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713514000,"log":"[2026-02-17T14:34:43Z INFO tonic::transport::server] grpc querier disconnected client_id=j65w50zj","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l0tjz16rii","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713515000,"log":"[2026-02-17T14:15:55Z DEBUG sqlx::query] DELETE FROM alerts returned 424 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dfkxunfv2l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713516000,"log":"[2026-02-17T13:22:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9092729571825062000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4cst8gptzv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713517000,"log":"[2026-02-17T14:47:37Z INFO tonic::transport::server] grpc ingester connected client_id=6eq051gh","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5cdqpbo8hb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713518000,"log":"[2026-02-17T14:58:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vslzojzf3a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713519000,"log":"[2026-02-17T13:48:40Z INFO actix_web::middleware::logger] POST /api/_search 400 181ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-39lptp9mf6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713520000,"log":"[2026-02-17T12:49:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o587we1ckg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713521000,"log":"[2026-02-17T13:43:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 91ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-px38uhwstp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713522000,"log":"[2026-02-17T14:24:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 840 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hgpu6ngwyz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713523000,"log":"[2026-02-17T14:08:08Z DEBUG sqlx::query] INSERT FROM users returned 486 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s5eigx0x4g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713524000,"log":"[2026-02-17T14:41:13Z INFO tonic::transport::server] grpc querier disconnected client_id=wxjm3m2g","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-20r8r3ppyy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713525000,"log":"[2026-02-17T12:37:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q46iedctzc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713526000,"log":"[2026-02-17T14:26:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 756 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dndpbczwmk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713527000,"log":"[2026-02-17T13:13:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/685848153937058700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ewoqtcn5fx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713528000,"log":"[2026-02-17T14:20:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 365 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y98ggxpkha","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713529000,"log":"[2026-02-17T13:46:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 257ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-urueqco7tx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713530000,"log":"[2026-02-17T13:32:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9823490324907514000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bwgq83ey0z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713531000,"log":"[2026-02-17T14:52:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6d9ebg4hje","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713532000,"log":"[2026-02-17T14:33:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=vwne2ig8","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-of7qvv94mf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713533000,"log":"[2026-02-17T13:16:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/831153818120651300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cthpi8yau5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713534000,"log":"[2026-02-17T13:58:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 315ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a9g68fetyq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713535000,"log":"[2026-02-17T13:23:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/633781923973340000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xcycbd8b7b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713536000,"log":"[2026-02-17T12:37:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m7su1erzvx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713537000,"log":"[2026-02-17T15:02:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9vektpoxmz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713538000,"log":"[2026-02-17T13:35:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1819149855236019700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-33zj4ndwlm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713539000,"log":"[2026-02-17T15:20:22Z TRACE cache::redis] cache delete key=user:session:6773","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hdc63tgd0u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713540000,"log":"[2026-02-17T15:04:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bez1g5f06b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713541000,"log":"[2026-02-17T13:13:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2888970809355186700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o5kxf17d3j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713542000,"log":"[2026-02-17T14:56:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4afnpabxhl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713543000,"log":"[2026-02-17T15:10:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=212","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2gm10yu253","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713544000,"log":"[2026-02-17T15:02:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h8gfszts2l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713545000,"log":"[2026-02-17T13:14:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6298550394701063.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kn2tulj477","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713546000,"log":"[2026-02-17T12:49:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gy9rmr9djt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713547000,"log":"[2026-02-17T14:55:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vrcfu91opd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713548000,"log":"[2026-02-17T14:10:47Z DEBUG sqlx::query] SELECT FROM streams returned 727 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-32bek9hckt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713549000,"log":"[2026-02-17T14:16:40Z DEBUG sqlx::query] DELETE FROM alerts returned 430 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dfbqgvfxpm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713550000,"log":"[2026-02-17T12:41:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wppp9hxxu2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713551000,"log":"[2026-02-17T14:18:50Z DEBUG sqlx::query] DELETE FROM alerts returned 277 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-le821ypi98","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713552000,"log":"[2026-02-17T13:14:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3362586744939726300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yh9heqqjz1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713553000,"log":"[2026-02-17T12:37:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wsnnxqr54p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713554000,"log":"[2026-02-17T14:02:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 83ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t39r0cfz27","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713555000,"log":"[2026-02-17T13:58:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 461ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j6kocbzfjh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713556000,"log":"[2026-02-17T13:23:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6892941206196241000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8y0o08fg13","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713557000,"log":"[2026-02-17T14:32:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=oude72c3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yjb2m3w4gf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713558000,"log":"[2026-02-17T13:36:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3607571612519260000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hh0gb0l21h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713559000,"log":"[2026-02-17T13:34:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3075923353847471600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e2kley1zxw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713560000,"log":"[2026-02-17T12:43:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e7adhtg5e9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713561000,"log":"[2026-02-17T13:00:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r6zdxr6wkb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713562000,"log":"[2026-02-17T14:43:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=cc9a59rl","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-evn5iukj89","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713563000,"log":"[2026-02-17T14:53:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8gu052r3gl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713564000,"log":"[2026-02-17T15:00:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dko3c8y8hr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713565000,"log":"[2026-02-17T14:36:09Z INFO tonic::transport::server] grpc router request_received client_id=d7koiq7r","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-926hzga53o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713566000,"log":"[2026-02-17T12:55:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zg234nr2ox","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713567000,"log":"[2026-02-17T14:13:13Z DEBUG sqlx::query] INSERT FROM users returned 773 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-aogxkhrt3n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713568000,"log":"[2026-02-17T13:45:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 491ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-edcbz57sz4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713569000,"log":"[2026-02-17T13:36:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4510389449643306000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jjqzcftkvo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713570000,"log":"[2026-02-17T13:38:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6035719190957671000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1cdk96pgcq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713571000,"log":"[2026-02-17T12:38:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x3t3jlpxr9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713572000,"log":"[2026-02-17T14:52:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-venucdwjff","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713573000,"log":"[2026-02-17T12:42:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ych4y3smlt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713574000,"log":"[2026-02-17T13:16:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1907549087388027400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zl66xw0u2e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713575000,"log":"[2026-02-17T15:17:11Z TRACE cache::redis] cache expire key=auth:token:3597","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x8hal7vjq2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713576000,"log":"[2026-02-17T14:24:55Z DEBUG sqlx::query] DELETE FROM alerts returned 743 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-couowj1v0m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713577000,"log":"[2026-02-17T13:47:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 302ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ahre2ik6mw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713578000,"log":"[2026-02-17T13:06:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s42mul3il1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713579000,"log":"[2026-02-17T14:37:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=25m5vm57","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nh8sxw74et","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713580000,"log":"[2026-02-17T15:10:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6414","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ole8i8e58e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713581000,"log":"[2026-02-17T14:18:55Z DEBUG sqlx::query] DELETE FROM alerts returned 972 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8z78h9k409","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713582000,"log":"[2026-02-17T15:11:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1911","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ulam7a1wpe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713583000,"log":"[2026-02-17T14:55:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4gd94bi0sc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713584000,"log":"[2026-02-17T14:09:23Z DEBUG sqlx::query] INSERT FROM users returned 45 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vvsq9l3pz3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713585000,"log":"[2026-02-17T14:54:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ag9lgmasyv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713586000,"log":"[2026-02-17T15:03:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-il9q1qpv7y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713587000,"log":"[2026-02-17T15:15:30Z TRACE cache::redis] cache hit key=query:result:775","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o3whfwn8e0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713588000,"log":"[2026-02-17T13:21:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6724030339120144000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bjj8f5yzd5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713589000,"log":"[2026-02-17T13:47:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 356ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h6r95yzez0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713590000,"log":"[2026-02-17T13:23:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/316243023270612600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bjgj4d405x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713591000,"log":"[2026-02-17T13:34:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/803178922654188500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9cc763nmkq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713592000,"log":"[2026-02-17T14:05:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 434ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-igykd6own5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713593000,"log":"[2026-02-17T14:43:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=py7gj23e","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0i0vgi3ps0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713594000,"log":"[2026-02-17T13:44:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 224ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-htu96aswda","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713595000,"log":"[2026-02-17T13:03:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pkb086awdx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713596000,"log":"[2026-02-17T14:00:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 475ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gfrqm9du1z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713597000,"log":"[2026-02-17T12:35:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jsuxdl9lm0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713598000,"log":"[2026-02-17T12:54:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lkudnl7xdv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713599000,"log":"[2026-02-17T14:01:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 395ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7maigy9qbm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713600000,"log":"[2026-02-17T15:08:10Z DEBUG metrics::reporter] recorded metric active_connections value=784","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ogn8p0eh88","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713601000,"log":"[2026-02-17T13:47:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 55ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aht4ye3kn2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713602000,"log":"[2026-02-17T13:17:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1095847041477069300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1i9dw3747f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713603000,"log":"[2026-02-17T12:37:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lh43vfb4g4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713604000,"log":"[2026-02-17T14:26:25Z DEBUG sqlx::query] DELETE FROM alerts returned 34 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8me788rtxu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713605000,"log":"[2026-02-17T13:40:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8629980422342978000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bqnspvrr6a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713606000,"log":"[2026-02-17T12:59:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-475ws3i3de","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713607000,"log":"[2026-02-17T14:02:25Z INFO actix_web::middleware::logger] GET /api/_search 200 101ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xdkggm4lbi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713608000,"log":"[2026-02-17T13:13:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6454790675557057000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-902ufdyfdn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713609000,"log":"[2026-02-17T13:09:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5879633206056800000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wngyo5yoii","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713610000,"log":"[2026-02-17T15:20:38Z TRACE cache::redis] cache get key=org:config:856","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1rsrivwm9p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713611000,"log":"[2026-02-17T13:20:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9050061143546468000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sbmx25pkkl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713612000,"log":"[2026-02-17T13:30:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4875995534051532000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9qtpbq7pew","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713613000,"log":"[2026-02-17T13:58:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 108ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8xc3js1lvk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713614000,"log":"[2026-02-17T14:52:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6iund0ir7a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713615000,"log":"[2026-02-17T14:40:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ot4i66mc","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-05wktyajnx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713616000,"log":"[2026-02-17T13:05:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v2xiwg9d97","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713617000,"log":"[2026-02-17T13:50:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 379ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tjdoigdrnz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713618000,"log":"[2026-02-17T14:57:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-493yuspjb2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713619000,"log":"[2026-02-17T13:51:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 156ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fo6opi93h3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713620000,"log":"[2026-02-17T13:29:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3899611812385397000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-52yhj8bveo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713621000,"log":"[2026-02-17T15:03:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3cdxueiph8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713622000,"log":"[2026-02-17T15:02:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ifv2dfy5bu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713623000,"log":"[2026-02-17T14:22:27Z DEBUG sqlx::query] SELECT FROM streams returned 494 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lne4gnvur9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713624000,"log":"[2026-02-17T13:36:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6520983333434278000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sc8zcya7hf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713625000,"log":"[2026-02-17T15:16:39Z TRACE cache::redis] cache set key=stream:schema:3258","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xa4yp25sxs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713626000,"log":"[2026-02-17T13:18:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/126946123554034780.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jbqqqcdzs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713627000,"log":"[2026-02-17T12:44:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o1sle510vj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713628000,"log":"[2026-02-17T15:11:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8079","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ny69udso67","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713629000,"log":"[2026-02-17T12:37:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-127ejxapom","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713630000,"log":"[2026-02-17T13:03:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pftj0krffa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713631000,"log":"[2026-02-17T15:19:16Z TRACE cache::redis] cache delete key=auth:token:8982","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j9wvgwyjvt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713632000,"log":"[2026-02-17T14:12:18Z DEBUG sqlx::query] INSERT FROM users returned 420 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g1qa0kw5zx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713633000,"log":"[2026-02-17T13:50:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 126ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mq9m3mkf4m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713634000,"log":"[2026-02-17T14:43:27Z INFO tonic::transport::server] grpc ingester connected client_id=48juedpy","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ojrolzdmrn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713635000,"log":"[2026-02-17T14:17:32Z DEBUG sqlx::query] SELECT FROM streams returned 2 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y2urqlv7tg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713636000,"log":"[2026-02-17T15:05:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1360","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x976zf5tq4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713637000,"log":"[2026-02-17T15:08:05Z DEBUG metrics::reporter] recorded metric active_connections value=5936","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qfld3tug1h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713638000,"log":"[2026-02-17T12:53:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6j5nhoc6xp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713639000,"log":"[2026-02-17T12:51:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hnkjoqf03h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713640000,"log":"[2026-02-17T14:21:13Z DEBUG sqlx::query] INSERT FROM users returned 379 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tb0qvijnz1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713641000,"log":"[2026-02-17T14:14:28Z DEBUG sqlx::query] INSERT FROM users returned 817 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o6qapk463o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713642000,"log":"[2026-02-17T12:37:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qhoar6to5g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713643000,"log":"[2026-02-17T14:40:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=3u74jyrl","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7oqju1g90p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713644000,"log":"[2026-02-17T14:01:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 495ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d4ad7qjwug","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713645000,"log":"[2026-02-17T13:38:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7762955684514297000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pkt45i8p0t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713646000,"log":"[2026-02-17T13:25:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5568585017181417000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8e9xlseu82","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713647000,"log":"[2026-02-17T13:56:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 423ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-urv48xgyq4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713648000,"log":"[2026-02-17T12:41:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2l62q25x56","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713649000,"log":"[2026-02-17T13:53:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 453ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yanyiy835g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713650000,"log":"[2026-02-17T12:38:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-554r5wofed","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713651000,"log":"[2026-02-17T14:15:37Z DEBUG sqlx::query] SELECT FROM streams returned 992 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hpg2gusr6v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713652000,"log":"[2026-02-17T14:05:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 267ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-flo77hwvyu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713653000,"log":"[2026-02-17T12:43:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vu5ldb5y9d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713654000,"log":"[2026-02-17T13:03:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zordiaxkx1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713655000,"log":"[2026-02-17T14:07:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 300 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kl6depgugn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713656000,"log":"[2026-02-17T14:10:50Z DEBUG sqlx::query] DELETE FROM alerts returned 349 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3gv605h2ru","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713657000,"log":"[2026-02-17T14:50:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-upievapw6e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713658000,"log":"[2026-02-17T14:44:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rfy57dy7","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e1q2jzikua","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713659000,"log":"[2026-02-17T13:29:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2343480099364285400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k2dnynmyow","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713660000,"log":"[2026-02-17T13:22:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7365303291999426000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wnstirxjwn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713661000,"log":"[2026-02-17T14:47:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=r700ijnw","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rhyi53uqvy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713662000,"log":"[2026-02-17T13:11:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8091051096806936000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6s0u2pg0zy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713663000,"log":"[2026-02-17T15:09:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=422","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-33kbuvacov","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713664000,"log":"[2026-02-17T13:44:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 100ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vj128h5yes","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713665000,"log":"[2026-02-17T12:39:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uosqthqfeo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713666000,"log":"[2026-02-17T12:55:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oqv3fyq3zz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713667000,"log":"[2026-02-17T14:18:23Z DEBUG sqlx::query] INSERT FROM users returned 426 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-30eguubxew","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713668000,"log":"[2026-02-17T15:18:27Z TRACE cache::redis] cache set key=user:session:603","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sap32bpm0i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713669000,"log":"[2026-02-17T15:11:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5911","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m8kenmwvuu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713670000,"log":"[2026-02-17T14:17:45Z DEBUG sqlx::query] DELETE FROM alerts returned 174 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6kifn2m48y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713671000,"log":"[2026-02-17T12:47:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ytcwckvyfx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713672000,"log":"[2026-02-17T14:38:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=5plmn439","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iocw5i2rov","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713673000,"log":"[2026-02-17T14:10:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 447 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-czxtvyvxdt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713674000,"log":"[2026-02-17T13:16:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7903794433433196000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-in62kbdshr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713675000,"log":"[2026-02-17T14:28:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 685 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rmd4twb6mv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713676000,"log":"[2026-02-17T14:01:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 176ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6z2hroa3rk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713677000,"log":"[2026-02-17T12:37:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ub3sdtxmp5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713678000,"log":"[2026-02-17T15:03:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2tn2pm2vsa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713679000,"log":"[2026-02-17T12:44:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gzim72d1db","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713680000,"log":"[2026-02-17T14:22:55Z DEBUG sqlx::query] DELETE FROM alerts returned 839 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j2pw3ac2fw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713681000,"log":"[2026-02-17T14:02:20Z INFO actix_web::middleware::logger] POST /api/_search 401 126ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ruh3rvvwsw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713682000,"log":"[2026-02-17T14:52:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f6ozhtlfaa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713683000,"log":"[2026-02-17T14:39:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=y48ckte7","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u63sypvh7a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713684000,"log":"[2026-02-17T13:46:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 428ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wtbq4u9y52","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713685000,"log":"[2026-02-17T14:20:53Z DEBUG sqlx::query] INSERT FROM users returned 89 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7lg4fiip4x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713686000,"log":"[2026-02-17T15:15:10Z TRACE cache::redis] cache delete key=query:result:2446","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kgyiy9yp8h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713687000,"log":"[2026-02-17T13:04:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ta34avvksu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713688000,"log":"[2026-02-17T14:42:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=v30tx03j","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-866p1yy34i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713689000,"log":"[2026-02-17T13:40:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8735973562824335000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eakxcl9j24","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713690000,"log":"[2026-02-17T15:07:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=4475","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lbda3n71oc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713691000,"log":"[2026-02-17T14:24:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 439 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tuzxwjjdpa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713692000,"log":"[2026-02-17T13:22:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7241993750642703000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gwcj5tkhw0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713693000,"log":"[2026-02-17T12:52:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gdvlukxwd1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713694000,"log":"[2026-02-17T13:29:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6663571045129515000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6owhn2dsz0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713695000,"log":"[2026-02-17T14:39:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=l0lr811w","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-id0f5oicuy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713696000,"log":"[2026-02-17T15:18:23Z TRACE cache::redis] cache expire key=org:config:3127","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zcb0i2zkpp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713697000,"log":"[2026-02-17T14:08:17Z DEBUG sqlx::query] SELECT FROM streams returned 282 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4z014wf10g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713698000,"log":"[2026-02-17T15:12:30Z DEBUG metrics::reporter] recorded metric active_connections value=9833","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n0tcjo3uix","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713699000,"log":"[2026-02-17T13:57:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 84ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6m54f3zktm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713700000,"log":"[2026-02-17T12:39:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1r41xespbg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713701000,"log":"[2026-02-17T13:18:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5994405660121038000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b3c9bfz6vx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713702000,"log":"[2026-02-17T14:50:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cbv9pwye5a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713703000,"log":"[2026-02-17T13:11:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8548298749448058000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eozi50drc5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713704000,"log":"[2026-02-17T14:15:28Z DEBUG sqlx::query] INSERT FROM users returned 375 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u3nuu7kio3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713705000,"log":"[2026-02-17T12:38:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xya3oqznqz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713706000,"log":"[2026-02-17T12:54:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-74dmx0lb3d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713707000,"log":"[2026-02-17T14:04:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 156ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5tabic6bwf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713708000,"log":"[2026-02-17T14:41:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=y9bex3nu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cgss333w4x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713709000,"log":"[2026-02-17T14:33:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=aklxh5ya","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mwm63kb1yq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713710000,"log":"[2026-02-17T14:48:29Z INFO tonic::transport::server] grpc router request_received client_id=0vvqt63s","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7wrukeqs8l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713711000,"log":"[2026-02-17T12:44:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-19jp9i916f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713712000,"log":"[2026-02-17T14:36:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=zntb95sy","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2jdddlgb51","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713713000,"log":"[2026-02-17T13:11:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8740673947334472000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lipjfdq6yt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713714000,"log":"[2026-02-17T13:50:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 274ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0h6qr3fk7c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713715000,"log":"[2026-02-17T14:40:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=bld5hk1b","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g6wy75jekd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713716000,"log":"[2026-02-17T14:36:59Z INFO tonic::transport::server] grpc router request_received client_id=esvsowit","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5wd38e9bm9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713717000,"log":"[2026-02-17T14:38:27Z INFO tonic::transport::server] grpc ingester connected client_id=qf64phep","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-93zlurv8p6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713718000,"log":"[2026-02-17T12:56:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pxy5n1cdil","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713719000,"log":"[2026-02-17T14:54:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4gem1l68by","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713720000,"log":"[2026-02-17T13:43:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 127ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wxq60ws6sb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713721000,"log":"[2026-02-17T13:24:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3406284235435051000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-okeq2wxzs8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713722000,"log":"[2026-02-17T12:42:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ack1u0fvzd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713723000,"log":"[2026-02-17T12:44:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-67z3rq3b7s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713724000,"log":"[2026-02-17T13:03:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r0e4x2emfb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713725000,"log":"[2026-02-17T15:11:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=4334","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ujgtnthitc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713726000,"log":"[2026-02-17T15:15:00Z TRACE cache::redis] cache hit key=query:result:1116","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0nu7kvthed","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713727000,"log":"[2026-02-17T14:57:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sv0omewrea","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713728000,"log":"[2026-02-17T15:08:25Z DEBUG metrics::reporter] recorded metric active_connections value=2548","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5xh84v567z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713729000,"log":"[2026-02-17T14:47:29Z INFO tonic::transport::server] grpc router request_received client_id=fjr9f836","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5kwt1gmm1k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713730000,"log":"[2026-02-17T13:58:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 170ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y6vo11set1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713731000,"log":"[2026-02-17T14:43:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pgt6644o","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4zs1tqm7cn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713732000,"log":"[2026-02-17T13:46:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 489ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j6pqv8b6y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713733000,"log":"[2026-02-17T12:46:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dleh8h1nf7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713734000,"log":"[2026-02-17T13:47:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 33ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fq51jt33it","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713735000,"log":"[2026-02-17T14:57:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bwumliztf0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713736000,"log":"[2026-02-17T13:39:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4264386971804410400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ondgzjibva","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713737000,"log":"[2026-02-17T13:47:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 150ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sifvgski96","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713738000,"log":"[2026-02-17T15:20:16Z TRACE cache::redis] cache delete key=auth:token:4865","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ohst71o2mq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713739000,"log":"[2026-02-17T13:56:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 299ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-td5rfvte5z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713740000,"log":"[2026-02-17T14:29:20Z DEBUG sqlx::query] DELETE FROM alerts returned 39 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tma3t7q22s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713741000,"log":"[2026-02-17T13:12:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4297188238430154000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tuy4f2k43u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713742000,"log":"[2026-02-17T12:41:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dzvgynwcb2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713743000,"log":"[2026-02-17T15:12:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2557","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gn1pf5c124","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713744000,"log":"[2026-02-17T14:07:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 870 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-roqg9aym36","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713745000,"log":"[2026-02-17T12:37:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vhbj1lbinh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713746000,"log":"[2026-02-17T14:12:03Z DEBUG sqlx::query] INSERT FROM users returned 124 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-szae2tbhsl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713747000,"log":"[2026-02-17T13:23:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9188596061803153000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0w7cxga7gi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713748000,"log":"[2026-02-17T13:09:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4793592094778352000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8j5ryidroq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713749000,"log":"[2026-02-17T13:58:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 143ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2gwnl4gw09","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713750000,"log":"[2026-02-17T14:50:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bkitg6o4ez","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713751000,"log":"[2026-02-17T13:03:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-za1g3bmud8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713752000,"log":"[2026-02-17T12:53:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-we3eqn5tcu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713753000,"log":"[2026-02-17T14:53:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h4vkx9cdu0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713754000,"log":"[2026-02-17T13:18:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3501018903399699500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1wmffww4ll","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713755000,"log":"[2026-02-17T14:54:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-umj8pqxdmf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713756000,"log":"[2026-02-17T13:19:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7266398886257146000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ckov6bdai","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713757000,"log":"[2026-02-17T14:36:33Z INFO tonic::transport::server] grpc querier disconnected client_id=2dmpzadc","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h8cidii8w9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713758000,"log":"[2026-02-17T13:04:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-13qlm65q8k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713759000,"log":"[2026-02-17T15:10:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6754","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7utrmwquhy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713760000,"log":"[2026-02-17T12:59:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-825m9vy7wz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713761000,"log":"[2026-02-17T15:16:50Z TRACE cache::redis] cache get key=query:result:6486","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4o7zfz9y3d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713762000,"log":"[2026-02-17T13:54:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 137ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kvmd0ryygd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713763000,"log":"[2026-02-17T14:16:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 939 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tu95tuc0im","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713764000,"log":"[2026-02-17T13:28:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4955270880320240000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wh8afdlshq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713765000,"log":"[2026-02-17T14:02:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 7ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nkjdl9bkdo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713766000,"log":"[2026-02-17T14:46:09Z INFO tonic::transport::server] grpc router request_received client_id=ar22zsxe","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e4xqvbdjnp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713767000,"log":"[2026-02-17T12:49:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y13ckfktao","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713768000,"log":"[2026-02-17T14:29:47Z DEBUG sqlx::query] SELECT FROM streams returned 335 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lbgdd28lf2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713769000,"log":"[2026-02-17T13:50:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 23ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-53try2pb2n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713770000,"log":"[2026-02-17T15:18:04Z TRACE cache::redis] cache delete key=stream:schema:9966","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i35xu7jvli","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713771000,"log":"[2026-02-17T12:39:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5f4brd3wvi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713772000,"log":"[2026-02-17T15:05:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6937","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5awo12yzfa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713773000,"log":"[2026-02-17T13:18:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7497562586202595000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-knkqr7dp0b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713774000,"log":"[2026-02-17T14:16:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 602 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4pdnilxfg6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713775000,"log":"[2026-02-17T14:22:05Z DEBUG sqlx::query] DELETE FROM alerts returned 400 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-baszfv27zs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713776000,"log":"[2026-02-17T15:01:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m4svysg1oj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713777000,"log":"[2026-02-17T15:10:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5177","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l5vr9bdyuq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713778000,"log":"[2026-02-17T13:33:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6956994960387903000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hhlivj6i7k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713779000,"log":"[2026-02-17T14:42:22Z INFO tonic::transport::server] grpc ingester connected client_id=j7n1qhvl","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5n73pfyskf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713780000,"log":"[2026-02-17T15:12:10Z DEBUG metrics::reporter] recorded metric active_connections value=9933","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qzfpeuqhyi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713781000,"log":"[2026-02-17T13:21:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9037301196729293000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8dtjy5w7bf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713782000,"log":"[2026-02-17T12:58:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-otpjp4v1ap","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713783000,"log":"[2026-02-17T15:00:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ls9ty5vz9g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713784000,"log":"[2026-02-17T12:40:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j6pmukxvxu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713785000,"log":"[2026-02-17T13:24:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8752148985124446000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v4hk4jm4dt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713786000,"log":"[2026-02-17T13:31:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3293425321821837000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-utro4nhss2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713787000,"log":"[2026-02-17T12:48:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t375lchu0z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713788000,"log":"[2026-02-17T14:09:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 170 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7dxxsxupn9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713789000,"log":"[2026-02-17T15:21:01Z TRACE cache::redis] cache miss key=auth:token:6721","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zjr28esfjt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713790000,"log":"[2026-02-17T13:45:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 366ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5q9g7tlt63","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713791000,"log":"[2026-02-17T14:00:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 484ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-svzfnde59p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713792000,"log":"[2026-02-17T12:35:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rn3pbr4fen","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713793000,"log":"[2026-02-17T14:25:30Z DEBUG sqlx::query] DELETE FROM alerts returned 740 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w8imwf9dio","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713794000,"log":"[2026-02-17T14:25:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 370 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bts7wfunn1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713795000,"log":"[2026-02-17T13:47:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6e1f11drsi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713796000,"log":"[2026-02-17T14:15:13Z DEBUG sqlx::query] INSERT FROM users returned 873 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6fszbxgkqh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713797000,"log":"[2026-02-17T13:05:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wkb1yo6764","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713798000,"log":"[2026-02-17T13:33:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2741851304388360700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eq7b7gh0so","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713799000,"log":"[2026-02-17T15:16:36Z TRACE cache::redis] cache hit key=auth:token:7990","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fb3k4icuvp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713800000,"log":"[2026-02-17T12:56:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vmwbyruh0d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713801000,"log":"[2026-02-17T14:00:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 148ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2w9rc843gh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713802000,"log":"[2026-02-17T14:07:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 903 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0f8icxgkcc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713803000,"log":"[2026-02-17T14:55:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-od51qp31nr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713804000,"log":"[2026-02-17T14:21:25Z DEBUG sqlx::query] DELETE FROM alerts returned 407 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-utld7nxdm4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713805000,"log":"[2026-02-17T14:55:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qhtsg21an7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713806000,"log":"[2026-02-17T13:20:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6337473805471829000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eq6o2yh1c3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713807000,"log":"[2026-02-17T12:58:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6s7jvxgpzp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713808000,"log":"[2026-02-17T13:27:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6048001340578089000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lh65iescqt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713809000,"log":"[2026-02-17T13:13:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4937284438255605000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-urvpldk4ka","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713810000,"log":"[2026-02-17T14:08:10Z DEBUG sqlx::query] DELETE FROM alerts returned 573 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-58bxc3t78z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713811000,"log":"[2026-02-17T15:18:31Z TRACE cache::redis] cache miss key=auth:token:9854","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h3t0h84wfh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713812000,"log":"[2026-02-17T12:57:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z2kogxw6oa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713813000,"log":"[2026-02-17T13:32:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2359214537275418000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lamwc29mok","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713814000,"log":"[2026-02-17T15:20:06Z TRACE cache::redis] cache hit key=auth:token:1335","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xkvkook6ys","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713815000,"log":"[2026-02-17T12:40:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7mtpnzez75","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713816000,"log":"[2026-02-17T14:28:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 792 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ko0t4enhwm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713817000,"log":"[2026-02-17T15:03:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-89gs4wluci","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713818000,"log":"[2026-02-17T13:30:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3120442384335064600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o3eqr9ouxi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713819000,"log":"[2026-02-17T12:37:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8qaqzanzj6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713820000,"log":"[2026-02-17T13:31:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5016068770562143000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6vbd68a4wr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713821000,"log":"[2026-02-17T13:45:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 220ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-br8ch9wxlb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713822000,"log":"[2026-02-17T14:19:38Z DEBUG sqlx::query] INSERT FROM users returned 398 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ceu0geuxve","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713823000,"log":"[2026-02-17T15:16:41Z TRACE cache::redis] cache expire key=auth:token:4472","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fjybe42sa5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713824000,"log":"[2026-02-17T15:04:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vpm5eisz4y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713825000,"log":"[2026-02-17T12:45:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g43i1nrr3h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713826000,"log":"[2026-02-17T14:05:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 13ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4puhz61t0x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713827000,"log":"[2026-02-17T12:57:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rax8mpeis2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713828000,"log":"[2026-02-17T12:45:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wfq231x8y2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713829000,"log":"[2026-02-17T14:32:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ffzm6llj","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-note0vqhxq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713830000,"log":"[2026-02-17T12:59:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wojb1slh18","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713831000,"log":"[2026-02-17T12:55:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-46udds47us","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713832000,"log":"[2026-02-17T14:45:43Z INFO tonic::transport::server] grpc querier disconnected client_id=dxmqp0rw","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lgr37bpv2x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713833000,"log":"[2026-02-17T13:06:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l1uvk2mmp5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713834000,"log":"[2026-02-17T14:09:13Z DEBUG sqlx::query] INSERT FROM users returned 920 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k42h2x3s3h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713835000,"log":"[2026-02-17T15:07:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7980","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n54su8dp7t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713836000,"log":"[2026-02-17T13:24:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1015422785634008000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y4hmszksez","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713837000,"log":"[2026-02-17T15:06:45Z DEBUG metrics::reporter] recorded metric active_connections value=7535","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fo8qv9hmap","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713838000,"log":"[2026-02-17T15:18:03Z TRACE cache::redis] cache set key=org:config:739","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fqaqm3pxf2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713839000,"log":"[2026-02-17T12:38:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-etqxvf5mrq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713840000,"log":"[2026-02-17T13:01:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vjjfyhmhzi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713841000,"log":"[2026-02-17T13:24:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3141348813127773000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k5n2j6nunh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713842000,"log":"[2026-02-17T15:12:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2340","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-acvck0mnzd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713843000,"log":"[2026-02-17T15:08:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=7411","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mwxyze8i96","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713844000,"log":"[2026-02-17T15:07:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=4188","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m93dwh40bg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713845000,"log":"[2026-02-17T12:47:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xpqkj0e8gu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713846000,"log":"[2026-02-17T13:14:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9008633679523959000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mtvqd1jsm3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713847000,"log":"[2026-02-17T15:08:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7348","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rgbzz969t8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713848000,"log":"[2026-02-17T13:04:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rbaq2w4wro","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713849000,"log":"[2026-02-17T13:01:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-52ucd4gbtw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713850000,"log":"[2026-02-17T13:15:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/475881044671502300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qjhxhkrclw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713851000,"log":"[2026-02-17T13:25:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5465290758132904000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cnep65zgfy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713852000,"log":"[2026-02-17T13:51:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 38ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p572s3shxk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713853000,"log":"[2026-02-17T14:30:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 700 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qgyioiu1ha","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713854000,"log":"[2026-02-17T12:45:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-efeqf1octl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713855000,"log":"[2026-02-17T14:31:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 17 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sb8f5okcjs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713856000,"log":"[2026-02-17T15:10:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6566","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x02nf76hqp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713857000,"log":"[2026-02-17T13:35:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2471564038052181000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1i45calmk1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713858000,"log":"[2026-02-17T14:44:24Z INFO tonic::transport::server] grpc router request_received client_id=s46v6e7n","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3p83oxr2mh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713859000,"log":"[2026-02-17T14:27:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 44 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l62u5akd44","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713860000,"log":"[2026-02-17T14:53:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-guy3stwkea","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713861000,"log":"[2026-02-17T13:18:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4256938717792513500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ri6an4hg88","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713862000,"log":"[2026-02-17T13:38:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6974530427382433000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mgh6ced2gw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713863000,"log":"[2026-02-17T14:09:22Z DEBUG sqlx::query] SELECT FROM streams returned 685 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1k469ygdcc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713864000,"log":"[2026-02-17T12:41:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vpy1dve01k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713865000,"log":"[2026-02-17T13:59:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 322ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n3yh05aaog","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713866000,"log":"[2026-02-17T15:02:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6xtoz6vw3s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713867000,"log":"[2026-02-17T14:20:18Z DEBUG sqlx::query] INSERT FROM users returned 939 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8u0nh7umdu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713868000,"log":"[2026-02-17T13:35:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6589295207795638000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-315mdi5kst","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713869000,"log":"[2026-02-17T12:59:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9t0jjhra5l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713870000,"log":"[2026-02-17T15:13:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=260","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4cswpnzz6m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713871000,"log":"[2026-02-17T15:05:40Z DEBUG metrics::reporter] recorded metric active_connections value=662","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0wxbq41ub8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713872000,"log":"[2026-02-17T14:12:43Z DEBUG sqlx::query] INSERT FROM users returned 489 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yn3dsjasij","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713873000,"log":"[2026-02-17T12:36:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bjkvk0h53f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713874000,"log":"[2026-02-17T12:43:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-34ul11eaj6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713875000,"log":"[2026-02-17T14:47:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=fk8ncll0","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mf5m0d829l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713876000,"log":"[2026-02-17T12:38:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kopda7mz32","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713877000,"log":"[2026-02-17T13:53:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 470ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8bj1ajymnb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713878000,"log":"[2026-02-17T12:56:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jhz08ybdwu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713879000,"log":"[2026-02-17T13:00:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9o8x4zmon0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713880000,"log":"[2026-02-17T13:45:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 475ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k7heesvlrc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713881000,"log":"[2026-02-17T14:59:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7320fawpxn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713882000,"log":"[2026-02-17T12:50:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qz7qtvc8g9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713883000,"log":"[2026-02-17T13:03:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xgfhddwfn7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713884000,"log":"[2026-02-17T14:07:15Z DEBUG sqlx::query] DELETE FROM alerts returned 171 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lmpp7lnp19","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713885000,"log":"[2026-02-17T15:04:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sd2td45fxo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713886000,"log":"[2026-02-17T15:00:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n04n7z0bix","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713887000,"log":"[2026-02-17T13:20:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/139171878340496050.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r3tz7bg62r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713888000,"log":"[2026-02-17T15:09:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=2292","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bgeh6tgp4u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713889000,"log":"[2026-02-17T14:01:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 250ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-klg3278mdk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713890000,"log":"[2026-02-17T12:53:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-71j34vkkv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713891000,"log":"[2026-02-17T13:14:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/835362917547877100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-abqudh2n9i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713892000,"log":"[2026-02-17T13:28:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4259384388412011000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h9hr9dq1ry","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713893000,"log":"[2026-02-17T15:03:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mbct4h7zdf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713894000,"log":"[2026-02-17T14:50:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d81m87j4fq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713895000,"log":"[2026-02-17T12:46:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9avottg4eq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713896000,"log":"[2026-02-17T15:21:35Z TRACE cache::redis] cache expire key=query:result:6736","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uxvi30wd95","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713897000,"log":"[2026-02-17T12:54:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mvzlewt1gj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713898000,"log":"[2026-02-17T12:45:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j76z0gvvfc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713899000,"log":"[2026-02-17T13:24:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1885059900033798700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w7e9ychwz6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713900000,"log":"[2026-02-17T13:16:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5875261686673070000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-089thxkgp6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713901000,"log":"[2026-02-17T13:07:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-89jkye60t4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713902000,"log":"[2026-02-17T13:02:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wtnylux5vw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713903000,"log":"[2026-02-17T12:45:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2cp3buauxn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713904000,"log":"[2026-02-17T14:18:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 149 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pnt1xt4rui","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713905000,"log":"[2026-02-17T12:36:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vghkvl7a7h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713906000,"log":"[2026-02-17T14:02:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 216ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q4v87o7sk5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713907000,"log":"[2026-02-17T13:49:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 410ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nwquae0546","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713908000,"log":"[2026-02-17T12:55:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-515ayvazdl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713909000,"log":"[2026-02-17T12:46:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tf8m9goz6o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713910000,"log":"[2026-02-17T15:00:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xf1hgf2dnn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713911000,"log":"[2026-02-17T13:43:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 417ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f8uymb764e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713912000,"log":"[2026-02-17T15:10:00Z DEBUG metrics::reporter] recorded metric active_connections value=826","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-psqsonvlu0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713913000,"log":"[2026-02-17T12:44:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t6u63ilo8k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713914000,"log":"[2026-02-17T14:53:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k89565oxek","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713915000,"log":"[2026-02-17T13:39:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7950949637938079000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-23cofi8ehv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713916000,"log":"[2026-02-17T12:49:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-593mqsvzv5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713917000,"log":"[2026-02-17T14:31:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 511 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p04rk0ozvn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713918000,"log":"[2026-02-17T14:57:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-aks363mxt9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713919000,"log":"[2026-02-17T13:36:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7350435416604763000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1f9w3gn29q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713920000,"log":"[2026-02-17T14:34:58Z INFO tonic::transport::server] grpc querier disconnected client_id=tjlyfi1i","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p30gwswxsx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713921000,"log":"[2026-02-17T14:26:33Z DEBUG sqlx::query] INSERT FROM users returned 535 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-31lo28s0os","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713922000,"log":"[2026-02-17T12:39:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uraz78vxax","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713923000,"log":"[2026-02-17T15:02:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a9tpesqtq7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713924000,"log":"[2026-02-17T15:13:52Z TRACE cache::redis] cache delete key=user:session:5282","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xjtjf774n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713925000,"log":"[2026-02-17T14:38:17Z INFO tonic::transport::server] grpc ingester connected client_id=1yt72cz7","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-64w7ubhk8r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713926000,"log":"[2026-02-17T12:55:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5hom4g9041","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713927000,"log":"[2026-02-17T13:37:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4560631767203375600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-97mdt95k4k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713928000,"log":"[2026-02-17T14:50:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mfjnlr85ww","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713929000,"log":"[2026-02-17T14:06:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 155ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ri0rk2mz1w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713930000,"log":"[2026-02-17T13:53:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 472ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jljw61akh9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713931000,"log":"[2026-02-17T13:02:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wpeq7m1reb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713932000,"log":"[2026-02-17T15:10:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=2552","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6hggge9zh1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713933000,"log":"[2026-02-17T14:41:18Z INFO tonic::transport::server] grpc querier disconnected client_id=966fly53","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xkswaz0upa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713934000,"log":"[2026-02-17T13:43:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 277ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j05veq0k1p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713935000,"log":"[2026-02-17T12:53:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xfp9j5c9u1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713936000,"log":"[2026-02-17T12:54:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-enoqtzqpo2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713937000,"log":"[2026-02-17T13:25:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9383083781217288000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-81eif1slw6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713938000,"log":"[2026-02-17T14:01:40Z INFO actix_web::middleware::logger] POST /api/_search 200 236ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m6e7h8veo4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713939000,"log":"[2026-02-17T14:09:08Z DEBUG sqlx::query] INSERT FROM users returned 400 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fg60ph0dz5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713940000,"log":"[2026-02-17T13:26:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2322882957906827000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ww61is6ry","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713941000,"log":"[2026-02-17T12:44:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b417klnack","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713942000,"log":"[2026-02-17T14:58:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yflrih6ahs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713943000,"log":"[2026-02-17T14:27:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 976 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-14h6pvb155","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713944000,"log":"[2026-02-17T12:36:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-88xofophhu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713945000,"log":"[2026-02-17T13:20:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8574397679486623000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-42g89zfe6h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713946000,"log":"[2026-02-17T15:18:20Z TRACE cache::redis] cache get key=query:result:4616","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qxjl48y9pl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713947000,"log":"[2026-02-17T12:48:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ethelltdts","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713948000,"log":"[2026-02-17T14:22:02Z DEBUG sqlx::query] SELECT FROM streams returned 377 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ojenq6bpj9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713949000,"log":"[2026-02-17T15:19:33Z TRACE cache::redis] cache set key=org:config:1287","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6qgidvr2ng","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713950000,"log":"[2026-02-17T13:18:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9701775113581718000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k6oazb2a0k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713951000,"log":"[2026-02-17T15:13:45Z TRACE cache::redis] cache set key=query:result:216","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xuy7ad7xsu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713952000,"log":"[2026-02-17T15:21:47Z TRACE cache::redis] cache expire key=user:session:1297","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0tn5g1cpxu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713953000,"log":"[2026-02-17T13:26:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7215886549936088000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g8l53ylcgb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713954000,"log":"[2026-02-17T13:02:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6j7h159zla","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713955000,"log":"[2026-02-17T15:17:19Z TRACE cache::redis] cache miss key=stream:schema:4565","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1o54f5mosx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713956000,"log":"[2026-02-17T13:48:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 362ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8siss749jn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713957000,"log":"[2026-02-17T12:57:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ku0dx6m1sn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713958000,"log":"[2026-02-17T13:13:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1605321745374870300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kw60cuqace","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713959000,"log":"[2026-02-17T12:38:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4q555vq4k1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713960000,"log":"[2026-02-17T14:15:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 552 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-osw0luo37f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713961000,"log":"[2026-02-17T14:30:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 620 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-phtg9nix7i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713962000,"log":"[2026-02-17T13:58:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 496ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hjgidmmj18","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713963000,"log":"[2026-02-17T13:51:25Z INFO actix_web::middleware::logger] GET /api/_search 404 53ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qtfrw3t7wb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713964000,"log":"[2026-02-17T14:54:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-glovhne7ht","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713965000,"log":"[2026-02-17T12:49:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h1n6v6mcu5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713966000,"log":"[2026-02-17T12:57:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q83djxv0nh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713967000,"log":"[2026-02-17T12:51:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7kapjmggov","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713968000,"log":"[2026-02-17T12:43:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ikqlt83cca","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713969000,"log":"[2026-02-17T14:05:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 32ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9x9ilh3l26","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713970000,"log":"[2026-02-17T13:31:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8534837406480784000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e7korz7xxy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713971000,"log":"[2026-02-17T12:56:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5ejydu9t60","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713972000,"log":"[2026-02-17T13:01:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ceu2qt91jh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713973000,"log":"[2026-02-17T14:40:33Z INFO tonic::transport::server] grpc querier disconnected client_id=07rng0m7","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kmgrfcqjz8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713974000,"log":"[2026-02-17T12:51:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rh672uzl8p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713975000,"log":"[2026-02-17T13:33:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2223304042288323300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bsvg8rdedo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713976000,"log":"[2026-02-17T14:29:55Z DEBUG sqlx::query] DELETE FROM alerts returned 845 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-514cvsrlwm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713977000,"log":"[2026-02-17T12:41:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ln93en59is","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713978000,"log":"[2026-02-17T15:16:09Z TRACE cache::redis] cache set key=stream:schema:5720","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a2bd3f34i6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713979000,"log":"[2026-02-17T14:19:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 628 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ey60gt6cdi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713980000,"log":"[2026-02-17T15:05:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6dzc2o29on","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713981000,"log":"[2026-02-17T13:39:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4994569571872438000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-650jblrttk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713982000,"log":"[2026-02-17T14:56:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3i9f4tq1xq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713983000,"log":"[2026-02-17T13:17:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6516398467632629000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tqgrls7ido","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713984000,"log":"[2026-02-17T12:36:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mua3l3boht","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713985000,"log":"[2026-02-17T15:00:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ff4apklch","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713986000,"log":"[2026-02-17T14:52:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cgjyipanlq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713987000,"log":"[2026-02-17T13:33:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2360488052132510000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-56l3y5m9qp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713988000,"log":"[2026-02-17T14:27:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 849 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q3goaoqia9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713989000,"log":"[2026-02-17T14:01:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 321ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bh5kplly40","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713990000,"log":"[2026-02-17T12:37:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-botet5rtco","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713991000,"log":"[2026-02-17T13:28:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9832396776190070000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hr7h9c9edf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713992000,"log":"[2026-02-17T13:57:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 431ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bi2pkg3fgi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713993000,"log":"[2026-02-17T15:07:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=792","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cwrjs0pc9q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713994000,"log":"[2026-02-17T15:11:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=5884","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bn2kxzmb4q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713995000,"log":"[2026-02-17T15:20:43Z TRACE cache::redis] cache miss key=org:config:6279","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u96uwzvbss","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713996000,"log":"[2026-02-17T13:26:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4298872721583091000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4qix4hdc94","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331713997000,"log":"[2026-02-17T15:15:50Z TRACE cache::redis] cache get key=query:result:6271","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cu8j1i0z9u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331713998000,"log":"[2026-02-17T15:06:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1188","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2oih6h8ii5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331713999000,"log":"[2026-02-17T13:41:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4406159358391801300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xr8zul7j7a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714000000,"log":"[2026-02-17T14:12:33Z DEBUG sqlx::query] INSERT FROM users returned 849 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-inqi0452ng","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714001000,"log":"[2026-02-17T13:11:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9086189573922536000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6rc0p0eool","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714002000,"log":"[2026-02-17T13:29:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8784303833008191000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2pk1hflx5y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714003000,"log":"[2026-02-17T14:31:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 978 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iu40y2t8xc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714004000,"log":"[2026-02-17T15:14:22Z TRACE cache::redis] cache delete key=user:session:7241","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gemj1epn3y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714005000,"log":"[2026-02-17T13:44:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 106ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t6mfz009th","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714006000,"log":"[2026-02-17T14:14:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 86 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dubwfr44vg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714007000,"log":"[2026-02-17T14:21:02Z DEBUG sqlx::query] SELECT FROM streams returned 317 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mf5kjpzavl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714008000,"log":"[2026-02-17T14:46:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=d39lz97j","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0774392asj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714009000,"log":"[2026-02-17T14:53:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xh7nihfcx4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714010000,"log":"[2026-02-17T14:31:08Z DEBUG sqlx::query] INSERT FROM users returned 8 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h32ngulhfk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714011000,"log":"[2026-02-17T13:16:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4626113189830707000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y66fs4fqey","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714012000,"log":"[2026-02-17T13:59:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 329ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fmslsereof","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714013000,"log":"[2026-02-17T12:52:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mels1a9r2j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714014000,"log":"[2026-02-17T12:35:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-40rej6rgro","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714015000,"log":"[2026-02-17T13:23:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6418211938924570000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tqdcuhx4le","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714016000,"log":"[2026-02-17T13:16:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5181356345040146000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uz6ov1mhog","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714017000,"log":"[2026-02-17T14:14:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 103 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xv17bhlo9l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714018000,"log":"[2026-02-17T13:45:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 335ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qcmrvnjup5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714019000,"log":"[2026-02-17T14:25:13Z DEBUG sqlx::query] INSERT FROM users returned 300 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2bpxstfnqo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714020000,"log":"[2026-02-17T14:19:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 38 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pg33vm4zfm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714021000,"log":"[2026-02-17T14:37:38Z INFO tonic::transport::server] grpc querier disconnected client_id=73iqftmg","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-47ipze1r9k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714022000,"log":"[2026-02-17T15:12:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4915","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5q8b1nu1uz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714023000,"log":"[2026-02-17T13:52:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 278ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-id9wgtlzmd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714024000,"log":"[2026-02-17T14:04:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 154ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s53hwln01o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714025000,"log":"[2026-02-17T13:49:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 356ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z85ntro8cd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714026000,"log":"[2026-02-17T12:41:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dkyfjbvewf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714027000,"log":"[2026-02-17T13:23:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4193009756456642600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-67lpajesld","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714028000,"log":"[2026-02-17T13:41:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5049861487969371000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mh8j7w17c2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714029000,"log":"[2026-02-17T14:34:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2gt991xw","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ukxigvpj37","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714030000,"log":"[2026-02-17T12:45:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lbta7z1fso","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714031000,"log":"[2026-02-17T14:51:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wsquhtpmry","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714032000,"log":"[2026-02-17T13:58:00Z INFO actix_web::middleware::logger] POST /api/_search 403 42ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5xwab918gy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714033000,"log":"[2026-02-17T13:04:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1pwi8lda79","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714034000,"log":"[2026-02-17T14:06:25Z INFO actix_web::middleware::logger] GET /api/_search 404 35ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6mxct004sc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714035000,"log":"[2026-02-17T14:32:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=nbg0af2j","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lc6njlkhrw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714036000,"log":"[2026-02-17T14:16:50Z DEBUG sqlx::query] DELETE FROM alerts returned 402 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hvvmzuhj0n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714037000,"log":"[2026-02-17T14:33:57Z INFO tonic::transport::server] grpc ingester connected client_id=qsy0fv1o","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uoeq04sd53","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714038000,"log":"[2026-02-17T12:38:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zxjncn4j3n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714039000,"log":"[2026-02-17T14:29:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 984 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rtndgccytu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714040000,"log":"[2026-02-17T13:53:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 447ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nfcd3d7sq3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714041000,"log":"[2026-02-17T14:47:04Z INFO tonic::transport::server] grpc router request_received client_id=kmijmwzz","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iuadotm3op","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714042000,"log":"[2026-02-17T14:57:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3fo6zyecqi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714043000,"log":"[2026-02-17T15:08:15Z DEBUG metrics::reporter] recorded metric active_connections value=5714","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ywxvtg1oqy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714044000,"log":"[2026-02-17T13:11:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2960444657989054000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-konz8jwdi7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714045000,"log":"[2026-02-17T14:54:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1ygtcgtnr6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714046000,"log":"[2026-02-17T14:00:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 493ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pka4pgv9n0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714047000,"log":"[2026-02-17T14:24:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 933 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zd7rggaw0d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714048000,"log":"[2026-02-17T13:57:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 186ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4e8ihxsfcx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714049000,"log":"[2026-02-17T12:46:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-csss3f3703","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714050000,"log":"[2026-02-17T13:02:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1vrsenb6ll","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714051000,"log":"[2026-02-17T14:17:13Z DEBUG sqlx::query] INSERT FROM users returned 803 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hhoqpxr59b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714052000,"log":"[2026-02-17T13:10:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7053242675413185000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rnz4lgd1ud","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714053000,"log":"[2026-02-17T13:57:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 428ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dwrdo03bm6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714054000,"log":"[2026-02-17T15:01:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b50w29004o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714055000,"log":"[2026-02-17T15:04:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i75cn6d8x1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714056000,"log":"[2026-02-17T14:07:18Z DEBUG sqlx::query] INSERT FROM users returned 742 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ouwrr5l989","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714057000,"log":"[2026-02-17T13:30:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1038960209946997500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jh27eoivjk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714058000,"log":"[2026-02-17T15:05:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3221","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z3g3unxmlu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714059000,"log":"[2026-02-17T13:24:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4447291231231762400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o6c6xwng18","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714060000,"log":"[2026-02-17T14:02:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 111ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-psdz4ws526","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714061000,"log":"[2026-02-17T14:11:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 105 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cg4or8pxuj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714062000,"log":"[2026-02-17T15:19:32Z TRACE cache::redis] cache get key=user:session:3653","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xdzfhds1fy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714063000,"log":"[2026-02-17T12:55:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u4hp0gvwnb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714064000,"log":"[2026-02-17T13:47:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 325ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yrlcyodmb9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714065000,"log":"[2026-02-17T14:55:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lkcicic4wl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714066000,"log":"[2026-02-17T13:31:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7479626653459490000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-93rr6avpwa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714067000,"log":"[2026-02-17T14:27:03Z DEBUG sqlx::query] INSERT FROM users returned 320 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2rbbjlhuwy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714068000,"log":"[2026-02-17T14:08:40Z DEBUG sqlx::query] DELETE FROM alerts returned 275 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z6kcwqmvae","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714069000,"log":"[2026-02-17T12:50:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oi5ld9i1jq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714070000,"log":"[2026-02-17T14:05:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 8ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ey3ik3nm5e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714071000,"log":"[2026-02-17T13:27:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/400854708252038600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cyxwnz3lo9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714072000,"log":"[2026-02-17T14:11:28Z DEBUG sqlx::query] INSERT FROM users returned 500 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6h7qfevt6r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714073000,"log":"[2026-02-17T15:04:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kk97ov96z4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714074000,"log":"[2026-02-17T15:05:55Z DEBUG metrics::reporter] recorded metric active_connections value=336","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2huv0c3jxu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714075000,"log":"[2026-02-17T14:00:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ztpcbn8oai","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714076000,"log":"[2026-02-17T13:22:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5029740798465638000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-op2gnu5txm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714077000,"log":"[2026-02-17T14:39:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=fm7du4eu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nhtbeyrhfx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714078000,"log":"[2026-02-17T14:22:07Z DEBUG sqlx::query] SELECT FROM streams returned 202 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e3ia6x96gr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714079000,"log":"[2026-02-17T14:36:43Z INFO tonic::transport::server] grpc querier disconnected client_id=syocqleh","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mmr1nk1x7f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714080000,"log":"[2026-02-17T14:43:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=cs6y376w","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v75ev53jnp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714081000,"log":"[2026-02-17T12:35:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2jmbnrd4ra","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714082000,"log":"[2026-02-17T14:08:52Z DEBUG sqlx::query] SELECT FROM streams returned 364 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6a8ye5f0gf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714083000,"log":"[2026-02-17T14:33:39Z INFO tonic::transport::server] grpc router request_received client_id=1bgfgvs2","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aw71wnruo2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714084000,"log":"[2026-02-17T14:24:40Z DEBUG sqlx::query] DELETE FROM alerts returned 884 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pb56s14nri","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714085000,"log":"[2026-02-17T13:09:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7924936737822068000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fyqo7meoo1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714086000,"log":"[2026-02-17T13:42:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 4ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5qfj1q9fxo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714087000,"log":"[2026-02-17T13:35:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/341172344269660160.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d3fx6es2g9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714088000,"log":"[2026-02-17T12:38:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bwnxcr55t7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714089000,"log":"[2026-02-17T13:39:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4638034581068371000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9lhxhpurpl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714090000,"log":"[2026-02-17T14:35:13Z INFO tonic::transport::server] grpc querier disconnected client_id=30dmmqnt","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r7gkl5swmz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714091000,"log":"[2026-02-17T12:40:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ji9tdo9wlu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714092000,"log":"[2026-02-17T13:02:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8x4jrls8yr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714093000,"log":"[2026-02-17T12:37:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ju0nxvuqn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714094000,"log":"[2026-02-17T14:06:58Z DEBUG sqlx::query] INSERT FROM users returned 221 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9f7r6hnrg5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714095000,"log":"[2026-02-17T14:34:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=bl2ra62u","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-voimuqyhpj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714096000,"log":"[2026-02-17T14:09:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 788 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fc2r5xvt7s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714097000,"log":"[2026-02-17T15:16:52Z TRACE cache::redis] cache delete key=user:session:8954","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6iheme87w7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714098000,"log":"[2026-02-17T15:16:51Z TRACE cache::redis] cache set key=auth:token:461","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kib2cn250g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714099000,"log":"[2026-02-17T13:45:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 474ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n7hxif4noq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714100000,"log":"[2026-02-17T14:25:42Z DEBUG sqlx::query] SELECT FROM streams returned 76 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gc6c0bhrda","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714101000,"log":"[2026-02-17T13:19:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8523053765880351000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ku952nii8c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714102000,"log":"[2026-02-17T14:51:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l2ygoyb8hc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714103000,"log":"[2026-02-17T14:01:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 90ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l6p9dfpy5v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714104000,"log":"[2026-02-17T14:03:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 500ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yk9vvxgqmk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714105000,"log":"[2026-02-17T14:11:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 885 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-whbtb3egka","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714106000,"log":"[2026-02-17T15:10:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6904","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nlske2vyyt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714107000,"log":"[2026-02-17T13:57:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 250ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j85wiw7i6o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714108000,"log":"[2026-02-17T15:11:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1516","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6sf49z5gsb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714109000,"log":"[2026-02-17T14:17:10Z DEBUG sqlx::query] DELETE FROM alerts returned 650 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4hq8uqbrz6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714110000,"log":"[2026-02-17T14:33:02Z INFO tonic::transport::server] grpc ingester connected client_id=9h5oi8nm","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ohp33c6bhp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714111000,"log":"[2026-02-17T12:59:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s5kg4gc7tz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714112000,"log":"[2026-02-17T14:29:00Z DEBUG sqlx::query] DELETE FROM alerts returned 978 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fr6kf4o350","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714113000,"log":"[2026-02-17T14:59:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9aog719c89","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714114000,"log":"[2026-02-17T12:43:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c6du866myh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714115000,"log":"[2026-02-17T13:26:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9923023375305957000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t83ptfgwjd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714116000,"log":"[2026-02-17T12:57:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nhstrgd1ht","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714117000,"log":"[2026-02-17T13:28:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2146412750333055500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e8fh76wihh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714118000,"log":"[2026-02-17T14:46:23Z INFO tonic::transport::server] grpc querier disconnected client_id=c7w0suhp","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gx6t6g1nnx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714119000,"log":"[2026-02-17T14:44:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=314m26rq","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xej8r6f842","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714120000,"log":"[2026-02-17T13:02:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f8egzf90ja","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714121000,"log":"[2026-02-17T12:47:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1ht6r00tn1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714122000,"log":"[2026-02-17T14:36:13Z INFO tonic::transport::server] grpc querier disconnected client_id=6rku32jx","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jlj0eqfenv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714123000,"log":"[2026-02-17T15:21:32Z TRACE cache::redis] cache get key=user:session:212","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wawdbpcadh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714124000,"log":"[2026-02-17T13:27:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4166413662336088000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9viq9s3rju","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714125000,"log":"[2026-02-17T12:37:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ebz5w9eb5t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714126000,"log":"[2026-02-17T12:49:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pnd6l2lo9g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714127000,"log":"[2026-02-17T12:51:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eewj1zk2ko","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714128000,"log":"[2026-02-17T12:46:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-izo88t2e9r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714129000,"log":"[2026-02-17T12:37:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fdywtxw3o2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714130000,"log":"[2026-02-17T14:39:28Z INFO tonic::transport::server] grpc querier disconnected client_id=6c1d2f3a","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qne33m9s7x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714131000,"log":"[2026-02-17T14:39:29Z INFO tonic::transport::server] grpc router request_received client_id=y59wqrfq","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uiaxqngtvm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714132000,"log":"[2026-02-17T14:41:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=8g2u77xv","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1dklj3lpru","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714133000,"log":"[2026-02-17T13:52:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 92ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7ql3qvp723","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714134000,"log":"[2026-02-17T14:49:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h6txdoq1dw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714135000,"log":"[2026-02-17T13:56:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 388ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l93ukrzspq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714136000,"log":"[2026-02-17T12:40:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6s97sxaylb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714137000,"log":"[2026-02-17T14:23:52Z DEBUG sqlx::query] SELECT FROM streams returned 923 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rmzwyinikz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714138000,"log":"[2026-02-17T13:29:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9636120142759694000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z5ofi7nqqr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714139000,"log":"[2026-02-17T13:17:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2625749455112027600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xrph5qvb0o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714140000,"log":"[2026-02-17T13:15:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5188996219468998000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iazccuiw0l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714141000,"log":"[2026-02-17T13:54:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 116ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jireyxo5b3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714142000,"log":"[2026-02-17T14:06:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jl88cqmron","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714143000,"log":"[2026-02-17T13:18:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1016114269531089800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b21jeghdlz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714144000,"log":"[2026-02-17T13:43:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 335ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gg862zt9fc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714145000,"log":"[2026-02-17T12:51:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7ibog7dbzq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714146000,"log":"[2026-02-17T15:04:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a7424kpp8b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714147000,"log":"[2026-02-17T14:03:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 154ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rj1wlfxst","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714148000,"log":"[2026-02-17T14:06:45Z INFO actix_web::middleware::logger] GET /api/_search 503 214ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c8bf10ztem","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714149000,"log":"[2026-02-17T12:56:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u4mz9fygfy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714150000,"log":"[2026-02-17T15:04:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s446usygls","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714151000,"log":"[2026-02-17T15:14:40Z TRACE cache::redis] cache delete key=query:result:1731","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i179o4yvkh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714152000,"log":"[2026-02-17T15:09:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8091","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-240b33eg6a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714153000,"log":"[2026-02-17T14:22:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 572 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v3comp2kbg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714154000,"log":"[2026-02-17T14:41:32Z INFO tonic::transport::server] grpc ingester connected client_id=gp5rrxzt","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x60jkrx3m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714155000,"log":"[2026-02-17T14:00:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 342ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pt6f7rcdsr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714156000,"log":"[2026-02-17T14:03:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 123ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xw7kkeysqx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714157000,"log":"[2026-02-17T12:56:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2pl60t0w6n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714158000,"log":"[2026-02-17T15:04:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ir4z6j8emc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714159000,"log":"[2026-02-17T14:16:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 892 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dyrd0vlx86","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714160000,"log":"[2026-02-17T13:09:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5929868088273007000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-79mqgholsa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714161000,"log":"[2026-02-17T15:20:42Z TRACE cache::redis] cache hit key=user:session:7215","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e1ddd3fc21","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714162000,"log":"[2026-02-17T12:52:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5o83g6d4bq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714163000,"log":"[2026-02-17T12:49:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hoh0on9f0z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714164000,"log":"[2026-02-17T14:12:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 962 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vd6xf8lzyy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714165000,"log":"[2026-02-17T13:30:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7905050209708700000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kfu3motq6o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714166000,"log":"[2026-02-17T14:09:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 400 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5i5ir5g7vg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714167000,"log":"[2026-02-17T13:29:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4569333486434388000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2o7k0olyri","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714168000,"log":"[2026-02-17T15:00:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-18evc0iq2r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714169000,"log":"[2026-02-17T14:24:00Z DEBUG sqlx::query] DELETE FROM alerts returned 592 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hyo22p0cxv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714170000,"log":"[2026-02-17T15:17:59Z TRACE cache::redis] cache expire key=stream:schema:2313","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vmx27vuyog","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714171000,"log":"[2026-02-17T13:08:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8617472258236105000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yofohp31dp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714172000,"log":"[2026-02-17T13:54:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 291ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-77wxph9m21","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714173000,"log":"[2026-02-17T15:07:20Z DEBUG metrics::reporter] recorded metric active_connections value=5304","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1zhtqbh5vy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714174000,"log":"[2026-02-17T14:33:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=hi6j81mt","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-97c3ks8aur","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714175000,"log":"[2026-02-17T14:20:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 727 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hglz4c99tx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714176000,"log":"[2026-02-17T14:26:23Z DEBUG sqlx::query] INSERT FROM users returned 399 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jb9fdmgj88","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714177000,"log":"[2026-02-17T13:14:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6336960249518422000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b3rjpgbuiv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714178000,"log":"[2026-02-17T14:12:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 46 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bivw378qtr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714179000,"log":"[2026-02-17T13:45:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 25ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o9rdnb5to2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714180000,"log":"[2026-02-17T13:02:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hegndac8u7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714181000,"log":"[2026-02-17T13:15:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/72922099921655100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ho00rawjgu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714182000,"log":"[2026-02-17T14:50:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pvjg5oyotz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714183000,"log":"[2026-02-17T14:05:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 406ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yx6updozoh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714184000,"log":"[2026-02-17T13:08:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bvsnbkomhx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714185000,"log":"[2026-02-17T14:20:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 388 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ca9a9yusf3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714186000,"log":"[2026-02-17T14:39:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=qsn1qyj6","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l3dkqotncr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714187000,"log":"[2026-02-17T13:26:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/331175918492789800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mi8inhpmmd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714188000,"log":"[2026-02-17T14:47:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pta6t59d","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-njas7buro9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714189000,"log":"[2026-02-17T13:58:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 192ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-634868vuev","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714190000,"log":"[2026-02-17T13:03:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o6bjob4myd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714191000,"log":"[2026-02-17T13:07:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-65quvst109","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714192000,"log":"[2026-02-17T14:10:45Z DEBUG sqlx::query] DELETE FROM alerts returned 614 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o3687xxpwm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714193000,"log":"[2026-02-17T13:20:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1914156203781172200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xjzhld46im","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714194000,"log":"[2026-02-17T12:49:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-27iuyix07t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714195000,"log":"[2026-02-17T14:56:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m5d9lsiqko","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714196000,"log":"[2026-02-17T14:05:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 377ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qp7fx4qjit","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714197000,"log":"[2026-02-17T12:54:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6zbodrcilr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714198000,"log":"[2026-02-17T13:57:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 228ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yrs0ojwfok","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714199000,"log":"[2026-02-17T13:50:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 282ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n1lozu1fq3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714200000,"log":"[2026-02-17T14:16:27Z DEBUG sqlx::query] SELECT FROM streams returned 403 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a82wshybwx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714201000,"log":"[2026-02-17T13:07:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c85svc3r8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714202000,"log":"[2026-02-17T15:11:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=493","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ixvl97jm78","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714203000,"log":"[2026-02-17T14:27:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 814 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8rz0lkdivc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714204000,"log":"[2026-02-17T15:15:28Z TRACE cache::redis] cache delete key=org:config:5623","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xzx7gagg9t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714205000,"log":"[2026-02-17T15:19:53Z TRACE cache::redis] cache expire key=org:config:6094","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hmvmk7u3o6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714206000,"log":"[2026-02-17T14:42:49Z INFO tonic::transport::server] grpc router request_received client_id=ls56jdqn","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3hx1q82q3f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714207000,"log":"[2026-02-17T13:12:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7466019325595312000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1cg1pvl9a8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714208000,"log":"[2026-02-17T12:41:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-741i4qo03p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714209000,"log":"[2026-02-17T13:28:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7879121090651855000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-phejfcueo9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714210000,"log":"[2026-02-17T12:42:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h56lb7urp5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714211000,"log":"[2026-02-17T13:48:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 291ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tk6xw1xkap","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714212000,"log":"[2026-02-17T13:04:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qqg764mnno","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714213000,"log":"[2026-02-17T13:22:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6323135564252867000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9e5826yvil","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714214000,"log":"[2026-02-17T13:36:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6131527683529936000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bm4tdv5vu4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714215000,"log":"[2026-02-17T15:08:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=8","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-czkg5u6gnm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714216000,"log":"[2026-02-17T15:14:25Z TRACE cache::redis] cache miss key=query:result:1911","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ctgkvnqwue","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714217000,"log":"[2026-02-17T13:55:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 89ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4ahzmn16ps","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714218000,"log":"[2026-02-17T13:11:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1265678033268167200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5rbtbu2o9p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714219000,"log":"[2026-02-17T13:57:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 370ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tgsycvmfp0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714220000,"log":"[2026-02-17T12:38:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g7skr2fil2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714221000,"log":"[2026-02-17T14:05:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 428ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a0icfsrvv6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714222000,"log":"[2026-02-17T12:54:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2edqknyelv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714223000,"log":"[2026-02-17T13:47:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-goipc0bdh3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714224000,"log":"[2026-02-17T13:51:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 313ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lxyxm1qgqk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714225000,"log":"[2026-02-17T13:21:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8383881207480319000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fqutbjpuv1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714226000,"log":"[2026-02-17T14:51:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mwetn40ecv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714227000,"log":"[2026-02-17T15:07:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1006","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5xbptn91d1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714228000,"log":"[2026-02-17T13:39:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7544819453696636000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4alkpluu9k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714229000,"log":"[2026-02-17T14:42:52Z INFO tonic::transport::server] grpc ingester connected client_id=629askrn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dzup6o8j65","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714230000,"log":"[2026-02-17T15:04:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h03os9h9c0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714231000,"log":"[2026-02-17T12:42:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z4ovc4efng","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714232000,"log":"[2026-02-17T14:38:58Z INFO tonic::transport::server] grpc querier disconnected client_id=pdtpkhm6","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-slgz78xk81","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714233000,"log":"[2026-02-17T14:45:02Z INFO tonic::transport::server] grpc ingester connected client_id=9aheg8gn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-60racis0ky","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714234000,"log":"[2026-02-17T14:51:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r1d57j393d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714235000,"log":"[2026-02-17T12:47:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ada5quwh6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714236000,"log":"[2026-02-17T12:49:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4wustryyj5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714237000,"log":"[2026-02-17T15:14:50Z TRACE cache::redis] cache get key=query:result:6779","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k97dxedh21","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714238000,"log":"[2026-02-17T12:59:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3aninfoil2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714239000,"log":"[2026-02-17T12:42:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5v40bspxnf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714240000,"log":"[2026-02-17T14:51:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dwpo0yz35m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714241000,"log":"[2026-02-17T14:02:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 413ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vnp7h2kejr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714242000,"log":"[2026-02-17T15:05:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4509","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-im0dr4q2ub","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714243000,"log":"[2026-02-17T14:51:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z3o8gxdpup","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714244000,"log":"[2026-02-17T14:37:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=k865cn6k","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vp27cotcv9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714245000,"log":"[2026-02-17T13:11:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8650227060943150000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mmb0izmsgn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714246000,"log":"[2026-02-17T13:40:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8929870917619707000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5jxyaobyfn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714247000,"log":"[2026-02-17T14:40:04Z INFO tonic::transport::server] grpc router request_received client_id=4opukmce","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2qpw4xsm14","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714248000,"log":"[2026-02-17T13:39:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9712327590043286000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ya1md7xscj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714249000,"log":"[2026-02-17T13:44:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 125ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4w9g8f5yqk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714250000,"log":"[2026-02-17T14:21:05Z DEBUG sqlx::query] DELETE FROM alerts returned 200 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l863y22zje","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714251000,"log":"[2026-02-17T13:11:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9135981867342937000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-akc1mfbzo9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714252000,"log":"[2026-02-17T14:16:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 338 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0iwf6zpciq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714253000,"log":"[2026-02-17T14:19:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 749 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ayqlixzl8i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714254000,"log":"[2026-02-17T13:32:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9706693155393227000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mbbx3kz6rc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714255000,"log":"[2026-02-17T14:03:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 486ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fso4o1wyww","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714256000,"log":"[2026-02-17T14:45:17Z INFO tonic::transport::server] grpc ingester connected client_id=m49ay5c6","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dqyww89pwn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714257000,"log":"[2026-02-17T13:13:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7348632413056833000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e1tkm2i3yu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714258000,"log":"[2026-02-17T13:29:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5080705073820373000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h0yr6wyuqc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714259000,"log":"[2026-02-17T13:12:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/997970842490203400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2cugczhkp1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714260000,"log":"[2026-02-17T13:47:05Z INFO actix_web::middleware::logger] GET /api/_search 500 65ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-94ge0w3qfp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714261000,"log":"[2026-02-17T12:49:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nizp0bdu7d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714262000,"log":"[2026-02-17T14:48:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=wqd7o7s0","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5liapsildw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714263000,"log":"[2026-02-17T13:51:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 329ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fu5ab1cz9l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714264000,"log":"[2026-02-17T13:31:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/846618779859520400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a0f12pvgvc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714265000,"log":"[2026-02-17T14:08:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 993 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p49f0k9w9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714266000,"log":"[2026-02-17T12:44:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ad5h9jq1u3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714267000,"log":"[2026-02-17T12:35:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0am2djch08","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714268000,"log":"[2026-02-17T13:57:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 349ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-swzhbarz8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714269000,"log":"[2026-02-17T13:27:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/474585817123875900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lwxgr0f207","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714270000,"log":"[2026-02-17T14:30:43Z DEBUG sqlx::query] INSERT FROM users returned 401 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-paipdsa3j8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714271000,"log":"[2026-02-17T14:40:42Z INFO tonic::transport::server] grpc ingester connected client_id=l3gzz0di","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m5r2i21joj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714272000,"log":"[2026-02-17T15:12:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=4652","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wvuhjvo2dh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714273000,"log":"[2026-02-17T14:46:52Z INFO tonic::transport::server] grpc ingester connected client_id=ha24rswx","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bcuvqhfye1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714274000,"log":"[2026-02-17T13:06:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q359mhogay","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714275000,"log":"[2026-02-17T12:57:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7om3k0p4o2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714276000,"log":"[2026-02-17T13:42:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 131ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0bs5val88i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714277000,"log":"[2026-02-17T13:58:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 282ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2lxh34baq6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714278000,"log":"[2026-02-17T13:07:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d0hlnxzy76","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714279000,"log":"[2026-02-17T15:00:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wx3zym4wnq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714280000,"log":"[2026-02-17T14:49:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nd4ldbh3hc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714281000,"log":"[2026-02-17T13:58:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 67ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wrt92okhhz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714282000,"log":"[2026-02-17T13:39:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/803516197563556400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-265ai36qx0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714283000,"log":"[2026-02-17T14:09:35Z DEBUG sqlx::query] DELETE FROM alerts returned 173 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2n2w7ihm1p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714284000,"log":"[2026-02-17T14:04:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 112ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0e7ixeiba6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714285000,"log":"[2026-02-17T13:17:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1313282137153759500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sags0u7s1s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714286000,"log":"[2026-02-17T12:59:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jbscx3sk9n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714287000,"log":"[2026-02-17T14:45:44Z INFO tonic::transport::server] grpc router request_received client_id=8h6ynza8","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p15z2ssyub","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714288000,"log":"[2026-02-17T14:48:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vewftt1oo6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714289000,"log":"[2026-02-17T14:04:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 191ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vhi7day79n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714290000,"log":"[2026-02-17T14:29:18Z DEBUG sqlx::query] INSERT FROM users returned 644 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-99qrhkd0pe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714291000,"log":"[2026-02-17T14:18:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 942 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r97m8xyxly","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714292000,"log":"[2026-02-17T13:49:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7yfekjjo3k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714293000,"log":"[2026-02-17T13:40:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2788574694452237300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-owcl36gq1v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714294000,"log":"[2026-02-17T13:43:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 168ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jdlwxqzjz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714295000,"log":"[2026-02-17T14:29:50Z DEBUG sqlx::query] DELETE FROM alerts returned 412 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5fsnf7sznw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714296000,"log":"[2026-02-17T15:08:20Z DEBUG metrics::reporter] recorded metric active_connections value=1735","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-unmiaha5x4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714297000,"log":"[2026-02-17T12:43:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fon58dvopq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714298000,"log":"[2026-02-17T14:36:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=3b2d0398","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lr6daw0gpt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714299000,"log":"[2026-02-17T13:14:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2118975002042065000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eclcnavyeq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714300000,"log":"[2026-02-17T12:45:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-31wfrady7y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714301000,"log":"[2026-02-17T15:10:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=4340","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d6wljgh2x0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714302000,"log":"[2026-02-17T14:33:04Z INFO tonic::transport::server] grpc router request_received client_id=8166d7pt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9ewdgp9zt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714303000,"log":"[2026-02-17T14:13:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 275 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xxh8d7cm3s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714304000,"log":"[2026-02-17T15:03:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qqohhslns2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714305000,"log":"[2026-02-17T14:00:00Z INFO actix_web::middleware::logger] POST /api/_search 503 421ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u4xz2xd5d8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714306000,"log":"[2026-02-17T13:22:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4927075884856482000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uw840t6518","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714307000,"log":"[2026-02-17T14:48:02Z INFO tonic::transport::server] grpc ingester connected client_id=16mpdx8s","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wxrab3jiej","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714308000,"log":"[2026-02-17T13:54:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 97ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ehhhyj3yi2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714309000,"log":"[2026-02-17T14:38:42Z INFO tonic::transport::server] grpc ingester connected client_id=b84n1umk","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hwl6a3czvc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714310000,"log":"[2026-02-17T13:31:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5736700752057964000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ptrnkuc5ub","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714311000,"log":"[2026-02-17T14:58:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uckpnza8t3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714312000,"log":"[2026-02-17T14:19:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 989 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vrbhwwh2qx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714313000,"log":"[2026-02-17T13:14:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4641175548488938000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uxvegwmtjp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714314000,"log":"[2026-02-17T14:14:12Z DEBUG sqlx::query] SELECT FROM streams returned 947 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b4n1el6hwh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714315000,"log":"[2026-02-17T15:10:45Z DEBUG metrics::reporter] recorded metric active_connections value=4693","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y3j8sxekz7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714316000,"log":"[2026-02-17T12:43:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0ezpst5170","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714317000,"log":"[2026-02-17T12:37:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ivt8bt4eoi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714318000,"log":"[2026-02-17T14:32:54Z INFO tonic::transport::server] grpc router request_received client_id=99z46dtt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0dn3rg2mu3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714319000,"log":"[2026-02-17T13:24:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7057472814767205000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-47d4cx451z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714320000,"log":"[2026-02-17T14:32:09Z INFO tonic::transport::server] grpc router request_received client_id=66ks1f3f","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g3xw1gtpsq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714321000,"log":"[2026-02-17T12:46:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k6mfz5258u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714322000,"log":"[2026-02-17T14:24:15Z DEBUG sqlx::query] DELETE FROM alerts returned 477 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sa9ro2kde0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714323000,"log":"[2026-02-17T12:58:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kcd28wuzb9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714324000,"log":"[2026-02-17T13:07:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z8xsfqvo78","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714325000,"log":"[2026-02-17T14:56:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ypxnnipey4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714326000,"log":"[2026-02-17T14:12:53Z DEBUG sqlx::query] INSERT FROM users returned 201 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ixxjhv51kp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714327000,"log":"[2026-02-17T13:10:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7256229824993368000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-boq2qqe0e3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714328000,"log":"[2026-02-17T15:02:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ejjglccpio","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714329000,"log":"[2026-02-17T14:21:47Z DEBUG sqlx::query] SELECT FROM streams returned 718 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i9p3gq9c57","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714330000,"log":"[2026-02-17T14:10:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 481 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-21sq2n0sot","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714331000,"log":"[2026-02-17T15:13:30Z DEBUG metrics::reporter] recorded metric active_connections value=1078","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2zwtrwlxon","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714332000,"log":"[2026-02-17T15:03:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cl0r3jm8h5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714333000,"log":"[2026-02-17T14:43:09Z INFO tonic::transport::server] grpc router request_received client_id=tdizsuvd","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2r2a0eyd3v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714334000,"log":"[2026-02-17T14:03:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 238ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0iksiqm3nz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714335000,"log":"[2026-02-17T14:35:37Z INFO tonic::transport::server] grpc ingester connected client_id=7pxcolyj","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rl8ragnmm9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714336000,"log":"[2026-02-17T15:21:02Z TRACE cache::redis] cache get key=user:session:2947","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n71jmctzfo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714337000,"log":"[2026-02-17T13:04:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-je87qfuvqt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714338000,"log":"[2026-02-17T14:27:23Z DEBUG sqlx::query] INSERT FROM users returned 385 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j982xzxzvl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714339000,"log":"[2026-02-17T14:29:03Z DEBUG sqlx::query] INSERT FROM users returned 289 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1frxmsog9f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714340000,"log":"[2026-02-17T14:44:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=1iovk29w","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-70y68a6dig","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714341000,"log":"[2026-02-17T13:52:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 470ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zjmj097tp4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714342000,"log":"[2026-02-17T13:25:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2892153248515674000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jkxf92q2xu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714343000,"log":"[2026-02-17T13:33:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9050169417538201000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-26ciolwf4n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714344000,"log":"[2026-02-17T14:18:00Z DEBUG sqlx::query] DELETE FROM alerts returned 510 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ojv3gtnl4s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714345000,"log":"[2026-02-17T13:19:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9950404328723993000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9zdgjm8qgy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714346000,"log":"[2026-02-17T13:52:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 196ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-knq5gjwidq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714347000,"log":"[2026-02-17T12:54:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ere76q3t19","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714348000,"log":"[2026-02-17T13:44:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 24ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n0fxpj8j16","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714349000,"log":"[2026-02-17T14:54:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dz6gueh9td","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714350000,"log":"[2026-02-17T13:26:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7681541541480288000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lz6v1sunat","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714351000,"log":"[2026-02-17T13:22:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4016814334087928300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5n0ifxhd9o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714352000,"log":"[2026-02-17T13:28:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9576569660000231000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nwnc2ac98z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714353000,"log":"[2026-02-17T13:15:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8854998149283255000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nrwjhtdk4j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714354000,"log":"[2026-02-17T13:57:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 194ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-svjc6df53x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714355000,"log":"[2026-02-17T12:52:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ihaff545q7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714356000,"log":"[2026-02-17T12:53:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fb5negxkq7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714357000,"log":"[2026-02-17T14:49:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mov8kck3vj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714358000,"log":"[2026-02-17T13:30:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5261094734626719000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-skspgcbnat","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714359000,"log":"[2026-02-17T13:28:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/634104498269587600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jla1z8uk5k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714360000,"log":"[2026-02-17T14:20:28Z DEBUG sqlx::query] INSERT FROM users returned 32 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zynian8mrw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714361000,"log":"[2026-02-17T13:59:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 489ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hcncsto3rg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714362000,"log":"[2026-02-17T15:09:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=2704","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4o31pu1bf6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714363000,"log":"[2026-02-17T13:54:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 157ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3hvrhg62in","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714364000,"log":"[2026-02-17T12:50:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bu8aot6tdi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714365000,"log":"[2026-02-17T13:03:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u0pxegxorm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714366000,"log":"[2026-02-17T15:16:11Z TRACE cache::redis] cache expire key=auth:token:5208","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-natx7xt01j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714367000,"log":"[2026-02-17T13:56:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 94ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lqr628z207","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714368000,"log":"[2026-02-17T13:46:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nw3z1fuk73","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714369000,"log":"[2026-02-17T13:55:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 313ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-unhhhy70pn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714370000,"log":"[2026-02-17T14:03:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 161ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vxtv3go0jq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714371000,"log":"[2026-02-17T12:58:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7hf6nrhlue","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714372000,"log":"[2026-02-17T12:39:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gsz92sxps0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714373000,"log":"[2026-02-17T12:53:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eflxx49mye","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714374000,"log":"[2026-02-17T15:08:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8705","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h3v97c8b6d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714375000,"log":"[2026-02-17T14:26:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 433 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-93fob4a9x3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714376000,"log":"[2026-02-17T12:36:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9mm0jh15c6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714377000,"log":"[2026-02-17T13:19:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8572575743029110000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sm2otgkk95","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714378000,"log":"[2026-02-17T13:17:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7998403739155742000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xc3d3n2yrb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714379000,"log":"[2026-02-17T13:14:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2123655599516989200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l48psw80zl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714380000,"log":"[2026-02-17T13:03:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hbazwj2s3e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714381000,"log":"[2026-02-17T12:54:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c31plfgsln","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714382000,"log":"[2026-02-17T12:51:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-081k8i4ny2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714383000,"log":"[2026-02-17T13:45:20Z INFO actix_web::middleware::logger] POST /api/_search 201 391ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-65qznwd1yh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714384000,"log":"[2026-02-17T14:45:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=hi3f1f9f","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bo5azsphgd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714385000,"log":"[2026-02-17T15:12:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8255","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-58vhl09c15","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714386000,"log":"[2026-02-17T13:46:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 423ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i2yv9q6f5o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714387000,"log":"[2026-02-17T13:56:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 280ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wpr2bu7lka","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714388000,"log":"[2026-02-17T13:16:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2357143697194741000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w4s669gsas","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714389000,"log":"[2026-02-17T12:44:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hw8zr0i8rf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714390000,"log":"[2026-02-17T13:38:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/439609638860489700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2elt8kdgk0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714391000,"log":"[2026-02-17T13:54:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 496ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1cminp1pv6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714392000,"log":"[2026-02-17T15:11:30Z DEBUG metrics::reporter] recorded metric active_connections value=2719","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xhlvyyx31t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714393000,"log":"[2026-02-17T13:16:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2089631218729488600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hairi1fqoe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714394000,"log":"[2026-02-17T14:23:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 419 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ar5rx6ielf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714395000,"log":"[2026-02-17T13:33:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6152341108143280000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qwjvh13f5h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714396000,"log":"[2026-02-17T13:10:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6398940426463892000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dvjybrtt8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714397000,"log":"[2026-02-17T13:17:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6164431176527645000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cqiwu9e0tl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714398000,"log":"[2026-02-17T14:43:52Z INFO tonic::transport::server] grpc ingester connected client_id=h5zz99hm","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ojlp33mfk9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714399000,"log":"[2026-02-17T15:05:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4083","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ue3rctjhbu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714400000,"log":"[2026-02-17T14:34:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=kvwjvy2s","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y0591g1q46","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714401000,"log":"[2026-02-17T13:39:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6994375741892740000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oqpzlp72mf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714402000,"log":"[2026-02-17T14:35:53Z INFO tonic::transport::server] grpc querier disconnected client_id=8htcpmrs","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gxkanbv26a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714403000,"log":"[2026-02-17T14:25:32Z DEBUG sqlx::query] SELECT FROM streams returned 34 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hl63dshw3h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714404000,"log":"[2026-02-17T13:00:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xefqylos47","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714405000,"log":"[2026-02-17T12:40:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-58bo9dllio","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714406000,"log":"[2026-02-17T12:47:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-us6szxkszy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714407000,"log":"[2026-02-17T15:13:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2232","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4bw8spluui","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714408000,"log":"[2026-02-17T13:23:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1243876362949445400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yynzcfnhgc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714409000,"log":"[2026-02-17T13:59:00Z INFO actix_web::middleware::logger] POST /api/_search 204 337ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-msdcdzjvzo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714410000,"log":"[2026-02-17T14:50:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jj4siz625s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714411000,"log":"[2026-02-17T15:01:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cnwz4v7qem","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714412000,"log":"[2026-02-17T14:49:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ccn8ktfp0v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714413000,"log":"[2026-02-17T14:14:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 461 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3f2p71r3x2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714414000,"log":"[2026-02-17T14:06:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 627 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oat77ust4f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714415000,"log":"[2026-02-17T13:12:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9818298811982672000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ay0cths8s3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714416000,"log":"[2026-02-17T14:46:49Z INFO tonic::transport::server] grpc router request_received client_id=w6n3ftd9","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a0v3b1dzuv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714417000,"log":"[2026-02-17T13:43:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 51ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ars1pwht6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714418000,"log":"[2026-02-17T15:04:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s0apiekmio","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714419000,"log":"[2026-02-17T14:47:22Z INFO tonic::transport::server] grpc ingester connected client_id=qpmp3n5r","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fv9r5ft57i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714420000,"log":"[2026-02-17T14:20:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 322 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-us9l9mpvrh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714421000,"log":"[2026-02-17T12:48:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aib2so41oi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714422000,"log":"[2026-02-17T13:51:05Z INFO actix_web::middleware::logger] GET /api/_search 401 132ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kgv29kwxfu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714423000,"log":"[2026-02-17T14:21:43Z DEBUG sqlx::query] INSERT FROM users returned 785 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ncogdbq8pt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714424000,"log":"[2026-02-17T12:45:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xt4zxnne41","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714425000,"log":"[2026-02-17T13:42:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 97ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o26q0f0hbw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714426000,"log":"[2026-02-17T13:40:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7575329396759958000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-79x9r2hd2u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714427000,"log":"[2026-02-17T14:32:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=vs2hom3p","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4xek6m66jq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714428000,"log":"[2026-02-17T14:13:18Z DEBUG sqlx::query] INSERT FROM users returned 300 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u1jfyz1fy0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714429000,"log":"[2026-02-17T13:42:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 353ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pcyuoci9pg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714430000,"log":"[2026-02-17T14:51:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jxx8h4dr17","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714431000,"log":"[2026-02-17T12:44:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-udri2pzyiy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714432000,"log":"[2026-02-17T12:35:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zph7aibs5e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714433000,"log":"[2026-02-17T14:21:55Z DEBUG sqlx::query] DELETE FROM alerts returned 51 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tf209q2cjs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714434000,"log":"[2026-02-17T13:18:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/976373827114043100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lgqwbk1lu5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714435000,"log":"[2026-02-17T13:40:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2247036911381166000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r61hls7uyb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714436000,"log":"[2026-02-17T14:24:53Z DEBUG sqlx::query] INSERT FROM users returned 466 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ldyqsjrfjg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714437000,"log":"[2026-02-17T14:45:53Z INFO tonic::transport::server] grpc querier disconnected client_id=gmwqk1ib","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6cp6y158k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714438000,"log":"[2026-02-17T14:47:54Z INFO tonic::transport::server] grpc router request_received client_id=6z1jszhn","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r15j61kd4e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714439000,"log":"[2026-02-17T14:16:58Z DEBUG sqlx::query] INSERT FROM users returned 685 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-39chefzta5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714440000,"log":"[2026-02-17T13:46:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 372ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vksyddlmcr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714441000,"log":"[2026-02-17T13:54:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 49ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-br351t19lu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714442000,"log":"[2026-02-17T13:12:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2783944105136715300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-64etjkfjn5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714443000,"log":"[2026-02-17T14:29:15Z DEBUG sqlx::query] DELETE FROM alerts returned 979 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8gyhj4du14","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714444000,"log":"[2026-02-17T14:45:22Z INFO tonic::transport::server] grpc ingester connected client_id=8z29nuj4","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t4a8425ejs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714445000,"log":"[2026-02-17T14:22:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 822 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uh2jknxpwa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714446000,"log":"[2026-02-17T12:57:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n84vd6tcph","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714447000,"log":"[2026-02-17T13:38:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4789352746250375000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5jumk5f15p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714448000,"log":"[2026-02-17T14:30:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 353 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jdnfv5jso3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714449000,"log":"[2026-02-17T12:56:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hcjolsp7pp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714450000,"log":"[2026-02-17T15:13:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7510","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fa1srglfwj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714451000,"log":"[2026-02-17T12:35:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kpamemsgzn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714452000,"log":"[2026-02-17T15:01:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vwlkdwoj7c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714453000,"log":"[2026-02-17T15:17:43Z TRACE cache::redis] cache miss key=org:config:9950","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jpg0377c94","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714454000,"log":"[2026-02-17T14:59:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bsxmw8qnyo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714455000,"log":"[2026-02-17T13:18:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9053752266586914000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dt3y0hxoz7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714456000,"log":"[2026-02-17T13:02:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cm1rmod8nm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714457000,"log":"[2026-02-17T13:14:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2258980300478785500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ghdpv5xnvb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714458000,"log":"[2026-02-17T14:50:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yux2nrj3j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714459000,"log":"[2026-02-17T14:13:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 647 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uw41uggcc4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714460000,"log":"[2026-02-17T13:41:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6458391407915252000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m57q7jk1xr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714461000,"log":"[2026-02-17T15:01:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qx0ygdvwfa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714462000,"log":"[2026-02-17T13:55:20Z INFO actix_web::middleware::logger] POST /api/_search 500 51ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e79n23h0fx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714463000,"log":"[2026-02-17T13:22:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/845450472316808800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gemrk32nhe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714464000,"log":"[2026-02-17T14:32:24Z INFO tonic::transport::server] grpc router request_received client_id=igyj3zi7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3oqqo3w8g0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714465000,"log":"[2026-02-17T14:56:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-or0dc53ugt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714466000,"log":"[2026-02-17T13:15:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9319368102296973000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lxqi0z8llw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714467000,"log":"[2026-02-17T14:34:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=52jun4m2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ssp24h7krz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714468000,"log":"[2026-02-17T14:13:10Z DEBUG sqlx::query] DELETE FROM alerts returned 666 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0qsj7fs8vn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714469000,"log":"[2026-02-17T13:15:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1537490704073361000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cjr2l9eev8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714470000,"log":"[2026-02-17T13:53:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 23ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ynjn89x1c6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714471000,"log":"[2026-02-17T14:08:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 365 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-axwvg0n20e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714472000,"log":"[2026-02-17T12:53:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-twnd8kzusv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714473000,"log":"[2026-02-17T13:20:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8148063810038741000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-duxg8hyzy4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714474000,"log":"[2026-02-17T13:31:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7483116857936836000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pjs3n0u9tq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714475000,"log":"[2026-02-17T14:03:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 224ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-58lhehm3kt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714476000,"log":"[2026-02-17T14:14:42Z DEBUG sqlx::query] SELECT FROM streams returned 933 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ay5g2ayx4m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714477000,"log":"[2026-02-17T15:19:21Z TRACE cache::redis] cache set key=auth:token:2886","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-22sh9e8e0r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714478000,"log":"[2026-02-17T14:59:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8x25qyrek6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714479000,"log":"[2026-02-17T13:18:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4192697205107791400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yeay42qykr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714480000,"log":"[2026-02-17T13:20:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7827133765276261000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kxqjrtnxwe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714481000,"log":"[2026-02-17T15:09:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=3253","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rdryr4u0zh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714482000,"log":"[2026-02-17T13:52:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 288ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qiuvhc0cne","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714483000,"log":"[2026-02-17T14:37:47Z INFO tonic::transport::server] grpc ingester connected client_id=af0t46a9","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6pg2wfx6d1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714484000,"log":"[2026-02-17T13:18:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/630741572831599000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-789ncxs55g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714485000,"log":"[2026-02-17T12:52:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-90cakgctg2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714486000,"log":"[2026-02-17T15:16:47Z TRACE cache::redis] cache expire key=user:session:2260","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6uo4z8eor8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714487000,"log":"[2026-02-17T12:41:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n98237i7ty","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714488000,"log":"[2026-02-17T13:46:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 203ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ghtrb9dlvj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714489000,"log":"[2026-02-17T14:43:59Z INFO tonic::transport::server] grpc router request_received client_id=k86o3i49","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1pzkmcli31","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714490000,"log":"[2026-02-17T15:18:44Z TRACE cache::redis] cache get key=stream:schema:6662","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y603yrsn0i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714491000,"log":"[2026-02-17T14:44:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=izp24kfd","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fq68632lep","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714492000,"log":"[2026-02-17T14:21:00Z DEBUG sqlx::query] DELETE FROM alerts returned 608 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1u7rr30vsd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714493000,"log":"[2026-02-17T14:08:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 385 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7kw81wgwo0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714494000,"log":"[2026-02-17T14:22:15Z DEBUG sqlx::query] DELETE FROM alerts returned 276 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qfrh8ivdw4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714495000,"log":"[2026-02-17T15:18:53Z TRACE cache::redis] cache expire key=org:config:1294","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ff87z522m9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714496000,"log":"[2026-02-17T14:08:55Z DEBUG sqlx::query] DELETE FROM alerts returned 364 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lrss19jvvi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714497000,"log":"[2026-02-17T14:19:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 814 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wnlio5157p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714498000,"log":"[2026-02-17T12:58:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-18dpaiitdk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714499000,"log":"[2026-02-17T14:39:44Z INFO tonic::transport::server] grpc router request_received client_id=t740wj6o","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ls26osydaz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714500000,"log":"[2026-02-17T12:49:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ljjkw1r0wx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714501000,"log":"[2026-02-17T15:16:46Z TRACE cache::redis] cache delete key=auth:token:1016","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3fvcpe9mw7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714502000,"log":"[2026-02-17T13:40:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8951345776977715000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-figul8b2zg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714503000,"log":"[2026-02-17T15:07:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1148","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mgwl1dwrur","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714504000,"log":"[2026-02-17T14:34:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=fiet0o3i","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xds25h00jr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714505000,"log":"[2026-02-17T15:06:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=8360","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6a0h8hdvtj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714506000,"log":"[2026-02-17T15:21:41Z TRACE cache::redis] cache expire key=auth:token:3563","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r98b8tul2y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714507000,"log":"[2026-02-17T15:09:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1721","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bu8g5ro74x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714508000,"log":"[2026-02-17T14:32:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=377jwgw7","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-891xqw060n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714509000,"log":"[2026-02-17T13:16:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4641417560506384000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2xedoezzb5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714510000,"log":"[2026-02-17T12:45:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w0ts9f834j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714511000,"log":"[2026-02-17T13:49:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 321ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3jhyznmytp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714512000,"log":"[2026-02-17T14:58:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lg6aypf8yd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714513000,"log":"[2026-02-17T12:38:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zsizoy6121","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714514000,"log":"[2026-02-17T12:46:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7117mf3s7v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714515000,"log":"[2026-02-17T14:17:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 709 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bxnh69k8ev","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714516000,"log":"[2026-02-17T12:51:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q0rzn7wczg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714517000,"log":"[2026-02-17T12:58:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zqfba1i4xj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714518000,"log":"[2026-02-17T12:37:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rbngh12nlj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714519000,"log":"[2026-02-17T15:18:29Z TRACE cache::redis] cache expire key=stream:schema:7314","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kfsfjwugy6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714520000,"log":"[2026-02-17T13:55:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 353ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i3bdf43s76","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714521000,"log":"[2026-02-17T13:45:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 197ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zixekb0q0w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714522000,"log":"[2026-02-17T14:11:38Z DEBUG sqlx::query] INSERT FROM users returned 11 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kaqhsykyk4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714523000,"log":"[2026-02-17T14:33:37Z INFO tonic::transport::server] grpc ingester connected client_id=m9z304tx","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a7wk5ody51","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714524000,"log":"[2026-02-17T13:15:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8455962754529394000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ifxiq09vyy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714525000,"log":"[2026-02-17T14:27:32Z DEBUG sqlx::query] SELECT FROM streams returned 832 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fdef02965d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714526000,"log":"[2026-02-17T13:58:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 318ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h3zvrbq582","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714527000,"log":"[2026-02-17T13:57:45Z INFO actix_web::middleware::logger] GET /api/_search 503 414ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u3ljyaozgb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714528000,"log":"[2026-02-17T13:02:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s2bgk82vb4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714529000,"log":"[2026-02-17T13:55:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 462ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y00u2rbmtp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714530000,"log":"[2026-02-17T14:10:58Z DEBUG sqlx::query] INSERT FROM users returned 166 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k2m1holos2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714531000,"log":"[2026-02-17T12:46:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-djnl3f5ii9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714532000,"log":"[2026-02-17T12:48:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1frfu9ofvl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714533000,"log":"[2026-02-17T15:19:41Z TRACE cache::redis] cache expire key=auth:token:9733","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-28use6rspx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714534000,"log":"[2026-02-17T13:47:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 369ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mcmp9jfizt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714535000,"log":"[2026-02-17T15:21:33Z TRACE cache::redis] cache set key=org:config:6108","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e28p0af4t0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714536000,"log":"[2026-02-17T13:27:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5196101995566922000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-210o9czg2u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714537000,"log":"[2026-02-17T14:33:49Z INFO tonic::transport::server] grpc router request_received client_id=3ftu9ut7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-meqzk7ut9x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714538000,"log":"[2026-02-17T13:46:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 143ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4449xntz9n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714539000,"log":"[2026-02-17T15:02:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vpv93hhmo3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714540000,"log":"[2026-02-17T14:52:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pz15xybw4g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714541000,"log":"[2026-02-17T14:30:08Z DEBUG sqlx::query] INSERT FROM users returned 887 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8094kep2np","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714542000,"log":"[2026-02-17T14:05:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 215ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-90owscy81e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714543000,"log":"[2026-02-17T13:09:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6847039335447709000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3laosvkp3i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714544000,"log":"[2026-02-17T13:50:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 19ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l9egvlrfpa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714545000,"log":"[2026-02-17T13:37:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1730650510000908300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fluhyqhhbb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714546000,"log":"[2026-02-17T14:01:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f26jhv6c53","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714547000,"log":"[2026-02-17T13:38:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7765810889026326000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vw9g301ndf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714548000,"log":"[2026-02-17T12:48:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kizmrmth5h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714549000,"log":"[2026-02-17T13:20:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1702118009361843700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o60il8uygf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714550000,"log":"[2026-02-17T13:17:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7789286500140101000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u3mscdcdyz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714551000,"log":"[2026-02-17T12:53:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2hhgrudve5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714552000,"log":"[2026-02-17T14:29:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 669 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2uci4u9rik","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714553000,"log":"[2026-02-17T14:12:37Z DEBUG sqlx::query] SELECT FROM streams returned 379 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i8i1o3pw1t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714554000,"log":"[2026-02-17T14:33:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=z1o8kdrx","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-letqaqaafv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714555000,"log":"[2026-02-17T14:13:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 784 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8q2rntolss","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714556000,"log":"[2026-02-17T13:25:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8816485914452325000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8c670og2u4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714557000,"log":"[2026-02-17T14:31:15Z DEBUG sqlx::query] DELETE FROM alerts returned 606 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-76lly67owz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714558000,"log":"[2026-02-17T13:43:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 276ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ar9nwg9gb3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714559000,"log":"[2026-02-17T14:56:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o2jescvnm5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714560000,"log":"[2026-02-17T14:33:22Z INFO tonic::transport::server] grpc ingester connected client_id=as3w5kyp","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v37kvd0iqg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714561000,"log":"[2026-02-17T14:58:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-34knzfq0ab","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714562000,"log":"[2026-02-17T13:17:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2291146459752522200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q5pw18i9pw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714563000,"log":"[2026-02-17T12:49:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i1ccsyjy75","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714564000,"log":"[2026-02-17T15:00:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s2p0gnu12t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714565000,"log":"[2026-02-17T14:03:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 196ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2kkfanu28e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714566000,"log":"[2026-02-17T13:11:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/645204140955739600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p237vs2t3y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714567000,"log":"[2026-02-17T12:59:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-njgy5jr2bi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714568000,"log":"[2026-02-17T15:12:55Z DEBUG metrics::reporter] recorded metric active_connections value=4078","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aubgr4exlv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714569000,"log":"[2026-02-17T14:04:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 235ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y07uoffzug","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714570000,"log":"[2026-02-17T12:36:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qipzxhs13u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714571000,"log":"[2026-02-17T15:15:22Z TRACE cache::redis] cache delete key=user:session:4916","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-boumw8vtz1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714572000,"log":"[2026-02-17T13:02:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6bir6vrf52","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714573000,"log":"[2026-02-17T14:04:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 219ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fn6kysq7jn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714574000,"log":"[2026-02-17T14:30:27Z DEBUG sqlx::query] SELECT FROM streams returned 738 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g75cosaxql","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714575000,"log":"[2026-02-17T12:47:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2i1kseb9m9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714576000,"log":"[2026-02-17T15:17:21Z TRACE cache::redis] cache set key=auth:token:1845","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0cewhzzgtx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714577000,"log":"[2026-02-17T15:15:03Z TRACE cache::redis] cache set key=org:config:3401","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wcg7cfd2er","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714578000,"log":"[2026-02-17T13:33:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3398104536278596600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-60bo4k6h6f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714579000,"log":"[2026-02-17T13:43:40Z INFO actix_web::middleware::logger] POST /api/_search 200 160ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e0witq3g77","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714580000,"log":"[2026-02-17T14:57:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y5aujkd475","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714581000,"log":"[2026-02-17T14:29:25Z DEBUG sqlx::query] DELETE FROM alerts returned 23 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rtz5f72zog","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714582000,"log":"[2026-02-17T14:32:39Z INFO tonic::transport::server] grpc router request_received client_id=p5fdm8kl","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sx0ojvnvol","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714583000,"log":"[2026-02-17T14:28:47Z DEBUG sqlx::query] SELECT FROM streams returned 646 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d0c9j43ku3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714584000,"log":"[2026-02-17T14:58:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i19mi6ixk9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714585000,"log":"[2026-02-17T14:35:49Z INFO tonic::transport::server] grpc router request_received client_id=lptpmc7s","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rn2s5z6qa5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714586000,"log":"[2026-02-17T13:42:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 352ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ktwm0v00ue","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714587000,"log":"[2026-02-17T13:38:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6208227570707414000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7eh703lj0w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714588000,"log":"[2026-02-17T13:13:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1777019270569200000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jbvwf5r6bx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714589000,"log":"[2026-02-17T13:50:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 472ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2l2dooni4d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714590000,"log":"[2026-02-17T13:44:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 59ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y72ymk5s1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714591000,"log":"[2026-02-17T14:12:20Z DEBUG sqlx::query] DELETE FROM alerts returned 185 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vnck41vrzw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714592000,"log":"[2026-02-17T14:20:02Z DEBUG sqlx::query] SELECT FROM streams returned 409 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5rasvas82k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714593000,"log":"[2026-02-17T13:12:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8923593478515708000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-epttp7cmuf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714594000,"log":"[2026-02-17T15:09:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=4097","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4nriuwnbdu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714595000,"log":"[2026-02-17T12:36:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y4mwql42p8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714596000,"log":"[2026-02-17T12:55:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-diukecqf0f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714597000,"log":"[2026-02-17T13:49:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 375ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z3x41mqvgm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714598000,"log":"[2026-02-17T13:50:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 227ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6ymwq8rtng","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714599000,"log":"[2026-02-17T13:30:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4623265339052032000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rlot2c0dt0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714600000,"log":"[2026-02-17T14:02:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 467ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j3wes84k9w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714601000,"log":"[2026-02-17T15:12:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=139","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ywv8z1sn6o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714602000,"log":"[2026-02-17T14:32:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rf03gwne","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8ci83hkaj1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714603000,"log":"[2026-02-17T14:55:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4ndv5650q8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714604000,"log":"[2026-02-17T14:13:50Z DEBUG sqlx::query] DELETE FROM alerts returned 606 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-emvb04spdo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714605000,"log":"[2026-02-17T14:59:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xysd6l8ykq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714606000,"log":"[2026-02-17T12:47:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7sboi635rf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714607000,"log":"[2026-02-17T14:19:18Z DEBUG sqlx::query] INSERT FROM users returned 279 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-70wo0gzaap","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714608000,"log":"[2026-02-17T14:40:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=t5y4yyfe","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b4sh0gl7gq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714609000,"log":"[2026-02-17T14:58:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-asu4wh6qv3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714610000,"log":"[2026-02-17T14:20:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 323 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dpomu9cunr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714611000,"log":"[2026-02-17T13:04:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3jicjer0c9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714612000,"log":"[2026-02-17T13:41:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8906978512304333000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fs0beyie90","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714613000,"log":"[2026-02-17T13:09:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8057933399387041000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qa9463ayp0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714614000,"log":"[2026-02-17T14:29:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 124 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xzlvahhott","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714615000,"log":"[2026-02-17T13:21:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/790320210362165400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uqrrtdoggm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714616000,"log":"[2026-02-17T14:35:28Z INFO tonic::transport::server] grpc querier disconnected client_id=33edgiip","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k286x2jq01","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714617000,"log":"[2026-02-17T13:57:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 104ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-az5gg6atlp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714618000,"log":"[2026-02-17T14:42:33Z INFO tonic::transport::server] grpc querier disconnected client_id=s6cti948","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fcipc7y6b0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714619000,"log":"[2026-02-17T14:24:20Z DEBUG sqlx::query] DELETE FROM alerts returned 824 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0csrm7z8rl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714620000,"log":"[2026-02-17T14:25:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 520 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bif58ehscz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714621000,"log":"[2026-02-17T13:27:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3025230295266021400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4dxc20u39y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714622000,"log":"[2026-02-17T12:47:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b7hzqac0bh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714623000,"log":"[2026-02-17T13:07:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x3usk0d8wi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714624000,"log":"[2026-02-17T14:00:05Z INFO actix_web::middleware::logger] GET /api/_search 401 147ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-31os3pmuuo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714625000,"log":"[2026-02-17T13:07:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vm4ltrfik","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714626000,"log":"[2026-02-17T12:40:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tlb8vzp59z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714627000,"log":"[2026-02-17T14:18:40Z DEBUG sqlx::query] DELETE FROM alerts returned 150 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fndxxdcpn2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714628000,"log":"[2026-02-17T13:30:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4392489344167134000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-purfeqs4s5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714629000,"log":"[2026-02-17T13:31:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9106540716400882000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ag53hwkfxi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714630000,"log":"[2026-02-17T13:26:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9628289116496490000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hrllcv9yit","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714631000,"log":"[2026-02-17T12:40:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c9bp7qleow","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714632000,"log":"[2026-02-17T14:00:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 241ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c7zrkzidwy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714633000,"log":"[2026-02-17T13:37:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8043412070646416000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8tg8lnx7u4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714634000,"log":"[2026-02-17T13:26:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3245434280660251000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-45r5cjd42t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714635000,"log":"[2026-02-17T14:16:18Z DEBUG sqlx::query] INSERT FROM users returned 16 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uox2i19zgp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714636000,"log":"[2026-02-17T15:12:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2541","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-us3tlx8y96","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714637000,"log":"[2026-02-17T12:50:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gs22gv7iy7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714638000,"log":"[2026-02-17T12:40:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eiau4khuut","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714639000,"log":"[2026-02-17T12:56:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-swifpdx6zk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714640000,"log":"[2026-02-17T12:46:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hrjo7fzk8z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714641000,"log":"[2026-02-17T13:34:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8500263179246959000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sru6gdxry6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714642000,"log":"[2026-02-17T14:06:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 190ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tupj6e300q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714643000,"log":"[2026-02-17T14:59:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-knr2pkaafs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714644000,"log":"[2026-02-17T14:30:05Z DEBUG sqlx::query] DELETE FROM alerts returned 619 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3gdgr0tkjx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714645000,"log":"[2026-02-17T13:19:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7169231688199424000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t549qz8c7f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714646000,"log":"[2026-02-17T13:12:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9091570990388380000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h7qv6m248e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714647000,"log":"[2026-02-17T12:54:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2h7gt8jgfh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714648000,"log":"[2026-02-17T14:58:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-znlefzp56t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714649000,"log":"[2026-02-17T14:08:22Z DEBUG sqlx::query] SELECT FROM streams returned 296 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2fa7ju6tw8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714650000,"log":"[2026-02-17T13:31:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5094694533956339000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gwgwoh2m4r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714651000,"log":"[2026-02-17T15:21:30Z TRACE cache::redis] cache hit key=query:result:9546","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9yhqwyqlvl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714652000,"log":"[2026-02-17T13:32:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1033715842000344400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-32hrqslmcr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714653000,"log":"[2026-02-17T12:56:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-scylp9zsly","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714654000,"log":"[2026-02-17T15:06:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3907","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hpjo5eomyp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714655000,"log":"[2026-02-17T13:09:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4888536742870982000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dx01rptczv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714656000,"log":"[2026-02-17T13:20:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3044583913013412400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-116nffw0ev","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714657000,"log":"[2026-02-17T12:40:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5w0ia758m7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714658000,"log":"[2026-02-17T13:42:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 408ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xbn6k42g4m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714659000,"log":"[2026-02-17T15:17:50Z TRACE cache::redis] cache get key=query:result:4073","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9v3j37kscb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714660000,"log":"[2026-02-17T12:41:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2xzriyde2t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714661000,"log":"[2026-02-17T14:19:32Z DEBUG sqlx::query] SELECT FROM streams returned 820 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qmaq34v4fk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714662000,"log":"[2026-02-17T13:46:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 232ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2qu81g9e10","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714663000,"log":"[2026-02-17T14:41:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=x75vfobx","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kiq6f3kk5m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714664000,"log":"[2026-02-17T13:55:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 61ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3bdnrdpeot","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714665000,"log":"[2026-02-17T15:04:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zo1fx1bhwy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714666000,"log":"[2026-02-17T14:49:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-45ibdw0mqh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714667000,"log":"[2026-02-17T14:13:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 891 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u9ziegdcil","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714668000,"log":"[2026-02-17T13:24:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1223549644249240000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5dpxgwto9w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714669000,"log":"[2026-02-17T13:35:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5962395293052260000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zgjz4m1rb6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714670000,"log":"[2026-02-17T15:09:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7128","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o94roq3198","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714671000,"log":"[2026-02-17T14:43:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=8knvtvol","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kuc07cjpw2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714672000,"log":"[2026-02-17T14:57:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5wk193kizc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714673000,"log":"[2026-02-17T14:05:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 26ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jt2alst2pb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714674000,"log":"[2026-02-17T14:10:43Z DEBUG sqlx::query] INSERT FROM users returned 780 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ihtukzg7yl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714675000,"log":"[2026-02-17T12:59:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o53py6qbue","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714676000,"log":"[2026-02-17T13:46:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 455ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8lrgy037la","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714677000,"log":"[2026-02-17T12:58:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u43ri2ae0p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714678000,"log":"[2026-02-17T14:28:55Z DEBUG sqlx::query] DELETE FROM alerts returned 834 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7dgjur5uy5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714679000,"log":"[2026-02-17T14:05:40Z INFO actix_web::middleware::logger] POST /api/_search 404 242ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-994j4nghym","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714680000,"log":"[2026-02-17T14:36:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=0jf1fs5f","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lai9e59r9h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714681000,"log":"[2026-02-17T15:11:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3029","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yeefhb6ouk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714682000,"log":"[2026-02-17T14:03:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 327ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y65ui0umms","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714683000,"log":"[2026-02-17T13:59:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 482ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lblqdrnj3i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714684000,"log":"[2026-02-17T13:19:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8181860825228012000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pg0nxj4te5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714685000,"log":"[2026-02-17T13:35:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7598443347133739000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sp6i0z1of6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714686000,"log":"[2026-02-17T13:49:40Z INFO actix_web::middleware::logger] POST /api/_search 200 440ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e41ej7d1xg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714687000,"log":"[2026-02-17T13:39:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6006087937709179000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o20ilp0lu4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714688000,"log":"[2026-02-17T14:52:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tg1tgyld8x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714689000,"log":"[2026-02-17T14:33:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ktteylia","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w1xgz2xmx5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714690000,"log":"[2026-02-17T13:11:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3665669102055855000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vw4ipwqwde","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714691000,"log":"[2026-02-17T13:06:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ucuvtgdqw3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714692000,"log":"[2026-02-17T12:38:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rw67w8xbx0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714693000,"log":"[2026-02-17T12:38:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n9t7oq7a8i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714694000,"log":"[2026-02-17T13:41:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9016504962246798000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7sqlen431x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714695000,"log":"[2026-02-17T12:52:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5hcgujoo20","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714696000,"log":"[2026-02-17T12:37:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i1ukgkdyts","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714697000,"log":"[2026-02-17T15:10:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4926","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kwd10spat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714698000,"log":"[2026-02-17T14:46:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=wfix8r1l","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d9czq7ua48","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714699000,"log":"[2026-02-17T15:13:33Z TRACE cache::redis] cache set key=org:config:5502","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s0sxh5vfen","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714700000,"log":"[2026-02-17T15:08:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=3095","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jh74ub8cs2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714701000,"log":"[2026-02-17T13:49:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 111ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-04f3v0w8ur","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714702000,"log":"[2026-02-17T13:17:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7737546525486792000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ii0xx9s3n9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714703000,"log":"[2026-02-17T14:12:52Z DEBUG sqlx::query] SELECT FROM streams returned 75 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rqddtus9sv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714704000,"log":"[2026-02-17T14:07:30Z DEBUG sqlx::query] DELETE FROM alerts returned 285 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wjk9upvmr9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714705000,"log":"[2026-02-17T14:04:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 197ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i9gcm8bbd8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714706000,"log":"[2026-02-17T13:11:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1375382538074094600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mzzjossjdj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714707000,"log":"[2026-02-17T12:54:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-axqvplbkgs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714708000,"log":"[2026-02-17T14:43:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=5daz8c7p","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ngc6jonys1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714709000,"log":"[2026-02-17T13:31:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5471212879802055000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wong3dx1wy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714710000,"log":"[2026-02-17T14:43:03Z INFO tonic::transport::server] grpc querier disconnected client_id=n8560fei","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tg4finxotp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714711000,"log":"[2026-02-17T14:27:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 659 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7v8qbeekmo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714712000,"log":"[2026-02-17T14:16:47Z DEBUG sqlx::query] SELECT FROM streams returned 90 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-99amq4up4g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714713000,"log":"[2026-02-17T13:18:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4104979319568242700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ae8ul0ifz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714714000,"log":"[2026-02-17T14:38:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=raznnfaw","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i8k5bnmmuv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714715000,"log":"[2026-02-17T12:41:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-isf6tx1tum","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714716000,"log":"[2026-02-17T13:24:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6732681296461813000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-exbpq4gnh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714717000,"log":"[2026-02-17T14:36:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=x7c3sp99","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1lofkrz28r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714718000,"log":"[2026-02-17T15:03:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kqhtwoznrg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714719000,"log":"[2026-02-17T12:41:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-taroyq3zsq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714720000,"log":"[2026-02-17T13:21:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/850837137520277200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-354tmuqr4i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714721000,"log":"[2026-02-17T13:57:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 243ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2i4abdys1t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714722000,"log":"[2026-02-17T13:31:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8527703407440044000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0cdlzmxanq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714723000,"log":"[2026-02-17T12:41:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tor2qlqinp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714724000,"log":"[2026-02-17T15:12:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=259","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6orjc6z6ck","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714725000,"log":"[2026-02-17T14:10:13Z DEBUG sqlx::query] INSERT FROM users returned 117 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dmpoha1vnp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714726000,"log":"[2026-02-17T13:55:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 394ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tlm18u8iy9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714727000,"log":"[2026-02-17T14:28:02Z DEBUG sqlx::query] SELECT FROM streams returned 909 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l0hk0m6w9d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714728000,"log":"[2026-02-17T12:54:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1s6l5k0l7e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714729000,"log":"[2026-02-17T14:10:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 53 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jd5csqr203","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714730000,"log":"[2026-02-17T14:15:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 658 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1jeun8zy8k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714731000,"log":"[2026-02-17T13:29:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9948828143095708000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-14muvn2n81","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714732000,"log":"[2026-02-17T12:46:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-db5ssj7hyz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714733000,"log":"[2026-02-17T12:59:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3dd6vhq9ok","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714734000,"log":"[2026-02-17T14:41:14Z INFO tonic::transport::server] grpc router request_received client_id=k7uxezrx","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ohmaiwluk5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714735000,"log":"[2026-02-17T13:41:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7999938627983428000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rl81zgpdkr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714736000,"log":"[2026-02-17T14:53:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mxghcdc9n4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714737000,"log":"[2026-02-17T13:08:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b60csti0ln","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714738000,"log":"[2026-02-17T14:23:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 680 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n5jo8bo89t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714739000,"log":"[2026-02-17T12:41:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cepu3fnxp4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714740000,"log":"[2026-02-17T14:26:42Z DEBUG sqlx::query] SELECT FROM streams returned 756 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ti3izaipyj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714741000,"log":"[2026-02-17T13:18:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3821452792531552000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9b6l7a7ian","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714742000,"log":"[2026-02-17T14:28:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 593 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y59zdk7brw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714743000,"log":"[2026-02-17T14:45:37Z INFO tonic::transport::server] grpc ingester connected client_id=3t28jnar","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w8i4afwmoo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714744000,"log":"[2026-02-17T13:48:25Z INFO actix_web::middleware::logger] GET /api/_search 404 182ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j6ps2mtu6c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714745000,"log":"[2026-02-17T14:39:27Z INFO tonic::transport::server] grpc ingester connected client_id=nylzv08w","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7rfkr5jupi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714746000,"log":"[2026-02-17T14:48:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-94vkdzdebv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714747000,"log":"[2026-02-17T14:56:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v9e4vncr34","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714748000,"log":"[2026-02-17T13:37:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/582307972924012200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6gr1vx0tnm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714749000,"log":"[2026-02-17T14:19:30Z DEBUG sqlx::query] DELETE FROM alerts returned 212 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-11wqorx116","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714750000,"log":"[2026-02-17T13:55:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 181ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x2fsb631ys","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714751000,"log":"[2026-02-17T12:45:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-imb8a1ebnl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714752000,"log":"[2026-02-17T14:41:49Z INFO tonic::transport::server] grpc router request_received client_id=wk4apv78","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xdcxgc1c9i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714753000,"log":"[2026-02-17T15:08:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3850","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k5zewxhyqi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714754000,"log":"[2026-02-17T13:37:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4213559598271896600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2nv71zlwxs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714755000,"log":"[2026-02-17T14:18:30Z DEBUG sqlx::query] DELETE FROM alerts returned 292 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f5goimvspu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714756000,"log":"[2026-02-17T13:36:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9490684517339509000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i9aqzu4fkc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714757000,"log":"[2026-02-17T13:55:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 320ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n0i8fn9aj3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714758000,"log":"[2026-02-17T14:56:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vcmiqd0vz2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714759000,"log":"[2026-02-17T15:14:33Z TRACE cache::redis] cache set key=org:config:2675","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9impnlo92b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714760000,"log":"[2026-02-17T13:26:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5311731900530896000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-44hcqcm9jo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714761000,"log":"[2026-02-17T14:36:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rxm0bjla","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ax4ez0m8yv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714762000,"log":"[2026-02-17T14:25:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 460 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xw24n7sxex","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714763000,"log":"[2026-02-17T13:48:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 139ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lkcsk3jqa4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714764000,"log":"[2026-02-17T14:39:23Z INFO tonic::transport::server] grpc querier disconnected client_id=mmrm7ib6","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q723oy8e4y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714765000,"log":"[2026-02-17T13:33:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6456688324219832000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wqszcfinun","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714766000,"log":"[2026-02-17T14:46:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=asyfn5xo","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3tdkcp3spz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714767000,"log":"[2026-02-17T14:19:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 350 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tv3ppvujbp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714768000,"log":"[2026-02-17T14:10:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 485 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s5yz6p0z9a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714769000,"log":"[2026-02-17T13:28:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7890310672053813000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1mdu76al65","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714770000,"log":"[2026-02-17T15:14:51Z TRACE cache::redis] cache set key=auth:token:1617","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wt2i9kbvx2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714771000,"log":"[2026-02-17T12:43:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ojszoehllf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714772000,"log":"[2026-02-17T15:07:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=3295","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b37qrb0ygw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714773000,"log":"[2026-02-17T13:47:25Z INFO actix_web::middleware::logger] GET /api/_search 200 296ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c9u4x5fr6o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714774000,"log":"[2026-02-17T13:09:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1561615749206846200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q3knghc7f7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714775000,"log":"[2026-02-17T15:12:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9822","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7p2q1btqh6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714776000,"log":"[2026-02-17T13:26:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8632989002482985000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hlf65bly1w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714777000,"log":"[2026-02-17T14:28:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 476 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i0m16mypoa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714778000,"log":"[2026-02-17T13:08:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3730843507034296000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-19q17r3fbb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714779000,"log":"[2026-02-17T14:53:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-16plt38l25","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714780000,"log":"[2026-02-17T14:12:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 978 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hvvivbkhaa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714781000,"log":"[2026-02-17T13:36:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6529644468111031000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oint5twoe1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714782000,"log":"[2026-02-17T12:55:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-srw4j68cko","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714783000,"log":"[2026-02-17T12:47:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6gfdv8999a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714784000,"log":"[2026-02-17T12:40:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lelkricwaf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714785000,"log":"[2026-02-17T13:21:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2162669031118040800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vyqss8o4pc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714786000,"log":"[2026-02-17T14:24:30Z DEBUG sqlx::query] DELETE FROM alerts returned 259 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tpot3uwyrt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714787000,"log":"[2026-02-17T12:55:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ojt8nnnqow","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714788000,"log":"[2026-02-17T13:57:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 454ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q31ayie5ny","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714789000,"log":"[2026-02-17T14:43:47Z INFO tonic::transport::server] grpc ingester connected client_id=a406rsj9","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-st33wyzaop","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714790000,"log":"[2026-02-17T14:26:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 524 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0e6iqp1oyp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714791000,"log":"[2026-02-17T13:31:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9537657531540531000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h0zbzghywe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714792000,"log":"[2026-02-17T12:37:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x8ibj038wv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714793000,"log":"[2026-02-17T14:54:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ggfycgtt3b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714794000,"log":"[2026-02-17T13:10:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8463232864822952000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7tq0ya189r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714795000,"log":"[2026-02-17T13:37:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8881293945838747000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vfxuzqckga","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714796000,"log":"[2026-02-17T14:26:07Z DEBUG sqlx::query] SELECT FROM streams returned 323 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sphes445c9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714797000,"log":"[2026-02-17T15:09:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2575","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tau4qf77aw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714798000,"log":"[2026-02-17T14:52:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ebljsoi2ac","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714799000,"log":"[2026-02-17T13:49:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q2gp5o5ukj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714800000,"log":"[2026-02-17T12:57:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zixm2a0kg9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714801000,"log":"[2026-02-17T14:33:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=4sdcg327","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mbj3frvzxz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714802000,"log":"[2026-02-17T13:13:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7771080563762482000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f2j5xv4p2o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714803000,"log":"[2026-02-17T13:43:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 468ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tyse4ayung","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714804000,"log":"[2026-02-17T14:57:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lmh7j8658u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714805000,"log":"[2026-02-17T14:38:39Z INFO tonic::transport::server] grpc router request_received client_id=7jxngug2","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-67u7v9d1d2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714806000,"log":"[2026-02-17T12:45:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ghabav6qdc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714807000,"log":"[2026-02-17T13:24:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6269063618260031000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-071wfqkdsm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714808000,"log":"[2026-02-17T14:18:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 193 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u6ba6lw39e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714809000,"log":"[2026-02-17T12:43:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bjsfuh2roc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714810000,"log":"[2026-02-17T13:45:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 469ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tylm7vced1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714811000,"log":"[2026-02-17T13:07:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-93lfnicadv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714812000,"log":"[2026-02-17T12:47:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oxd2bdu7id","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714813000,"log":"[2026-02-17T14:36:28Z INFO tonic::transport::server] grpc querier disconnected client_id=p51bsvrf","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qaj9mlysch","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714814000,"log":"[2026-02-17T12:51:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-259geuz10i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714815000,"log":"[2026-02-17T13:02:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lwys73tds1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714816000,"log":"[2026-02-17T14:24:27Z DEBUG sqlx::query] SELECT FROM streams returned 197 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uvrjlvk89v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714817000,"log":"[2026-02-17T14:53:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zfd9i96pd4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714818000,"log":"[2026-02-17T14:37:09Z INFO tonic::transport::server] grpc router request_received client_id=1n0ssrhb","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sq5lrnu7gi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714819000,"log":"[2026-02-17T14:50:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-scdtbcl71j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714820000,"log":"[2026-02-17T13:07:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7ppg8ji7xo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714821000,"log":"[2026-02-17T14:58:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xf7kozmfjd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714822000,"log":"[2026-02-17T14:26:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 51 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yov8y7r7kk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714823000,"log":"[2026-02-17T14:16:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 396 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dnhdgup6sy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714824000,"log":"[2026-02-17T14:07:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 423 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8srlea1mzb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714825000,"log":"[2026-02-17T13:18:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6446244765158820000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-949yhbkfsh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714826000,"log":"[2026-02-17T15:18:09Z TRACE cache::redis] cache set key=stream:schema:4862","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y1vxsxr7sk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714827000,"log":"[2026-02-17T12:41:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-03pt1vrrpf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714828000,"log":"[2026-02-17T14:24:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 231 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pihl1isk5w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714829000,"log":"[2026-02-17T14:35:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=be3p4brb","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bfdp7rhdwi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714830000,"log":"[2026-02-17T13:13:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3721914396138623500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2lpjwqnnny","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714831000,"log":"[2026-02-17T13:22:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/132516409686065380.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5rk69oucje","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714832000,"log":"[2026-02-17T13:46:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 25ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cb1e29c3rs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714833000,"log":"[2026-02-17T13:58:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 125ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8vekpq0asr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714834000,"log":"[2026-02-17T15:02:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4gcmn4dhj0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714835000,"log":"[2026-02-17T13:21:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4853794832432088000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lzjpsesbea","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714836000,"log":"[2026-02-17T13:49:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 181ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8451w3bxw6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714837000,"log":"[2026-02-17T13:56:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 350ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3bhw488994","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714838000,"log":"[2026-02-17T14:09:18Z DEBUG sqlx::query] INSERT FROM users returned 586 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vklw3rskhw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714839000,"log":"[2026-02-17T14:52:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a7rjanie2n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714840000,"log":"[2026-02-17T13:00:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fmotkk94g6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714841000,"log":"[2026-02-17T13:01:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rx4l5oxzg9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714842000,"log":"[2026-02-17T15:21:36Z TRACE cache::redis] cache hit key=auth:token:4340","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uq6hf0yu7c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714843000,"log":"[2026-02-17T15:05:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=9030","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sy7gc1vgsx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714844000,"log":"[2026-02-17T13:03:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2hlbzwuca9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714845000,"log":"[2026-02-17T14:19:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 215 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v89gsfdq2u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714846000,"log":"[2026-02-17T13:57:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 200ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x4ml2ijwt7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714847000,"log":"[2026-02-17T13:17:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2574480037921023000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dgvj08zna8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714848000,"log":"[2026-02-17T15:01:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iq8dzqe12y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714849000,"log":"[2026-02-17T13:30:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9039474095448139000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-05rq8xfmcj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714850000,"log":"[2026-02-17T14:28:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 323 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qknx6ilvcr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714851000,"log":"[2026-02-17T13:32:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3372602368960371000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6j5qu6d1po","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714852000,"log":"[2026-02-17T14:50:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vj8pmsa3tj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714853000,"log":"[2026-02-17T14:41:23Z INFO tonic::transport::server] grpc querier disconnected client_id=zggt47lf","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q6wdypm0yq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714854000,"log":"[2026-02-17T14:49:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fdz3ih9ree","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714855000,"log":"[2026-02-17T12:41:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i975yseknh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714856000,"log":"[2026-02-17T14:38:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=jsq91h9o","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-otft07ho80","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714857000,"log":"[2026-02-17T14:16:32Z DEBUG sqlx::query] SELECT FROM streams returned 698 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sb31z7ra0k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714858000,"log":"[2026-02-17T13:52:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 261ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nsj4w28lxy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714859000,"log":"[2026-02-17T14:34:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=z58r3ahf","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r0ov24t97j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714860000,"log":"[2026-02-17T13:40:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2566856189109703000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3rjvksa9ko","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714861000,"log":"[2026-02-17T14:04:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 442ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-45rqjo7wpp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714862000,"log":"[2026-02-17T15:20:12Z TRACE cache::redis] cache hit key=user:session:184","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pofhyzw1mm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714863000,"log":"[2026-02-17T12:50:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5mojqpjjc4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714864000,"log":"[2026-02-17T15:03:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wir7kjwuxc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714865000,"log":"[2026-02-17T12:45:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-29rokdry45","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714866000,"log":"[2026-02-17T12:47:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-81buamwgu2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714867000,"log":"[2026-02-17T14:53:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hqxee5w08d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714868000,"log":"[2026-02-17T15:13:58Z TRACE cache::redis] cache delete key=org:config:9766","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3wmkb45732","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714869000,"log":"[2026-02-17T13:38:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2837197516642087000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cvm2vg09ge","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714870000,"log":"[2026-02-17T15:01:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-75nwjmy7p7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714871000,"log":"[2026-02-17T13:41:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8865056456302225000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o4j34m2me0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714872000,"log":"[2026-02-17T12:47:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jpt3kuvcpm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714873000,"log":"[2026-02-17T13:12:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1891133498281778400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2gdawh5qxp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714874000,"log":"[2026-02-17T12:35:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f94uhixf4y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714875000,"log":"[2026-02-17T13:38:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2834083315461952500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gehl9fb82t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714876000,"log":"[2026-02-17T14:04:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 354ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lshz8h2rin","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714877000,"log":"[2026-02-17T14:04:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 399ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k1iyrv9bc4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714878000,"log":"[2026-02-17T15:07:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6407","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3zrr6mj4zu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714879000,"log":"[2026-02-17T12:55:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1oc5xjt1n7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714880000,"log":"[2026-02-17T14:33:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=qy1wy8gp","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9tngop18yr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714881000,"log":"[2026-02-17T14:54:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0mkkk9ep56","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714882000,"log":"[2026-02-17T13:00:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-74vyeioccf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714883000,"log":"[2026-02-17T14:08:35Z DEBUG sqlx::query] DELETE FROM alerts returned 565 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5g3ruldvod","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714884000,"log":"[2026-02-17T13:09:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4863626421847025000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gz4dreffqb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714885000,"log":"[2026-02-17T13:04:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8k8m04c9q6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714886000,"log":"[2026-02-17T13:31:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3673753596316977700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ygjjapg3el","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714887000,"log":"[2026-02-17T14:40:14Z INFO tonic::transport::server] grpc router request_received client_id=xdqz71bj","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ctibv194tm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714888000,"log":"[2026-02-17T13:28:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6671707653422923000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6y2inxobr4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714889000,"log":"[2026-02-17T14:07:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 764 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uphhcliayp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714890000,"log":"[2026-02-17T12:44:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j14i6pwabw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714891000,"log":"[2026-02-17T13:31:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1839499370592554500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bo7rku7g19","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714892000,"log":"[2026-02-17T14:45:04Z INFO tonic::transport::server] grpc router request_received client_id=gufwab9w","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l3wt5f0lft","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714893000,"log":"[2026-02-17T12:58:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u1mo3y0izk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714894000,"log":"[2026-02-17T13:19:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5943782577112719000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5k1mvdiqnd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714895000,"log":"[2026-02-17T12:55:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tk0oezrlvh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714896000,"log":"[2026-02-17T14:23:48Z DEBUG sqlx::query] INSERT FROM users returned 904 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lx9fkli5k2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714897000,"log":"[2026-02-17T14:08:25Z DEBUG sqlx::query] DELETE FROM alerts returned 332 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z7jfqdwgoy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714898000,"log":"[2026-02-17T13:00:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m0dsm9rqx8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714899000,"log":"[2026-02-17T14:06:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 470 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-87l3pbka7m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714900000,"log":"[2026-02-17T14:58:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-00xcriqjq5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714901000,"log":"[2026-02-17T14:07:32Z DEBUG sqlx::query] SELECT FROM streams returned 809 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-basdmzhiqy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714902000,"log":"[2026-02-17T15:18:18Z TRACE cache::redis] cache hit key=org:config:7108","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xk308fs4vr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714903000,"log":"[2026-02-17T14:38:48Z INFO tonic::transport::server] grpc querier disconnected client_id=g3zk7sv6","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qt4ynizrmf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714904000,"log":"[2026-02-17T13:36:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/421159949334917200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tsze4yq3ml","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714905000,"log":"[2026-02-17T15:12:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4015","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4z6e3yh8q4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714906000,"log":"[2026-02-17T15:20:18Z TRACE cache::redis] cache hit key=org:config:7900","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-li5o2uch30","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714907000,"log":"[2026-02-17T15:03:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9wujxiu86v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714908000,"log":"[2026-02-17T15:09:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1900","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mcixtr7zpw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714909000,"log":"[2026-02-17T15:20:57Z TRACE cache::redis] cache set key=user:session:9901","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ukoiuz6mu4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714910000,"log":"[2026-02-17T15:05:45Z DEBUG metrics::reporter] recorded metric active_connections value=314","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uxjlknflg4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714911000,"log":"[2026-02-17T13:52:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 313ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ywz8k7z8t4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714912000,"log":"[2026-02-17T14:50:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z0bpxh4zla","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714913000,"log":"[2026-02-17T12:46:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r0zhldvnwp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714914000,"log":"[2026-02-17T14:54:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3p8t88regm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714915000,"log":"[2026-02-17T13:29:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1813910548823432000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9auy3obtq1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714916000,"log":"[2026-02-17T13:48:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 386ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fpndheqzwp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714917000,"log":"[2026-02-17T13:28:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7091969768813085000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pq1p7cfydj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714918000,"log":"[2026-02-17T14:47:53Z INFO tonic::transport::server] grpc querier disconnected client_id=h6ox1ff0","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uywuzxzqc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714919000,"log":"[2026-02-17T15:02:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hqg7ldzgxm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714920000,"log":"[2026-02-17T12:36:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5vz6jkomt5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714921000,"log":"[2026-02-17T13:06:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-htqjf4uir7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714922000,"log":"[2026-02-17T12:42:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qjpem807nl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714923000,"log":"[2026-02-17T14:44:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=uyjlrg6n","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v4ihqxlyz7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714924000,"log":"[2026-02-17T12:52:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4pygdx8wq6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714925000,"log":"[2026-02-17T13:09:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/571402385021373440.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n98fup7udr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714926000,"log":"[2026-02-17T13:38:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1435722277896245200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yw8zuqgyy9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714927000,"log":"[2026-02-17T13:24:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5132354750968362000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9krnr6ok9w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714928000,"log":"[2026-02-17T13:57:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 406ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ucomwx250f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714929000,"log":"[2026-02-17T14:29:10Z DEBUG sqlx::query] DELETE FROM alerts returned 969 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0lpgkbukkg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714930000,"log":"[2026-02-17T14:01:00Z INFO actix_web::middleware::logger] POST /api/_search 403 449ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b7tj7lz6f7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714931000,"log":"[2026-02-17T13:21:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2793772546857902000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oa041or52g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714932000,"log":"[2026-02-17T13:27:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9804133357295116000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7cqt8pr7n2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714933000,"log":"[2026-02-17T13:04:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i7hkwjwdkc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714934000,"log":"[2026-02-17T12:49:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0iqww0mqu2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714935000,"log":"[2026-02-17T14:31:58Z INFO tonic::transport::server] grpc querier disconnected client_id=0eok3i6l","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r3kigmha7a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714936000,"log":"[2026-02-17T13:13:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7286501343941045000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ku02lq369h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714937000,"log":"[2026-02-17T14:06:53Z DEBUG sqlx::query] INSERT FROM users returned 454 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xw04bzts24","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714938000,"log":"[2026-02-17T13:55:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 205ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bz9thixzsm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714939000,"log":"[2026-02-17T14:48:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=t9zo18j4","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4txz0l7imo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714940000,"log":"[2026-02-17T15:17:25Z TRACE cache::redis] cache miss key=query:result:6269","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1rr1rd3nzg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714941000,"log":"[2026-02-17T12:40:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ijx4z3aqoq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714942000,"log":"[2026-02-17T12:57:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j98ekno6qy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714943000,"log":"[2026-02-17T14:44:29Z INFO tonic::transport::server] grpc router request_received client_id=sdwjp6nj","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-keq95585mf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714944000,"log":"[2026-02-17T13:14:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8760307062753357000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-edm9dmty7k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714945000,"log":"[2026-02-17T15:07:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8176","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jk2ytzbyrj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714946000,"log":"[2026-02-17T13:09:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/388203361874892540.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sz4avjobqm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714947000,"log":"[2026-02-17T14:11:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 366 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1b2lq0gj3w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714948000,"log":"[2026-02-17T15:18:54Z TRACE cache::redis] cache hit key=stream:schema:7669","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q1zspf1gun","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714949000,"log":"[2026-02-17T14:37:12Z INFO tonic::transport::server] grpc ingester connected client_id=d0va2cdj","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3fx45mh8e6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714950000,"log":"[2026-02-17T14:04:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 197ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e42m9ktz4l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714951000,"log":"[2026-02-17T14:49:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-664b0gpl2t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714952000,"log":"[2026-02-17T13:32:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2109024887551931400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0kccicgkqs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714953000,"log":"[2026-02-17T13:06:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qqdkuv9zh3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714954000,"log":"[2026-02-17T14:04:20Z INFO actix_web::middleware::logger] POST /api/_search 500 206ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gso71pq27p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714955000,"log":"[2026-02-17T12:36:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2tgsdr8u08","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714956000,"log":"[2026-02-17T13:19:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2918649108220037000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xzdgn8lmwm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714957000,"log":"[2026-02-17T13:23:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4238865298773180000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o9dznp06b6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714958000,"log":"[2026-02-17T14:48:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=3bell42t","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1022z6r2j8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714959000,"log":"[2026-02-17T14:11:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 12 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b0vxm4pech","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714960000,"log":"[2026-02-17T14:08:45Z DEBUG sqlx::query] DELETE FROM alerts returned 751 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j8kagltrxo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714961000,"log":"[2026-02-17T15:18:37Z TRACE cache::redis] cache miss key=user:session:7970","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d146j9paho","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714962000,"log":"[2026-02-17T14:15:07Z DEBUG sqlx::query] SELECT FROM streams returned 727 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-20brrvlqh5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714963000,"log":"[2026-02-17T12:49:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1c1l0wsw3e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714964000,"log":"[2026-02-17T13:35:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/514452463556962500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qtg60qywjk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714965000,"log":"[2026-02-17T14:51:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wlf7rta9ao","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714966000,"log":"[2026-02-17T14:19:40Z DEBUG sqlx::query] DELETE FROM alerts returned 52 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6xdwy34pp3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714967000,"log":"[2026-02-17T15:07:35Z DEBUG metrics::reporter] recorded metric active_connections value=1524","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hg0zltmzu8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714968000,"log":"[2026-02-17T15:09:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=5440","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dl49q16lsa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714969000,"log":"[2026-02-17T14:36:57Z INFO tonic::transport::server] grpc ingester connected client_id=vexdh76q","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-57pcy6ypwb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714970000,"log":"[2026-02-17T13:12:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3450708408079967700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1tn6d6feq4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714971000,"log":"[2026-02-17T13:47:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 193ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r1r61ft8ki","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714972000,"log":"[2026-02-17T13:30:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/909234503523110900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h172cyplz3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714973000,"log":"[2026-02-17T14:23:33Z DEBUG sqlx::query] INSERT FROM users returned 883 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g9fxt8b93s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714974000,"log":"[2026-02-17T12:57:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j67ou7qjte","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714975000,"log":"[2026-02-17T15:18:43Z TRACE cache::redis] cache miss key=org:config:2731","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gghn5zr27e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714976000,"log":"[2026-02-17T15:08:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5867","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tgvizx6wen","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714977000,"log":"[2026-02-17T13:10:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2551319134587977000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rnmngfvs9b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714978000,"log":"[2026-02-17T14:52:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e950yw2c82","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714979000,"log":"[2026-02-17T14:07:35Z DEBUG sqlx::query] DELETE FROM alerts returned 523 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b6y5s6qu2k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714980000,"log":"[2026-02-17T13:16:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1303274384087827000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ajtjao2q0d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714981000,"log":"[2026-02-17T13:02:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zx6w6euavh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714982000,"log":"[2026-02-17T12:43:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-brkdn9l1o7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714983000,"log":"[2026-02-17T13:41:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7768408661522659000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k2yuq6q3t8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714984000,"log":"[2026-02-17T13:35:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1289783192753806000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sv1e82ko38","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714985000,"log":"[2026-02-17T13:20:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/808012269732945000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rcetyjzv5f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714986000,"log":"[2026-02-17T12:42:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2r1sd7fu0v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714987000,"log":"[2026-02-17T14:36:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=8qbhnnx6","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-myao13ft9q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714988000,"log":"[2026-02-17T14:14:57Z DEBUG sqlx::query] SELECT FROM streams returned 93 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fzwdl6nc49","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714989000,"log":"[2026-02-17T12:44:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0izzms4qsr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714990000,"log":"[2026-02-17T13:09:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8692826687656186000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pm4yrxpcij","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714991000,"log":"[2026-02-17T12:39:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8mynx6nswx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714992000,"log":"[2026-02-17T14:25:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 439 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zdlx7ymy0q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714993000,"log":"[2026-02-17T13:00:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4pvref5adn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331714994000,"log":"[2026-02-17T14:19:57Z DEBUG sqlx::query] SELECT FROM streams returned 136 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qwdoklrnte","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714995000,"log":"[2026-02-17T13:30:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8243249393768731000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9cx17zbyrz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714996000,"log":"[2026-02-17T15:06:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=7481","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c03ldqzyqz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714997000,"log":"[2026-02-17T14:48:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8eviaic5nb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331714998000,"log":"[2026-02-17T15:16:26Z TRACE cache::redis] cache get key=auth:token:4752","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cp4whbcbhy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331714999000,"log":"[2026-02-17T12:46:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8rhkcreqbp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715000000,"log":"[2026-02-17T15:10:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4045","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ez8ei6gjb8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715001000,"log":"[2026-02-17T14:04:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 65ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8dbcjjjmci","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715002000,"log":"[2026-02-17T14:11:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 30 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ll03br5fo4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715003000,"log":"[2026-02-17T14:17:58Z DEBUG sqlx::query] INSERT FROM users returned 439 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5k5f0kp6jx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715004000,"log":"[2026-02-17T15:21:14Z TRACE cache::redis] cache get key=stream:schema:3863","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jw1xqp99h9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715005000,"log":"[2026-02-17T13:09:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9710962015293080000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zn71r1yz4k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715006000,"log":"[2026-02-17T14:38:52Z INFO tonic::transport::server] grpc ingester connected client_id=pzwa0xfa","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b45fii45v0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715007000,"log":"[2026-02-17T14:41:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=3hb87opg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yccrvn9ogw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715008000,"log":"[2026-02-17T14:31:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 652 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3eyeeg43of","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715009000,"log":"[2026-02-17T14:58:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-227etyp4x1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715010000,"log":"[2026-02-17T13:03:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lw6wpzh6na","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715011000,"log":"[2026-02-17T13:11:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3025072737080339000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3s0dtziwpf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715012000,"log":"[2026-02-17T12:37:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-23nld46ms4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715013000,"log":"[2026-02-17T13:05:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qwnflm96d6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715014000,"log":"[2026-02-17T14:12:42Z DEBUG sqlx::query] SELECT FROM streams returned 945 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ny8pmitvgv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715015000,"log":"[2026-02-17T12:38:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i7eu0af64n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715016000,"log":"[2026-02-17T14:46:08Z INFO tonic::transport::server] grpc querier disconnected client_id=skpam678","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m6luw9upyu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715017000,"log":"[2026-02-17T13:53:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 466ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u2dkxjwlgt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715018000,"log":"[2026-02-17T13:00:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0jm24xsv9m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715019000,"log":"[2026-02-17T14:47:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=nivfra0q","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vaulvge1wl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715020000,"log":"[2026-02-17T15:05:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7139","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xag07zajls","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715021000,"log":"[2026-02-17T13:53:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 41ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zs49xbgekn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715022000,"log":"[2026-02-17T12:47:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dhtefwr27y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715023000,"log":"[2026-02-17T13:56:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 312ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iq6nl9vdql","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715024000,"log":"[2026-02-17T12:51:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xu9gavfhkt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715025000,"log":"[2026-02-17T14:31:42Z DEBUG sqlx::query] SELECT FROM streams returned 892 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zdpof3ijwk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715026000,"log":"[2026-02-17T15:11:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=781","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tuw8d2vjys","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715027000,"log":"[2026-02-17T14:23:03Z DEBUG sqlx::query] INSERT FROM users returned 529 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w36wz0ez7e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715028000,"log":"[2026-02-17T12:50:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-274clsynhp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715029000,"log":"[2026-02-17T13:26:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2967028000356540000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9pz2bqxh86","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715030000,"log":"[2026-02-17T13:54:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 298ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-03096n99lo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715031000,"log":"[2026-02-17T12:35:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tbb9t7g14i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715032000,"log":"[2026-02-17T15:02:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-agkru5yf8l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715033000,"log":"[2026-02-17T14:01:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 167ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3zaqr3kxm1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715034000,"log":"[2026-02-17T13:01:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6olvzqnx3b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715035000,"log":"[2026-02-17T15:20:48Z TRACE cache::redis] cache hit key=org:config:5992","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bbk5353eg6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715036000,"log":"[2026-02-17T13:41:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7553142467987881000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xaxcd20mnv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715037000,"log":"[2026-02-17T12:36:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s3twcoopbx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715038000,"log":"[2026-02-17T13:58:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 403ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5267kqyb4l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715039000,"log":"[2026-02-17T13:59:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 38ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h2go7r11q0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715040000,"log":"[2026-02-17T13:57:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 286ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yzhnusw4jc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715041000,"log":"[2026-02-17T13:52:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 393ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q4wvwpotaq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715042000,"log":"[2026-02-17T14:43:17Z INFO tonic::transport::server] grpc ingester connected client_id=rrzl0xnz","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mbevfakn4v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715043000,"log":"[2026-02-17T15:09:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=719","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rocet3x4p5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715044000,"log":"[2026-02-17T14:29:13Z DEBUG sqlx::query] INSERT FROM users returned 672 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ve8md1cr4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715045000,"log":"[2026-02-17T14:39:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=0rst06em","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0k03pvbdxs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715046000,"log":"[2026-02-17T15:11:40Z DEBUG metrics::reporter] recorded metric active_connections value=2534","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u1e8rs1wsa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715047000,"log":"[2026-02-17T12:58:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mfwv6hje5u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715048000,"log":"[2026-02-17T13:35:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3642666094702578000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z12yupoem7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715049000,"log":"[2026-02-17T13:31:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5593010473722096000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fbqxxjb8y9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715050000,"log":"[2026-02-17T13:23:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6872911386896043000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c9q9cu47mk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715051000,"log":"[2026-02-17T14:31:02Z DEBUG sqlx::query] SELECT FROM streams returned 906 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ii4gayob71","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715052000,"log":"[2026-02-17T12:57:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9ik7d7oz74","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715053000,"log":"[2026-02-17T12:53:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nv4w9d4bye","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715054000,"log":"[2026-02-17T12:56:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ze2fzh44oe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715055000,"log":"[2026-02-17T15:01:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4x9wy1o7hl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715056000,"log":"[2026-02-17T14:15:02Z DEBUG sqlx::query] SELECT FROM streams returned 697 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-18uzkm6g97","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715057000,"log":"[2026-02-17T12:42:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zyqrbyxvoe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715058000,"log":"[2026-02-17T15:00:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s410gnp7fm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715059000,"log":"[2026-02-17T13:37:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7134086275685771000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4okvfui5o7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715060000,"log":"[2026-02-17T14:19:45Z DEBUG sqlx::query] DELETE FROM alerts returned 468 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fg43df5hb1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715061000,"log":"[2026-02-17T14:05:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 278ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8r92d4gt68","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715062000,"log":"[2026-02-17T12:59:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kgy6e6z034","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715063000,"log":"[2026-02-17T15:13:15Z DEBUG metrics::reporter] recorded metric active_connections value=9982","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mgj6xgm46w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715064000,"log":"[2026-02-17T13:44:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 371ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qzuscjzdzh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715065000,"log":"[2026-02-17T14:52:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qv6mgyfmao","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715066000,"log":"[2026-02-17T13:07:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-za5isyjofz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715067000,"log":"[2026-02-17T13:04:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kw889xtfb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715068000,"log":"[2026-02-17T15:06:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6673","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s2r8awj6xe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715069000,"log":"[2026-02-17T13:38:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5392987716886737000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hngzh3gkw5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715070000,"log":"[2026-02-17T14:43:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=7ibbtqx4","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qt5lxwxgpv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715071000,"log":"[2026-02-17T14:11:22Z DEBUG sqlx::query] SELECT FROM streams returned 168 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-poi8af6z2l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715072000,"log":"[2026-02-17T14:15:18Z DEBUG sqlx::query] INSERT FROM users returned 170 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5s5t227vli","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715073000,"log":"[2026-02-17T12:55:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uku7ymusyd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715074000,"log":"[2026-02-17T14:44:59Z INFO tonic::transport::server] grpc router request_received client_id=3dt5foj7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kl697eomra","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715075000,"log":"[2026-02-17T13:22:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3589040687632869400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rq24lcvxni","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715076000,"log":"[2026-02-17T14:21:53Z DEBUG sqlx::query] INSERT FROM users returned 195 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-arrxytddtv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715077000,"log":"[2026-02-17T13:19:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4901281708998186000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i9dy5lvwqh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715078000,"log":"[2026-02-17T14:46:44Z INFO tonic::transport::server] grpc router request_received client_id=kwvi4r7b","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mt27744ylg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715079000,"log":"[2026-02-17T14:53:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gocx5bzzwz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715080000,"log":"[2026-02-17T12:58:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dyfeysn679","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715081000,"log":"[2026-02-17T13:16:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2337333505799474700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g3lm9b7u3g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715082000,"log":"[2026-02-17T15:04:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nlzw733lhv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715083000,"log":"[2026-02-17T14:54:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d3m0bulv5e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715084000,"log":"[2026-02-17T12:52:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2umc802s3w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715085000,"log":"[2026-02-17T13:30:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1170770976265376300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6uyw2xcdh9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715086000,"log":"[2026-02-17T14:07:05Z DEBUG sqlx::query] DELETE FROM alerts returned 745 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ckyauy5at6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715087000,"log":"[2026-02-17T15:20:23Z TRACE cache::redis] cache expire key=org:config:3578","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m075xddl61","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715088000,"log":"[2026-02-17T14:16:05Z DEBUG sqlx::query] DELETE FROM alerts returned 304 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zxvgiwqro3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715089000,"log":"[2026-02-17T12:55:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-orrvlcpbwu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715090000,"log":"[2026-02-17T14:36:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=umf7j1j5","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-019l1hawkf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715091000,"log":"[2026-02-17T13:34:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8765005719729759000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o6pi5na0ce","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715092000,"log":"[2026-02-17T13:37:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8488845551057200000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mtoaf0n8uz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715093000,"log":"[2026-02-17T14:03:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 192ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i925c36uq0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715094000,"log":"[2026-02-17T12:52:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c98hcieg9j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715095000,"log":"[2026-02-17T14:49:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mn0wcl5scj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715096000,"log":"[2026-02-17T14:51:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eleu4g5e3f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715097000,"log":"[2026-02-17T15:07:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=5386","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bzu3pwbr7t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715098000,"log":"[2026-02-17T13:27:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8277354423674199000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o9is7oebft","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715099000,"log":"[2026-02-17T13:30:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3519577650184389000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xnr6rkl8k8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715100000,"log":"[2026-02-17T15:15:34Z TRACE cache::redis] cache delete key=stream:schema:8153","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nw3dfvmgrb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715101000,"log":"[2026-02-17T14:23:32Z DEBUG sqlx::query] SELECT FROM streams returned 680 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-af4osvccig","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715102000,"log":"[2026-02-17T14:25:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 452 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uka585jrk1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715103000,"log":"[2026-02-17T15:14:59Z TRACE cache::redis] cache expire key=stream:schema:9425","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p6aalaahi4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715104000,"log":"[2026-02-17T14:55:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p1t61l9nm2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715105000,"log":"[2026-02-17T13:19:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4822077011200742000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r9yah7fhjf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715106000,"log":"[2026-02-17T14:29:27Z DEBUG sqlx::query] SELECT FROM streams returned 513 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3ro5eck3uk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715107000,"log":"[2026-02-17T14:02:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 228ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ozrwyjes4j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715108000,"log":"[2026-02-17T12:56:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5ipx2vo290","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715109000,"log":"[2026-02-17T14:52:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fnjfps121g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715110000,"log":"[2026-02-17T14:19:52Z DEBUG sqlx::query] SELECT FROM streams returned 297 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e9wds9x05b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715111000,"log":"[2026-02-17T15:08:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1532","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zwnn4suixz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715112000,"log":"[2026-02-17T13:54:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 143ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-841brzc13h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715113000,"log":"[2026-02-17T14:50:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4zpitrgp34","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715114000,"log":"[2026-02-17T15:09:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5402","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fmaqiv4txi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715115000,"log":"[2026-02-17T13:34:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4867324847642515000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7euxfp37pp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715116000,"log":"[2026-02-17T13:15:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6900432372510281000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fg2z47gxxu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715117000,"log":"[2026-02-17T13:12:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2486953747158495700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pyws1s9j6d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715118000,"log":"[2026-02-17T12:45:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mcpyvpwcq6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715119000,"log":"[2026-02-17T13:01:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pyn2tw8wqg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715120000,"log":"[2026-02-17T14:30:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 242 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gzi60w7rxz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715121000,"log":"[2026-02-17T13:09:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/564947653393845760.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gvk37i6p15","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715122000,"log":"[2026-02-17T15:18:42Z TRACE cache::redis] cache hit key=user:session:884","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pwd3j50ujh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715123000,"log":"[2026-02-17T14:55:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xpg95cq4fk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715124000,"log":"[2026-02-17T15:16:40Z TRACE cache::redis] cache delete key=query:result:1157","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ez87bqrqrj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715125000,"log":"[2026-02-17T14:14:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 807 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ix5mm59chi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715126000,"log":"[2026-02-17T13:43:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 408ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-33xii28ohi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715127000,"log":"[2026-02-17T14:54:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pu01rz67ya","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715128000,"log":"[2026-02-17T13:16:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8504175085422007000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pwdmppnic9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715129000,"log":"[2026-02-17T14:52:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vnli3gh15b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715130000,"log":"[2026-02-17T14:07:07Z DEBUG sqlx::query] SELECT FROM streams returned 859 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x31xf43bue","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715131000,"log":"[2026-02-17T13:41:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5678077823770202000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wj3iyes2b0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715132000,"log":"[2026-02-17T13:58:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 227ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f3hqhl4zae","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715133000,"log":"[2026-02-17T13:23:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7409289441639666000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a745adhacz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715134000,"log":"[2026-02-17T12:57:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xe262fbu56","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715135000,"log":"[2026-02-17T14:23:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 719 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kcrhrufqks","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715136000,"log":"[2026-02-17T14:58:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gz7xskngiw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715137000,"log":"[2026-02-17T15:17:26Z TRACE cache::redis] cache get key=auth:token:9699","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zerokssn2y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715138000,"log":"[2026-02-17T13:50:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 312ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-571wr3t8kg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715139000,"log":"[2026-02-17T13:18:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6116881563582404000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-30tn7j0zsw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715140000,"log":"[2026-02-17T13:38:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5536647601219824000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1sgbbds9fn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715141000,"log":"[2026-02-17T12:51:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kag7t4heso","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715142000,"log":"[2026-02-17T13:22:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7180765675663676000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cihilz8rdf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715143000,"log":"[2026-02-17T13:08:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1fc76sk5wm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715144000,"log":"[2026-02-17T14:07:23Z DEBUG sqlx::query] INSERT FROM users returned 103 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yeps2l28ls","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715145000,"log":"[2026-02-17T13:16:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1637365692154015700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3t72mv2ju5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715146000,"log":"[2026-02-17T13:07:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zpnwqzns6u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715147000,"log":"[2026-02-17T15:07:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=9661","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5qpe07bnuj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715148000,"log":"[2026-02-17T12:57:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aj4d78y1rf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715149000,"log":"[2026-02-17T13:18:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2825353588376498000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xtdkrwk49l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715150000,"log":"[2026-02-17T12:50:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vt6xdxtvxo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715151000,"log":"[2026-02-17T13:51:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 221ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-die71bec8h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715152000,"log":"[2026-02-17T14:21:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 26 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4jhwav61uy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715153000,"log":"[2026-02-17T14:26:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 998 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j8ewmqwhqb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715154000,"log":"[2026-02-17T14:10:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 444 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m9xdeyyaf2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715155000,"log":"[2026-02-17T15:09:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8087","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v23jnrhqi5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715156000,"log":"[2026-02-17T13:58:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 359ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xzttueomah","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715157000,"log":"[2026-02-17T13:16:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5273060157435379000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p10rokpyze","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715158000,"log":"[2026-02-17T14:51:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-om5eevxcjv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715159000,"log":"[2026-02-17T14:07:55Z DEBUG sqlx::query] DELETE FROM alerts returned 254 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oovbgrvlz1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715160000,"log":"[2026-02-17T14:14:25Z DEBUG sqlx::query] DELETE FROM alerts returned 717 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cs46iekpge","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715161000,"log":"[2026-02-17T13:52:25Z INFO actix_web::middleware::logger] GET /api/_search 400 109ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nwtah7ze9o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715162000,"log":"[2026-02-17T14:38:22Z INFO tonic::transport::server] grpc ingester connected client_id=943ohrpp","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g0mv6ntcmf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715163000,"log":"[2026-02-17T12:35:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-efvaese02l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715164000,"log":"[2026-02-17T13:42:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 290ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2eafpqv54s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715165000,"log":"[2026-02-17T14:09:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 849 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gf2ph7olzd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715166000,"log":"[2026-02-17T14:52:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9c20vhdpqr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715167000,"log":"[2026-02-17T14:07:17Z DEBUG sqlx::query] SELECT FROM streams returned 898 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cpb0ifgt8m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715168000,"log":"[2026-02-17T14:10:30Z DEBUG sqlx::query] DELETE FROM alerts returned 327 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-crp9or2gp4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715169000,"log":"[2026-02-17T14:58:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-djfauqzru1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715170000,"log":"[2026-02-17T13:03:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1u5971uvnz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715171000,"log":"[2026-02-17T12:54:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-srbj7k5wxh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715172000,"log":"[2026-02-17T15:19:34Z TRACE cache::redis] cache delete key=stream:schema:1991","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n6qqpnjom6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715173000,"log":"[2026-02-17T13:19:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8275025636019016000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b6h9sxob4t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715174000,"log":"[2026-02-17T12:38:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e9z33rb4qn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715175000,"log":"[2026-02-17T13:01:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ofcn1o95ma","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715176000,"log":"[2026-02-17T15:03:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vx2nmfoz46","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715177000,"log":"[2026-02-17T13:47:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5lkq5i0ols","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715178000,"log":"[2026-02-17T13:13:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8924386749470659000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qpdau65hpm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715179000,"log":"[2026-02-17T14:18:37Z DEBUG sqlx::query] SELECT FROM streams returned 152 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dpbz2pveby","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715180000,"log":"[2026-02-17T13:56:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 368ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1llow177p7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715181000,"log":"[2026-02-17T15:03:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yjfy79zy83","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715182000,"log":"[2026-02-17T15:04:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-or44c9xp9g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715183000,"log":"[2026-02-17T15:05:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cjz0pcamyc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715184000,"log":"[2026-02-17T15:21:04Z TRACE cache::redis] cache delete key=stream:schema:4569","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v52dceotax","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715185000,"log":"[2026-02-17T14:53:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uwvz2gmia","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715186000,"log":"[2026-02-17T12:42:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-okflczf1wi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715187000,"log":"[2026-02-17T13:48:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 493ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dhjwwfp51w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715188000,"log":"[2026-02-17T15:20:05Z TRACE cache::redis] cache expire key=query:result:4684","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-voocjr9nt7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715189000,"log":"[2026-02-17T13:33:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2259705027635863300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-egiavav32u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715190000,"log":"[2026-02-17T14:52:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-10h1kjtgeu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715191000,"log":"[2026-02-17T13:07:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ckhydo4b8h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715192000,"log":"[2026-02-17T14:17:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 89 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-68j5wkgsdf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715193000,"log":"[2026-02-17T13:24:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5612237603622683000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l004aagk9g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715194000,"log":"[2026-02-17T13:24:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9308899828293607000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pphy2ey5gd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715195000,"log":"[2026-02-17T13:06:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qgej01d6e1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715196000,"log":"[2026-02-17T13:33:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/659822551649182800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2xmnt89wy6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715197000,"log":"[2026-02-17T14:19:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 743 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z4l004s0hc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715198000,"log":"[2026-02-17T14:34:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=rqjcvjyg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x5340gjvbt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715199000,"log":"[2026-02-17T13:30:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3261063458930045000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5kik4xmv22","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715200000,"log":"[2026-02-17T14:45:54Z INFO tonic::transport::server] grpc router request_received client_id=z6qlyd8g","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mjfre7tvno","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715201000,"log":"[2026-02-17T13:26:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4915546417890255000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yowrooen6j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715202000,"log":"[2026-02-17T15:06:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6128","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-62qdphtso1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715203000,"log":"[2026-02-17T13:14:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/139645439785256450.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pivwsafzd2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715204000,"log":"[2026-02-17T13:39:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8578144277174822000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d25sahnmb9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715205000,"log":"[2026-02-17T15:03:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xizuc3huu9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715206000,"log":"[2026-02-17T15:10:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1696","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3kq2i4r889","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715207000,"log":"[2026-02-17T15:17:04Z TRACE cache::redis] cache delete key=stream:schema:626","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ec8vd6uu6u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715208000,"log":"[2026-02-17T13:25:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9439813516080613000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ebq4fofjmw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715209000,"log":"[2026-02-17T14:32:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=yv6axkfa","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m4toeq7n9x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715210000,"log":"[2026-02-17T13:34:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5027429684683429000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4xoc2goioc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715211000,"log":"[2026-02-17T14:03:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 213ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rufqdpv1rp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715212000,"log":"[2026-02-17T13:28:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/258681863373009250.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uxssmcerik","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715213000,"log":"[2026-02-17T13:18:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9138191394065873000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v0pctfosq3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715214000,"log":"[2026-02-17T14:17:38Z DEBUG sqlx::query] INSERT FROM users returned 959 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4jxiwp9bhp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715215000,"log":"[2026-02-17T15:06:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5374","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wpltjeirmo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715216000,"log":"[2026-02-17T13:08:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4546881161656340000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6hppk7x2fl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715217000,"log":"[2026-02-17T15:07:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2629","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hiy0o9egnr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715218000,"log":"[2026-02-17T14:22:52Z DEBUG sqlx::query] SELECT FROM streams returned 771 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fc6w99azer","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715219000,"log":"[2026-02-17T14:27:22Z DEBUG sqlx::query] SELECT FROM streams returned 53 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hav7nzlo1r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715220000,"log":"[2026-02-17T15:06:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=2675","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5pr1d8cfyv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715221000,"log":"[2026-02-17T13:17:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3210747873928636400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fr7od814tq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715222000,"log":"[2026-02-17T14:48:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6cbjpclmcv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715223000,"log":"[2026-02-17T13:03:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xum7znm2d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715224000,"log":"[2026-02-17T13:43:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 318ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uibcbhasuj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715225000,"log":"[2026-02-17T14:08:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 286 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cm4vhmo4un","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715226000,"log":"[2026-02-17T13:58:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 56ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5q29csk7xn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715227000,"log":"[2026-02-17T14:11:45Z DEBUG sqlx::query] DELETE FROM alerts returned 853 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5i8aw4hlw0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715228000,"log":"[2026-02-17T13:58:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-es8xdj9cjg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715229000,"log":"[2026-02-17T15:03:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tghogk5hkg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715230000,"log":"[2026-02-17T14:42:13Z INFO tonic::transport::server] grpc querier disconnected client_id=hat7aowm","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zywghhad0l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715231000,"log":"[2026-02-17T14:14:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 338 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9em40q8jd4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715232000,"log":"[2026-02-17T12:51:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xmhr6ks7bh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715233000,"log":"[2026-02-17T13:04:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f5fdcqba7m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715234000,"log":"[2026-02-17T14:53:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r5vdsdsc1s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715235000,"log":"[2026-02-17T13:48:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 138ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6uwbujkct6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715236000,"log":"[2026-02-17T15:14:17Z TRACE cache::redis] cache expire key=user:session:5447","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5g76otz1lv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715237000,"log":"[2026-02-17T14:17:52Z DEBUG sqlx::query] SELECT FROM streams returned 896 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v51l1w72c6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715238000,"log":"[2026-02-17T14:48:08Z INFO tonic::transport::server] grpc querier disconnected client_id=t9e1im2t","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d6ujcl3twg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715239000,"log":"[2026-02-17T14:37:22Z INFO tonic::transport::server] grpc ingester connected client_id=4fuefqq2","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dpdjnekpj8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715240000,"log":"[2026-02-17T15:00:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t1fg2429fk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715241000,"log":"[2026-02-17T13:11:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8941786872941875000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ipogtf0w4v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715242000,"log":"[2026-02-17T12:46:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d1z16mzyb2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715243000,"log":"[2026-02-17T13:59:05Z INFO actix_web::middleware::logger] GET /api/_search 500 3ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dichcreguu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715244000,"log":"[2026-02-17T15:07:10Z DEBUG metrics::reporter] recorded metric active_connections value=2206","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-06snv4h4w2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715245000,"log":"[2026-02-17T14:48:04Z INFO tonic::transport::server] grpc router request_received client_id=5xu8imbt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zygl0utvul","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715246000,"log":"[2026-02-17T12:37:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rhqh0d82s4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715247000,"log":"[2026-02-17T14:07:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 779 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gje0dbzg5x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715248000,"log":"[2026-02-17T12:45:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s7q460h1yy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715249000,"log":"[2026-02-17T14:49:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hze3irzq28","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715250000,"log":"[2026-02-17T14:47:49Z INFO tonic::transport::server] grpc router request_received client_id=5wp8ilvm","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xmoj66vyl7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715251000,"log":"[2026-02-17T14:37:14Z INFO tonic::transport::server] grpc router request_received client_id=tf91j3wz","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-72vab0u6ml","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715252000,"log":"[2026-02-17T12:37:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e82urvis5k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715253000,"log":"[2026-02-17T14:46:02Z INFO tonic::transport::server] grpc ingester connected client_id=7ht5xj4a","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tq588spxdi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715254000,"log":"[2026-02-17T13:35:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/983588711051881500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qsmj1ga4br","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715255000,"log":"[2026-02-17T13:16:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2331545396064192000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rpfykjmdt2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715256000,"log":"[2026-02-17T13:59:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 222ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yl8dz81lmn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715257000,"log":"[2026-02-17T13:28:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4453876048588512000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ekyh47z7fo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715258000,"log":"[2026-02-17T14:19:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 191 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yk9jk6k7p4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715259000,"log":"[2026-02-17T13:01:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x0hyamya8b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715260000,"log":"[2026-02-17T13:21:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/496728205687015940.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ihvzbe6246","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715261000,"log":"[2026-02-17T15:18:21Z TRACE cache::redis] cache set key=auth:token:5507","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qt9euzqf4f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715262000,"log":"[2026-02-17T14:31:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 108 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zcd9o71hs5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715263000,"log":"[2026-02-17T14:29:12Z DEBUG sqlx::query] SELECT FROM streams returned 562 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a9991s3dt8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715264000,"log":"[2026-02-17T14:16:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 92 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k5de13zm5a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715265000,"log":"[2026-02-17T13:10:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6075759786124308000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sllopyns8f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715266000,"log":"[2026-02-17T13:07:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lssgywil8t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715267000,"log":"[2026-02-17T15:13:25Z DEBUG metrics::reporter] recorded metric active_connections value=6702","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1vxisies0y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715268000,"log":"[2026-02-17T15:20:08Z TRACE cache::redis] cache get key=org:config:695","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5j25fuokd1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715269000,"log":"[2026-02-17T14:41:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=w16yenez","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fipzxem5oa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715270000,"log":"[2026-02-17T12:47:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xtb4l3l9st","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715271000,"log":"[2026-02-17T13:46:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 40ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2qv3glirm3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715272000,"log":"[2026-02-17T15:01:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3ednpfx1ws","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715273000,"log":"[2026-02-17T14:03:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 16ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vraylrh3un","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715274000,"log":"[2026-02-17T13:33:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6841542398959663000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x8lnbjheku","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715275000,"log":"[2026-02-17T15:17:32Z TRACE cache::redis] cache get key=user:session:9503","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cny6vci7ai","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715276000,"log":"[2026-02-17T13:47:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 109ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n9hh6yx5u7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715277000,"log":"[2026-02-17T15:06:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8817","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u7dt538xll","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715278000,"log":"[2026-02-17T15:10:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8471","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-idxcl9xyqw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715279000,"log":"[2026-02-17T13:43:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 25ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5dychdtyby","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715280000,"log":"[2026-02-17T14:09:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 29 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6yymm1o1c9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715281000,"log":"[2026-02-17T12:56:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r74alc6bip","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715282000,"log":"[2026-02-17T13:04:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9dia8nto90","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715283000,"log":"[2026-02-17T13:48:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 350ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-giutmy0t4i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715284000,"log":"[2026-02-17T15:14:54Z TRACE cache::redis] cache hit key=stream:schema:6303","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-42t5tcdjxz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715285000,"log":"[2026-02-17T14:53:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-78z79vxpds","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715286000,"log":"[2026-02-17T13:56:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 373ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-krimqen8bw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715287000,"log":"[2026-02-17T13:14:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9070882909645496000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fxeq6zb2b5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715288000,"log":"[2026-02-17T12:56:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u92wt7j3pt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715289000,"log":"[2026-02-17T14:34:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=6xetbs0k","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-61utlpnrak","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715290000,"log":"[2026-02-17T14:37:32Z INFO tonic::transport::server] grpc ingester connected client_id=cc6vzmoe","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0dbc5w3y5n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715291000,"log":"[2026-02-17T13:53:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 277ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dpzdxdpv67","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715292000,"log":"[2026-02-17T14:51:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fr46nphxr0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715293000,"log":"[2026-02-17T13:32:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/936247613816605300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hu9mlb44rz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715294000,"log":"[2026-02-17T15:02:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gmg4oht9ok","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715295000,"log":"[2026-02-17T15:07:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2216","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q6u1fgvsmu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715296000,"log":"[2026-02-17T13:31:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/420049611503947600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-spsxstwbj1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715297000,"log":"[2026-02-17T13:16:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9314236461479526000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p23bll4kzm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715298000,"log":"[2026-02-17T13:21:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7776363071420002000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pm5a37kg59","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715299000,"log":"[2026-02-17T14:25:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 224 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2xrczgo8wm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715300000,"log":"[2026-02-17T15:20:33Z TRACE cache::redis] cache set key=org:config:213","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hoj4zddfic","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715301000,"log":"[2026-02-17T15:17:45Z TRACE cache::redis] cache set key=query:result:6436","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iqkbjbfiv5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715302000,"log":"[2026-02-17T13:05:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0u13hrc2u3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715303000,"log":"[2026-02-17T15:09:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8278","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x2j8x8p34u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715304000,"log":"[2026-02-17T13:08:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7zy0tg2a4c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715305000,"log":"[2026-02-17T14:29:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 938 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fc09dplfg7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715306000,"log":"[2026-02-17T15:19:09Z TRACE cache::redis] cache set key=stream:schema:1598","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-owp0h8bvi3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715307000,"log":"[2026-02-17T14:32:07Z INFO tonic::transport::server] grpc ingester connected client_id=n6l18q3x","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5cqyag771q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715308000,"log":"[2026-02-17T13:27:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/163639259287959870.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nxn07h1yw4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715309000,"log":"[2026-02-17T13:59:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 259ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-onhgl3gttr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715310000,"log":"[2026-02-17T14:27:43Z DEBUG sqlx::query] INSERT FROM users returned 715 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pyxeegbqni","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715311000,"log":"[2026-02-17T14:14:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 95 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-00dsia66b1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715312000,"log":"[2026-02-17T14:39:39Z INFO tonic::transport::server] grpc router request_received client_id=jjbd3l5u","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c82kaexdpx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715313000,"log":"[2026-02-17T15:20:30Z TRACE cache::redis] cache hit key=query:result:1658","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tqv74fburf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715314000,"log":"[2026-02-17T12:51:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p2l50qo08p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715315000,"log":"[2026-02-17T14:41:08Z INFO tonic::transport::server] grpc querier disconnected client_id=82004e4z","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ept70wag1u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715316000,"log":"[2026-02-17T14:12:00Z DEBUG sqlx::query] DELETE FROM alerts returned 785 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cq2v327nab","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715317000,"log":"[2026-02-17T14:11:03Z DEBUG sqlx::query] INSERT FROM users returned 741 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-09xu6ygih2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715318000,"log":"[2026-02-17T13:10:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/268619510816590080.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w0o98a6qhl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715319000,"log":"[2026-02-17T13:13:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3445914184336374000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h5hnoz9ahl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715320000,"log":"[2026-02-17T15:08:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3522","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z95tq2xag8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715321000,"log":"[2026-02-17T13:12:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8122217009012029000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bk9j1sxn8g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715322000,"log":"[2026-02-17T15:21:48Z TRACE cache::redis] cache hit key=org:config:1573","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tafop38h8c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715323000,"log":"[2026-02-17T13:34:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8753003138466450000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tmal89cgfd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715324000,"log":"[2026-02-17T14:06:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 348ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8ulw7pqvt8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715325000,"log":"[2026-02-17T15:09:35Z DEBUG metrics::reporter] recorded metric active_connections value=4690","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jyqiiv55ai","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715326000,"log":"[2026-02-17T15:01:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0dcq8kqywz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715327000,"log":"[2026-02-17T14:17:27Z DEBUG sqlx::query] SELECT FROM streams returned 987 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-87pcdsftrf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715328000,"log":"[2026-02-17T14:47:12Z INFO tonic::transport::server] grpc ingester connected client_id=ox6vlwir","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bxfbgktop2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715329000,"log":"[2026-02-17T14:02:40Z INFO actix_web::middleware::logger] POST /api/_search 404 196ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gsewyc6s7h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715330000,"log":"[2026-02-17T14:07:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 681 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4wdw49r4sa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715331000,"log":"[2026-02-17T15:13:42Z TRACE cache::redis] cache hit key=user:session:1857","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bqulr4gi68","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715332000,"log":"[2026-02-17T13:15:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9028738143905068000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5b0gnw38mp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715333000,"log":"[2026-02-17T13:16:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/321157851689657400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hj8koxumjd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715334000,"log":"[2026-02-17T14:27:02Z DEBUG sqlx::query] SELECT FROM streams returned 831 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y8p45ovwlg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715335000,"log":"[2026-02-17T12:46:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ynwy1dhhes","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715336000,"log":"[2026-02-17T13:56:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 248ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vm3nxkyyuk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715337000,"log":"[2026-02-17T14:01:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 114ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vs154jw4tc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715338000,"log":"[2026-02-17T14:55:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5vrf5ge74n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715339000,"log":"[2026-02-17T14:58:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o2z552vjek","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715340000,"log":"[2026-02-17T12:46:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2f0gkf8o6n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715341000,"log":"[2026-02-17T14:21:42Z DEBUG sqlx::query] SELECT FROM streams returned 324 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wmv2st9nf6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715342000,"log":"[2026-02-17T13:18:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9559251570481127000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2jlzivfgs6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715343000,"log":"[2026-02-17T13:38:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7496337192160336000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-401qrbyiek","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715344000,"log":"[2026-02-17T13:35:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2978109365695198000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-951tgv3tj1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715345000,"log":"[2026-02-17T13:05:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-95uchmjltv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715346000,"log":"[2026-02-17T14:14:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 149 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ay53wdknab","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715347000,"log":"[2026-02-17T14:51:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h0x7jtrc2m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715348000,"log":"[2026-02-17T14:11:00Z DEBUG sqlx::query] DELETE FROM alerts returned 103 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s4ydaz1kvo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715349000,"log":"[2026-02-17T14:31:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 25 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5voovjsv84","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715350000,"log":"[2026-02-17T13:53:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 267ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5ce10kf5fy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715351000,"log":"[2026-02-17T14:54:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pufgx07pyk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715352000,"log":"[2026-02-17T13:58:45Z INFO actix_web::middleware::logger] GET /api/_search 403 83ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sm61dya7p1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715353000,"log":"[2026-02-17T13:38:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4291650996384583700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6efnq6t563","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715354000,"log":"[2026-02-17T14:39:54Z INFO tonic::transport::server] grpc router request_received client_id=ddu6zfvm","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0dztquops7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715355000,"log":"[2026-02-17T14:52:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0ohtaepszk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715356000,"log":"[2026-02-17T14:50:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fhykwj497i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715357000,"log":"[2026-02-17T14:57:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1ww7xmavx2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715358000,"log":"[2026-02-17T13:11:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1536143048467251200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r223dckvaf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715359000,"log":"[2026-02-17T13:48:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 259ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qdfy1f3fe8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715360000,"log":"[2026-02-17T13:39:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5613542072236548000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-obuwrad81c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715361000,"log":"[2026-02-17T15:15:12Z TRACE cache::redis] cache hit key=user:session:631","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-64savfmew0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715362000,"log":"[2026-02-17T13:59:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 231ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a0hjhldgal","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715363000,"log":"[2026-02-17T14:11:53Z DEBUG sqlx::query] INSERT FROM users returned 258 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y3xqf50wnv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715364000,"log":"[2026-02-17T13:51:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 351ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-949l5oxlt1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715365000,"log":"[2026-02-17T14:20:07Z DEBUG sqlx::query] SELECT FROM streams returned 448 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-98z94l2x9n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715366000,"log":"[2026-02-17T13:25:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7085393362646935000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g9v3xc2apk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715367000,"log":"[2026-02-17T14:01:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 164ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nozoxdsprm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715368000,"log":"[2026-02-17T13:44:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 1ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ioghhjwnpa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715369000,"log":"[2026-02-17T13:56:00Z INFO actix_web::middleware::logger] POST /api/_search 204 299ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rq9vptmx3t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715370000,"log":"[2026-02-17T12:54:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3rtevbbblo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715371000,"log":"[2026-02-17T12:43:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h5809erza2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715372000,"log":"[2026-02-17T14:04:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a7sg9rl6tg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715373000,"log":"[2026-02-17T15:04:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3h5esw6vno","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715374000,"log":"[2026-02-17T13:00:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jflhiq9802","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715375000,"log":"[2026-02-17T15:07:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=5695","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aub9wwyjk9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715376000,"log":"[2026-02-17T15:16:14Z TRACE cache::redis] cache get key=stream:schema:2402","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bdnz03578f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715377000,"log":"[2026-02-17T15:15:23Z TRACE cache::redis] cache expire key=org:config:210","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v1gaiqul1c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715378000,"log":"[2026-02-17T13:49:45Z INFO actix_web::middleware::logger] GET /api/_search 403 135ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e7d2f1rqms","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715379000,"log":"[2026-02-17T13:39:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1345189882299033900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ipbgm1enbs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715380000,"log":"[2026-02-17T13:47:45Z INFO actix_web::middleware::logger] GET /api/_search 204 475ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x08hc38bsc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715381000,"log":"[2026-02-17T14:33:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=7rzqc0yo","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gtjz1qw559","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715382000,"log":"[2026-02-17T12:50:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ss2dkeg2kb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715383000,"log":"[2026-02-17T15:11:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1593","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y32icos2vz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715384000,"log":"[2026-02-17T12:55:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9y6vqav4bn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715385000,"log":"[2026-02-17T14:05:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 315ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nm4qcbjl2x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715386000,"log":"[2026-02-17T13:27:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6518955864366822000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jw6tdcttm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715387000,"log":"[2026-02-17T13:12:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1279557782352891400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gjatxglels","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715388000,"log":"[2026-02-17T15:15:13Z TRACE cache::redis] cache miss key=org:config:3735","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sb4j16s1e8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715389000,"log":"[2026-02-17T14:26:52Z DEBUG sqlx::query] SELECT FROM streams returned 773 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l80m9kr5n5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715390000,"log":"[2026-02-17T14:31:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 26 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tvqofckgr1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715391000,"log":"[2026-02-17T13:38:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5572248944052238000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-koze5kf9h1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715392000,"log":"[2026-02-17T14:38:32Z INFO tonic::transport::server] grpc ingester connected client_id=gabl0xmg","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9cajzjkf0z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715393000,"log":"[2026-02-17T12:47:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-edx0nzm1yi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715394000,"log":"[2026-02-17T13:33:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9381080786490845000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qjc5497acq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715395000,"log":"[2026-02-17T14:37:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=yit88sx0","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gs89bsyxr2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715396000,"log":"[2026-02-17T15:21:29Z TRACE cache::redis] cache expire key=stream:schema:9221","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-secn4lp1xe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715397000,"log":"[2026-02-17T12:58:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-10f796x2z3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715398000,"log":"[2026-02-17T15:14:32Z TRACE cache::redis] cache get key=user:session:6332","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3ud9qtevam","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715399000,"log":"[2026-02-17T12:39:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9or2cwjtcc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715400000,"log":"[2026-02-17T14:18:42Z DEBUG sqlx::query] SELECT FROM streams returned 674 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tfyuo7381j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715401000,"log":"[2026-02-17T13:06:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t0ts3tuemx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715402000,"log":"[2026-02-17T13:43:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 132ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hbvu36b0th","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715403000,"log":"[2026-02-17T14:30:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 731 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jqfjmzmdme","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715404000,"log":"[2026-02-17T15:07:05Z DEBUG metrics::reporter] recorded metric active_connections value=176","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vdv51c2qzq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715405000,"log":"[2026-02-17T15:11:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=2485","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3hx023afhk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715406000,"log":"[2026-02-17T12:38:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jequ61nc3h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715407000,"log":"[2026-02-17T13:26:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3836224773338175000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2u4cn9alcs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715408000,"log":"[2026-02-17T13:58:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 346ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5ang5szxbn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715409000,"log":"[2026-02-17T13:51:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 336ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4wvvfxwn8u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715410000,"log":"[2026-02-17T12:35:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zexbf8i5d7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715411000,"log":"[2026-02-17T13:36:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2680078538501738000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a92pbcjzq7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715412000,"log":"[2026-02-17T13:04:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kvxe0v1ptp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715413000,"log":"[2026-02-17T12:48:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uhdims109b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715414000,"log":"[2026-02-17T14:45:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=48mehqx0","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fdxysl5gzf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715415000,"log":"[2026-02-17T12:42:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0yftbq3xo2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715416000,"log":"[2026-02-17T14:51:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-71ngmzhzyp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715417000,"log":"[2026-02-17T13:03:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sxziwfakmv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715418000,"log":"[2026-02-17T14:46:33Z INFO tonic::transport::server] grpc querier disconnected client_id=86erttgv","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h1ncnafrju","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715419000,"log":"[2026-02-17T13:22:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/931766061206553700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i59j6a6nbe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715420000,"log":"[2026-02-17T12:42:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kr01tbyz8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715421000,"log":"[2026-02-17T15:01:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mj857zmahj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715422000,"log":"[2026-02-17T13:07:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dz1nucb9tj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715423000,"log":"[2026-02-17T15:08:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8525","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g5dznix095","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715424000,"log":"[2026-02-17T13:45:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 462ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pxyvgzbdzs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715425000,"log":"[2026-02-17T13:09:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9949168788863840000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7jyb1sofwk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715426000,"log":"[2026-02-17T12:50:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qnt9qg422g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715427000,"log":"[2026-02-17T15:16:29Z TRACE cache::redis] cache expire key=stream:schema:5995","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xfuyhmzync","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715428000,"log":"[2026-02-17T14:00:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 209ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-burcwx1iyp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715429000,"log":"[2026-02-17T14:04:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 173ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z3qyw7rgj6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715430000,"log":"[2026-02-17T12:46:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y6hfrkmj35","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715431000,"log":"[2026-02-17T12:40:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9uwpgxkq3w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715432000,"log":"[2026-02-17T13:04:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zeewk7pz0j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715433000,"log":"[2026-02-17T15:06:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9215","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-71ocjqvph6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715434000,"log":"[2026-02-17T13:23:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2600596150963481000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r7zn7k1ho7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715435000,"log":"[2026-02-17T12:44:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s84qeys6s4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715436000,"log":"[2026-02-17T14:03:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 139ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-izxsonc0vg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715437000,"log":"[2026-02-17T13:36:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3762867830481138000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-olk8ecy70p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715438000,"log":"[2026-02-17T14:43:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=p8phkl2y","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p4xhkr16h5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715439000,"log":"[2026-02-17T13:13:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2912789919052918300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gy23jj2264","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715440000,"log":"[2026-02-17T13:35:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6706974367393457000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aqou508d00","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715441000,"log":"[2026-02-17T13:15:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3569523622447723500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-66yjmpyi8l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715442000,"log":"[2026-02-17T12:44:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g4hi8jcoam","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715443000,"log":"[2026-02-17T14:14:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 786 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c3wjymv9x3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715444000,"log":"[2026-02-17T12:45:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-18qegftiay","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715445000,"log":"[2026-02-17T13:55:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 154ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ijp8kek5k6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715446000,"log":"[2026-02-17T14:02:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 361ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vxf47k3x7c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715447000,"log":"[2026-02-17T12:42:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c0aaajkoze","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715448000,"log":"[2026-02-17T15:01:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xnebch3k69","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715449000,"log":"[2026-02-17T13:21:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3666759138482096000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qwuhylk92d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715450000,"log":"[2026-02-17T15:02:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j6mn9r1wwj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715451000,"log":"[2026-02-17T14:23:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 568 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8abhb74hwo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715452000,"log":"[2026-02-17T14:13:38Z DEBUG sqlx::query] INSERT FROM users returned 780 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6wx5i9lput","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715453000,"log":"[2026-02-17T14:17:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 353 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vpz0vargm5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715454000,"log":"[2026-02-17T14:34:37Z INFO tonic::transport::server] grpc ingester connected client_id=mgoi50zw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nfom7mmwfz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715455000,"log":"[2026-02-17T14:26:28Z DEBUG sqlx::query] INSERT FROM users returned 30 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dtthtnklfi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715456000,"log":"[2026-02-17T15:02:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-665gj52il5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715457000,"log":"[2026-02-17T14:04:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 92ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ujt2jjtf5h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715458000,"log":"[2026-02-17T12:47:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wojx5ripv0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715459000,"log":"[2026-02-17T13:13:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7127201686065978000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jyz1epu1f7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715460000,"log":"[2026-02-17T13:17:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3745560589239447000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j1p0u2slg9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715461000,"log":"[2026-02-17T12:36:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yavep177dx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715462000,"log":"[2026-02-17T13:46:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 456ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-up8pd1hw83","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715463000,"log":"[2026-02-17T13:48:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 62ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eh3a21l019","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715464000,"log":"[2026-02-17T13:59:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 415ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w7tr2owp7v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715465000,"log":"[2026-02-17T12:57:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-feyg9jgrpu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715466000,"log":"[2026-02-17T12:40:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dmtnivlojx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715467000,"log":"[2026-02-17T14:21:48Z DEBUG sqlx::query] INSERT FROM users returned 373 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d08i640m7i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715468000,"log":"[2026-02-17T14:55:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-volltfpza2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715469000,"log":"[2026-02-17T14:12:12Z DEBUG sqlx::query] SELECT FROM streams returned 193 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sptf5ypcjr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715470000,"log":"[2026-02-17T13:41:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8346857274131401000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-phuvbmipuv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715471000,"log":"[2026-02-17T14:21:40Z DEBUG sqlx::query] DELETE FROM alerts returned 972 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o4c491awik","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715472000,"log":"[2026-02-17T14:46:03Z INFO tonic::transport::server] grpc querier disconnected client_id=x0fz6x1j","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1n367k00s6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715473000,"log":"[2026-02-17T14:01:20Z INFO actix_web::middleware::logger] POST /api/_search 500 399ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-spux8akph3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715474000,"log":"[2026-02-17T14:17:28Z DEBUG sqlx::query] INSERT FROM users returned 388 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9g562egzc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715475000,"log":"[2026-02-17T14:35:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=s0trxy3q","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n3jy2vb2pn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715476000,"log":"[2026-02-17T13:35:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8352257069810907000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wet9w38ge7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715477000,"log":"[2026-02-17T12:49:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-74kpvfvzjy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715478000,"log":"[2026-02-17T14:12:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 716 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9gv6gcdo6p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715479000,"log":"[2026-02-17T13:34:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8895445119681187000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u3jevgjv3i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715480000,"log":"[2026-02-17T14:58:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4c6v3qikji","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715481000,"log":"[2026-02-17T15:15:37Z TRACE cache::redis] cache miss key=user:session:1255","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qqyeuz4t4w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715482000,"log":"[2026-02-17T13:39:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7907087509706978000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f3hzpp6303","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715483000,"log":"[2026-02-17T15:00:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o0721698ab","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715484000,"log":"[2026-02-17T15:06:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=746","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-46spw3qyws","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715485000,"log":"[2026-02-17T12:59:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lsm3euvq61","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715486000,"log":"[2026-02-17T13:26:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4155189346720369000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b5bljdf6ls","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715487000,"log":"[2026-02-17T14:23:08Z DEBUG sqlx::query] INSERT FROM users returned 640 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fvwhy59qwu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715488000,"log":"[2026-02-17T14:18:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 174 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qrb3bvxm8o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715489000,"log":"[2026-02-17T12:49:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zbzu8jsifo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715490000,"log":"[2026-02-17T12:53:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gxt8br97kr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715491000,"log":"[2026-02-17T12:49:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-escg5y0xo0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715492000,"log":"[2026-02-17T12:54:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qxiqklwt2z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715493000,"log":"[2026-02-17T13:25:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3396400567258077700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v0umdhvnxu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715494000,"log":"[2026-02-17T12:39:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rfuci5qrql","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715495000,"log":"[2026-02-17T15:00:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-45cnh31wkl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715496000,"log":"[2026-02-17T14:41:07Z INFO tonic::transport::server] grpc ingester connected client_id=mgx749hx","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qkf4p2m1mj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715497000,"log":"[2026-02-17T14:35:14Z INFO tonic::transport::server] grpc router request_received client_id=g7nlvkjd","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3hxj0e2yre","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715498000,"log":"[2026-02-17T12:45:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ok7l9js5gi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715499000,"log":"[2026-02-17T13:34:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9910355527615404000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oscja3iffz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715500000,"log":"[2026-02-17T14:03:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 403ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ez65a7616o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715501000,"log":"[2026-02-17T13:10:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3459329323894116000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4m08qvda0u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715502000,"log":"[2026-02-17T13:44:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 451ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l97lqsn1o3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715503000,"log":"[2026-02-17T13:30:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8437597937629630000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dghm1cx673","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715504000,"log":"[2026-02-17T14:08:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 376 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5b2l9alsjj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715505000,"log":"[2026-02-17T15:04:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z5q0xfi26e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715506000,"log":"[2026-02-17T13:33:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6656430013818788000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zn5cwtyvwb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715507000,"log":"[2026-02-17T13:29:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5214813800621548000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xzcalwxd2s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715508000,"log":"[2026-02-17T15:11:45Z DEBUG metrics::reporter] recorded metric active_connections value=2196","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r94g0n0tof","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715509000,"log":"[2026-02-17T15:20:02Z TRACE cache::redis] cache get key=user:session:1639","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c6hd24x10h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715510000,"log":"[2026-02-17T14:30:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 688 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-neteda9sux","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715511000,"log":"[2026-02-17T13:24:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3100605897620116500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-js2gntz0lh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715512000,"log":"[2026-02-17T12:45:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sjr7kdewm5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715513000,"log":"[2026-02-17T14:32:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=1fxtbxpd","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lh5vcngfqa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715514000,"log":"[2026-02-17T14:55:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mlsuk7a1zs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715515000,"log":"[2026-02-17T14:17:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 690 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4jv50v0oua","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715516000,"log":"[2026-02-17T13:18:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5533682528125703000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7wyqxrh6z1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715517000,"log":"[2026-02-17T15:20:36Z TRACE cache::redis] cache hit key=auth:token:9561","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mtss583clr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715518000,"log":"[2026-02-17T15:21:40Z TRACE cache::redis] cache delete key=query:result:2460","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ce6sm4ai2l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715519000,"log":"[2026-02-17T13:20:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8169893378372930000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a2de7th2b3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715520000,"log":"[2026-02-17T14:07:03Z DEBUG sqlx::query] INSERT FROM users returned 809 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-njrm49fyxi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715521000,"log":"[2026-02-17T13:40:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6547397117935163000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n3oayp5j7p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715522000,"log":"[2026-02-17T12:56:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2yl1lb351p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715523000,"log":"[2026-02-17T14:42:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=fbp24q6n","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-etti7c300n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715524000,"log":"[2026-02-17T13:27:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3577857951022509600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j2s9nhh5d7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715525000,"log":"[2026-02-17T14:44:38Z INFO tonic::transport::server] grpc querier disconnected client_id=mmdv65aa","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v9lxc96m41","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715526000,"log":"[2026-02-17T14:26:13Z DEBUG sqlx::query] INSERT FROM users returned 580 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tk719fsqzl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715527000,"log":"[2026-02-17T13:11:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7854825900730183000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-32leu25sej","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715528000,"log":"[2026-02-17T13:47:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 351ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-65lzvup5ta","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715529000,"log":"[2026-02-17T13:24:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8787720043575184000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2val0qft3k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715530000,"log":"[2026-02-17T14:13:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 347 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yi3q18rwpg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715531000,"log":"[2026-02-17T14:14:08Z DEBUG sqlx::query] INSERT FROM users returned 951 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c5bgz0w6av","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715532000,"log":"[2026-02-17T13:11:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6386563894875270000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vyc073ughj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715533000,"log":"[2026-02-17T13:34:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8275198756430441000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9sc666w942","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715534000,"log":"[2026-02-17T15:14:35Z TRACE cache::redis] cache expire key=query:result:8632","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-evz330w75k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715535000,"log":"[2026-02-17T13:42:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 14ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-trwdnj0z1t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715536000,"log":"[2026-02-17T14:56:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-24hipgy25g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715537000,"log":"[2026-02-17T14:16:53Z DEBUG sqlx::query] INSERT FROM users returned 308 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-avbk7rxuxy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715538000,"log":"[2026-02-17T14:36:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=1r1wy7kq","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-65ccktgu29","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715539000,"log":"[2026-02-17T13:42:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 460ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n5nlb28dsu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715540000,"log":"[2026-02-17T14:18:20Z DEBUG sqlx::query] DELETE FROM alerts returned 635 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wpr06vhvbm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715541000,"log":"[2026-02-17T13:40:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/463771156631089800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hzzsiydtph","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715542000,"log":"[2026-02-17T14:35:08Z INFO tonic::transport::server] grpc querier disconnected client_id=2w7t8znl","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mwdzvgkrvf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715543000,"log":"[2026-02-17T14:23:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 358 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1bh0p609rk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715544000,"log":"[2026-02-17T14:21:18Z DEBUG sqlx::query] INSERT FROM users returned 414 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-acabcb1snu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715545000,"log":"[2026-02-17T13:14:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9788186557762490000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tfzw0fjs4w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715546000,"log":"[2026-02-17T13:56:40Z INFO actix_web::middleware::logger] POST /api/_search 404 278ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9v9edi2jj7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715547000,"log":"[2026-02-17T14:47:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=zgjosbhq","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mqodiditta","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715548000,"log":"[2026-02-17T14:09:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 25 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uyki4ul5jt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715549000,"log":"[2026-02-17T15:21:22Z TRACE cache::redis] cache delete key=user:session:7167","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ikqd5exv7l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715550000,"log":"[2026-02-17T14:23:40Z DEBUG sqlx::query] DELETE FROM alerts returned 328 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-px7ge657hu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715551000,"log":"[2026-02-17T13:10:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2044409242589136000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e5mjbfjheu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715552000,"log":"[2026-02-17T13:09:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9364043966113083000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iv3teueuxx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715553000,"log":"[2026-02-17T14:36:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=7n9mw6dw","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-21cukc0zn1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715554000,"log":"[2026-02-17T13:57:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 343ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nplj3gabnm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715555000,"log":"[2026-02-17T14:44:58Z INFO tonic::transport::server] grpc querier disconnected client_id=gnum2kev","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iml7ejeyfs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715556000,"log":"[2026-02-17T13:22:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6783389424866291000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-20iodryj9y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715557000,"log":"[2026-02-17T12:55:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u88j7s6uwm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715558000,"log":"[2026-02-17T14:58:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-96371seto3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715559000,"log":"[2026-02-17T13:51:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 141ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mwe9st84mo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715560000,"log":"[2026-02-17T14:27:38Z DEBUG sqlx::query] INSERT FROM users returned 680 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ux3iq3zprm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715561000,"log":"[2026-02-17T13:05:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8ev4z32t3d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715562000,"log":"[2026-02-17T12:38:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-txzw5cyz8u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715563000,"log":"[2026-02-17T13:52:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 319ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e17xkjjjbd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715564000,"log":"[2026-02-17T12:36:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-73bdwmgihm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715565000,"log":"[2026-02-17T13:45:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 452ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0tkqeex81c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715566000,"log":"[2026-02-17T14:24:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 785 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1hs5ik6wut","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715567000,"log":"[2026-02-17T13:48:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 25ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jzi011fihh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715568000,"log":"[2026-02-17T14:13:58Z DEBUG sqlx::query] INSERT FROM users returned 511 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vzk48l20k6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715569000,"log":"[2026-02-17T14:14:52Z DEBUG sqlx::query] SELECT FROM streams returned 817 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2cyiw5cp1p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715570000,"log":"[2026-02-17T13:19:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/326458589294242900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s8gpqq52qc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715571000,"log":"[2026-02-17T14:03:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 412ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-klz73wmo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715572000,"log":"[2026-02-17T12:40:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s0mi624ixw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715573000,"log":"[2026-02-17T14:26:17Z DEBUG sqlx::query] SELECT FROM streams returned 697 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j3mz5urcoe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715574000,"log":"[2026-02-17T15:18:28Z TRACE cache::redis] cache delete key=org:config:8011","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1zoj6pxq0m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715575000,"log":"[2026-02-17T13:47:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 283ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mec4xroqwj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715576000,"log":"[2026-02-17T14:09:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 769 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sipuo4c13s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715577000,"log":"[2026-02-17T14:25:07Z DEBUG sqlx::query] SELECT FROM streams returned 580 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uu7lr3x5to","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715578000,"log":"[2026-02-17T14:33:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=0vb961ew","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3oo16zjwt9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715579000,"log":"[2026-02-17T14:54:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-equnzd1enu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715580000,"log":"[2026-02-17T13:52:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 273ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-myzmemqycx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715581000,"log":"[2026-02-17T15:12:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2136","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5hy04gl6yk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715582000,"log":"[2026-02-17T15:03:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wmb4yf36oc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715583000,"log":"[2026-02-17T12:56:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t7z7k8fo3g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715584000,"log":"[2026-02-17T14:17:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 135 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-miz3mg4csg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715585000,"log":"[2026-02-17T14:06:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 397ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a321p1feuj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715586000,"log":"[2026-02-17T14:29:02Z DEBUG sqlx::query] SELECT FROM streams returned 395 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7mxanuwuqc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715587000,"log":"[2026-02-17T14:15:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 420 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0znryrzdk8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715588000,"log":"[2026-02-17T12:49:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t3shsnkjhy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715589000,"log":"[2026-02-17T13:51:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 220ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n483w0s8nc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715590000,"log":"[2026-02-17T14:52:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vd87idv8cm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715591000,"log":"[2026-02-17T14:50:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-my1k2oe6n1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715592000,"log":"[2026-02-17T12:44:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-agmosrif54","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715593000,"log":"[2026-02-17T14:41:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=uj2qvoam","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e7tc79nvwc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715594000,"log":"[2026-02-17T15:13:53Z TRACE cache::redis] cache expire key=org:config:5624","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yrmk97i3x1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715595000,"log":"[2026-02-17T15:19:36Z TRACE cache::redis] cache hit key=auth:token:7936","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dmxkde8fsb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715596000,"log":"[2026-02-17T13:06:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ibttgss1pn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715597000,"log":"[2026-02-17T13:09:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6558051292893912000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0cm8vhuvis","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715598000,"log":"[2026-02-17T15:16:18Z TRACE cache::redis] cache hit key=org:config:6188","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1rr8kg4eyt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715599000,"log":"[2026-02-17T14:28:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 502 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-punyotff8b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715600000,"log":"[2026-02-17T15:19:25Z TRACE cache::redis] cache miss key=query:result:1087","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-la9qrzdzaq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715601000,"log":"[2026-02-17T14:36:23Z INFO tonic::transport::server] grpc querier disconnected client_id=u71nm2o1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p600vz70sy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715602000,"log":"[2026-02-17T12:47:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ks0mv66zoe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715603000,"log":"[2026-02-17T14:41:37Z INFO tonic::transport::server] grpc ingester connected client_id=ye6bs83n","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qdgnmpft4p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715604000,"log":"[2026-02-17T14:21:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 13 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3swd7pmfj8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715605000,"log":"[2026-02-17T13:08:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rgxfvco28b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715606000,"log":"[2026-02-17T14:49:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rrn8gaa23q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715607000,"log":"[2026-02-17T14:21:45Z DEBUG sqlx::query] DELETE FROM alerts returned 125 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f7g0b0j7c8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715608000,"log":"[2026-02-17T14:04:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 247ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3e77fx0l21","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715609000,"log":"[2026-02-17T12:50:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uh9mirmeqh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715610000,"log":"[2026-02-17T14:41:38Z INFO tonic::transport::server] grpc querier disconnected client_id=jgbc82ve","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fn6d32y9kc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715611000,"log":"[2026-02-17T14:58:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iaj5jsq6tl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715612000,"log":"[2026-02-17T15:06:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=853","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bg8vfdzjvr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715613000,"log":"[2026-02-17T13:22:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6988647698659201000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nhbvouuwjv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715614000,"log":"[2026-02-17T13:34:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1311988919728068900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kfrzs7wwk6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715615000,"log":"[2026-02-17T13:35:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7605058481856654000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tiveytb1tz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715616000,"log":"[2026-02-17T13:29:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6008669834111364000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nnbtyxz4rg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715617000,"log":"[2026-02-17T15:08:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=9770","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i52daxrawi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715618000,"log":"[2026-02-17T15:13:36Z TRACE cache::redis] cache hit key=auth:token:1078","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kbpyuua6bt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715619000,"log":"[2026-02-17T14:03:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 177ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rugx6gnk9c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715620000,"log":"[2026-02-17T15:11:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9348","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b0ey42pxv3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715621000,"log":"[2026-02-17T15:15:45Z TRACE cache::redis] cache set key=query:result:8122","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lk9wsvj3w2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715622000,"log":"[2026-02-17T14:32:57Z INFO tonic::transport::server] grpc ingester connected client_id=qz9ewxf0","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-syqai0lluv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715623000,"log":"[2026-02-17T12:53:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mcjre0fo32","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715624000,"log":"[2026-02-17T13:28:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7799756298646905000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ikhcf1qw1v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715625000,"log":"[2026-02-17T12:41:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ivy3jda0v5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715626000,"log":"[2026-02-17T13:25:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9738473117012736000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t1trh6epwi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715627000,"log":"[2026-02-17T14:30:22Z DEBUG sqlx::query] SELECT FROM streams returned 398 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t8930841uk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715628000,"log":"[2026-02-17T13:52:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 485ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r7m6e3bzz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715629000,"log":"[2026-02-17T13:32:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6042125714270086000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2en6nb7nor","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715630000,"log":"[2026-02-17T12:54:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kggpdlhhtj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715631000,"log":"[2026-02-17T12:46:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vql4x4l2i3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715632000,"log":"[2026-02-17T12:52:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-emu3f9xl9s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715633000,"log":"[2026-02-17T14:07:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 983 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5tniepknnq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715634000,"log":"[2026-02-17T13:59:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 174ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-va5xdgz4j9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715635000,"log":"[2026-02-17T13:45:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 166ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5wyp3t6gba","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715636000,"log":"[2026-02-17T13:27:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1693972352788508700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-esf9f5jrao","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715637000,"log":"[2026-02-17T13:25:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1831784091863917300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hy8uq1xaj3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715638000,"log":"[2026-02-17T14:37:08Z INFO tonic::transport::server] grpc querier disconnected client_id=rbzq0mml","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c6a44y6s3c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715639000,"log":"[2026-02-17T14:10:08Z DEBUG sqlx::query] INSERT FROM users returned 587 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kmdh2dcbzz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715640000,"log":"[2026-02-17T13:15:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2318113214494371000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r9haxmuggv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715641000,"log":"[2026-02-17T13:26:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/108517828232075340.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7xl3kvabv7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715642000,"log":"[2026-02-17T12:44:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4qd3xzy8ep","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715643000,"log":"[2026-02-17T13:10:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3991218757408104000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sckrt1e66g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715644000,"log":"[2026-02-17T14:11:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 197 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-31v18qa652","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715645000,"log":"[2026-02-17T14:29:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 727 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3ppjcqz8ew","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715646000,"log":"[2026-02-17T14:28:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 909 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f3zcfqiugi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715647000,"log":"[2026-02-17T15:12:35Z DEBUG metrics::reporter] recorded metric active_connections value=4404","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9osf7t2myf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715648000,"log":"[2026-02-17T13:23:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8122204815816667000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k6sayyjp7c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715649000,"log":"[2026-02-17T14:41:52Z INFO tonic::transport::server] grpc ingester connected client_id=7ga06jrb","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6hyrni8djh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715650000,"log":"[2026-02-17T12:43:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g0b2zrfweu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715651000,"log":"[2026-02-17T15:12:25Z DEBUG metrics::reporter] recorded metric active_connections value=709","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xjgqw7js9y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715652000,"log":"[2026-02-17T13:16:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9454551429162340000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m15vn8p1pk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715653000,"log":"[2026-02-17T14:54:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s4i1ilcrli","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715654000,"log":"[2026-02-17T13:39:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7767168104143092000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1mr3deeif6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715655000,"log":"[2026-02-17T13:05:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3m55xbd4g0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715656000,"log":"[2026-02-17T12:47:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c9e8pjvs6y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715657000,"log":"[2026-02-17T15:16:05Z TRACE cache::redis] cache expire key=query:result:6076","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p4otcq8bb6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715658000,"log":"[2026-02-17T15:10:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1912","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vu7muv9pcf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715659000,"log":"[2026-02-17T13:16:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5832204129456526000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8t9unk9s1a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715660000,"log":"[2026-02-17T12:49:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-57hzdyawkw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715661000,"log":"[2026-02-17T12:57:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sheagvld0g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715662000,"log":"[2026-02-17T13:32:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5180039931374165000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4423094eu0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715663000,"log":"[2026-02-17T13:32:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2994266794189944000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5woig6i8xf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715664000,"log":"[2026-02-17T15:05:25Z DEBUG metrics::reporter] recorded metric active_connections value=6693","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7gf4o98wa9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715665000,"log":"[2026-02-17T13:00:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-juunbimniz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715666000,"log":"[2026-02-17T14:07:27Z DEBUG sqlx::query] SELECT FROM streams returned 353 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u1ygj4hkjy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715667000,"log":"[2026-02-17T14:59:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mkk9tf9wvf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715668000,"log":"[2026-02-17T15:13:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=7894","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qplqfonasb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715669000,"log":"[2026-02-17T15:03:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dqn1yrlxsm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715670000,"log":"[2026-02-17T13:52:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 150ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9s6at6bm09","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715671000,"log":"[2026-02-17T14:59:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gpkt9jvltw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715672000,"log":"[2026-02-17T14:06:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 106ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tn8xuoc3lf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715673000,"log":"[2026-02-17T13:22:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2903922590115926000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-917qlsy8ey","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715674000,"log":"[2026-02-17T12:48:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wk2bgohdh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715675000,"log":"[2026-02-17T13:18:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7686012532317990000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x6k9d2ajq9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715676000,"log":"[2026-02-17T15:17:29Z TRACE cache::redis] cache expire key=stream:schema:9215","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mgb7vlktdc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715677000,"log":"[2026-02-17T14:54:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4kan2f2wi1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715678000,"log":"[2026-02-17T13:01:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5iusgl4d4r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715679000,"log":"[2026-02-17T13:32:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5787348430054611000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6sosf7flwq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715680000,"log":"[2026-02-17T13:26:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2987416926301003300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mss8hvghsw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715681000,"log":"[2026-02-17T14:03:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 26ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3sdu5x1d2n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715682000,"log":"[2026-02-17T14:11:15Z DEBUG sqlx::query] DELETE FROM alerts returned 850 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vmr74emaju","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715683000,"log":"[2026-02-17T12:42:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mr7c3jim6w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715684000,"log":"[2026-02-17T14:22:37Z DEBUG sqlx::query] SELECT FROM streams returned 907 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8pr6616n8u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715685000,"log":"[2026-02-17T15:05:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-szvlhbmays","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715686000,"log":"[2026-02-17T12:42:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cdsdhp6mkz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715687000,"log":"[2026-02-17T14:17:00Z DEBUG sqlx::query] DELETE FROM alerts returned 949 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t31ey2r42r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715688000,"log":"[2026-02-17T13:24:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9988710200906572000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c7e727ucps","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715689000,"log":"[2026-02-17T15:00:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nhcthyclq4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715690000,"log":"[2026-02-17T14:05:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 256ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rvdrd4qgde","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715691000,"log":"[2026-02-17T14:17:57Z DEBUG sqlx::query] SELECT FROM streams returned 720 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vq2i4m4oe9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715692000,"log":"[2026-02-17T12:48:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fdrgv5k8as","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715693000,"log":"[2026-02-17T13:30:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7830285414875302000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hnw9z91st6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715694000,"log":"[2026-02-17T14:46:43Z INFO tonic::transport::server] grpc querier disconnected client_id=og15y2l1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2oobomqs7f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715695000,"log":"[2026-02-17T15:02:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-50swbh5jy2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715696000,"log":"[2026-02-17T13:32:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5578764327857022000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5cwo7jjz3r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715697000,"log":"[2026-02-17T12:49:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vi495xmxv0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715698000,"log":"[2026-02-17T13:08:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1375788233183095000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gfvb44zpo4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715699000,"log":"[2026-02-17T14:06:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 168ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m86bpaexql","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715700000,"log":"[2026-02-17T12:41:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uzhoj1gm61","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715701000,"log":"[2026-02-17T14:41:34Z INFO tonic::transport::server] grpc router request_received client_id=cexcpr5a","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-43kue12955","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715702000,"log":"[2026-02-17T13:35:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4419713260698845000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nz8h3clqwc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715703000,"log":"[2026-02-17T15:10:15Z DEBUG metrics::reporter] recorded metric active_connections value=662","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sn8lx9yed4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715704000,"log":"[2026-02-17T13:28:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/803373288908693100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sxro3eiuar","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715705000,"log":"[2026-02-17T15:12:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=3707","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-smy6fhmit2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715706000,"log":"[2026-02-17T12:36:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-es4rfthklm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715707000,"log":"[2026-02-17T13:01:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8o3m38nrbg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715708000,"log":"[2026-02-17T12:48:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rijl4380zh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715709000,"log":"[2026-02-17T14:27:40Z DEBUG sqlx::query] DELETE FROM alerts returned 127 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w971a5g3tr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715710000,"log":"[2026-02-17T13:06:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t3c5knwred","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715711000,"log":"[2026-02-17T13:10:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2742687663630767000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-78cz2xcuma","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715712000,"log":"[2026-02-17T12:54:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-takyqgpsxt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715713000,"log":"[2026-02-17T14:17:18Z DEBUG sqlx::query] INSERT FROM users returned 907 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-trgx46egiv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715714000,"log":"[2026-02-17T15:07:25Z DEBUG metrics::reporter] recorded metric active_connections value=5961","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ntfhnsyma","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715715000,"log":"[2026-02-17T14:30:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 248 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mnzr1t6wuh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715716000,"log":"[2026-02-17T13:39:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8859345441899240000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-07jrxy1l4l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715717000,"log":"[2026-02-17T14:59:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ax20qp46wj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715718000,"log":"[2026-02-17T13:05:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ab66he53vo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715719000,"log":"[2026-02-17T12:49:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ao7inngsq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715720000,"log":"[2026-02-17T13:23:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/248842416525252300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-35ttcplr23","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715721000,"log":"[2026-02-17T14:23:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 631 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m2ybzec95h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715722000,"log":"[2026-02-17T15:04:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lmbywp41jb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715723000,"log":"[2026-02-17T15:03:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zwo3l3md2u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715724000,"log":"[2026-02-17T12:59:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cwzcdg5jxf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715725000,"log":"[2026-02-17T14:48:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fjatjy0g30","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715726000,"log":"[2026-02-17T14:11:05Z DEBUG sqlx::query] DELETE FROM alerts returned 680 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9uegs9vnu9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715727000,"log":"[2026-02-17T13:37:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3477299340826379000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w3fp18qjag","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715728000,"log":"[2026-02-17T14:34:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=6t9wbgi0","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yijg8vm7ai","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715729000,"log":"[2026-02-17T13:01:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fuky2tgtvq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715730000,"log":"[2026-02-17T13:55:45Z INFO actix_web::middleware::logger] GET /api/_search 403 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2yrxgpwv0v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715731000,"log":"[2026-02-17T13:02:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8fo4algnuz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715732000,"log":"[2026-02-17T13:22:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7090017514741633000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-68xaqxpr3c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715733000,"log":"[2026-02-17T15:02:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d88etwurxy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715734000,"log":"[2026-02-17T12:51:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qvos9euueb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715735000,"log":"[2026-02-17T13:07:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ax405473fn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715736000,"log":"[2026-02-17T13:49:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 359ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x6xdjnp0os","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715737000,"log":"[2026-02-17T14:09:50Z DEBUG sqlx::query] DELETE FROM alerts returned 97 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hntcqtsc70","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715738000,"log":"[2026-02-17T12:41:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5a6cd38jrd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715739000,"log":"[2026-02-17T13:07:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h19e70smy4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715740000,"log":"[2026-02-17T13:15:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8821440125815955000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uu1vzrbov1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715741000,"log":"[2026-02-17T13:49:00Z INFO actix_web::middleware::logger] POST /api/_search 403 246ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uoqa0aixht","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715742000,"log":"[2026-02-17T13:24:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4661262700495317000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qmn15dza3v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715743000,"log":"[2026-02-17T13:10:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6993391394824693000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lopgjmbl8t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715744000,"log":"[2026-02-17T14:42:44Z INFO tonic::transport::server] grpc router request_received client_id=jwkfeadb","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m5pazp5774","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715745000,"log":"[2026-02-17T15:10:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=124","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jfphwwgahe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715746000,"log":"[2026-02-17T15:04:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dnz1xx4or1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715747000,"log":"[2026-02-17T12:56:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0x7ixyke9h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715748000,"log":"[2026-02-17T15:08:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1269","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0s44qvcv5s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715749000,"log":"[2026-02-17T14:00:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 242ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x6gn5n6k1r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715750000,"log":"[2026-02-17T12:46:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5s0zq8ks67","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715751000,"log":"[2026-02-17T12:39:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4k4hwiu63f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715752000,"log":"[2026-02-17T15:08:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8800","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w482otpytq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715753000,"log":"[2026-02-17T15:18:08Z TRACE cache::redis] cache get key=org:config:7806","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ziiobnfcwy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715754000,"log":"[2026-02-17T13:01:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-42xlv6hpfn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715755000,"log":"[2026-02-17T14:14:20Z DEBUG sqlx::query] DELETE FROM alerts returned 248 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eebqdm19hy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715756000,"log":"[2026-02-17T14:34:12Z INFO tonic::transport::server] grpc ingester connected client_id=dxofoh7w","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nb4kka6pyf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715757000,"log":"[2026-02-17T14:00:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 189ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5m2i03wbv1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715758000,"log":"[2026-02-17T12:37:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p2oyb32p37","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715759000,"log":"[2026-02-17T13:41:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1206425609102286300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6io9rkc48y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715760000,"log":"[2026-02-17T15:07:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5396","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7hyukm678c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715761000,"log":"[2026-02-17T13:15:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2023787520651321000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-acyx4yssqh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715762000,"log":"[2026-02-17T13:26:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3578751189147952600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mf0sa72jyr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715763000,"log":"[2026-02-17T13:01:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5v3tjzssy5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715764000,"log":"[2026-02-17T15:12:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=5406","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gectzyc5p8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715765000,"log":"[2026-02-17T15:16:06Z TRACE cache::redis] cache hit key=auth:token:3560","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-60gxkn6sl1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715766000,"log":"[2026-02-17T14:18:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 965 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f1ro6riska","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715767000,"log":"[2026-02-17T13:33:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7297536023348059000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8sp6wjx8av","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715768000,"log":"[2026-02-17T14:46:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=gbdf9y92","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hk689rqfgb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715769000,"log":"[2026-02-17T14:24:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 358 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u6r6ncepxt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715770000,"log":"[2026-02-17T13:33:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3747245034609458700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-19vfqu3f8o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715771000,"log":"[2026-02-17T14:27:42Z DEBUG sqlx::query] SELECT FROM streams returned 494 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q18xf5x7ce","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715772000,"log":"[2026-02-17T13:46:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 244ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-91oumznu6v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715773000,"log":"[2026-02-17T13:19:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7997541194388111000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fy8qr8gd4r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715774000,"log":"[2026-02-17T12:49:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7gtx895vrz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715775000,"log":"[2026-02-17T13:37:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7193090102171449000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9it8xatbj2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715776000,"log":"[2026-02-17T15:00:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6e27brekom","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715777000,"log":"[2026-02-17T15:12:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=706","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mi2vrsgbqn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715778000,"log":"[2026-02-17T14:57:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jvyyjp2d7e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715779000,"log":"[2026-02-17T14:45:14Z INFO tonic::transport::server] grpc router request_received client_id=mqfv0a1a","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m4ecxgo3wc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715780000,"log":"[2026-02-17T14:18:12Z DEBUG sqlx::query] SELECT FROM streams returned 210 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m4xgtz11v2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715781000,"log":"[2026-02-17T13:15:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5800701510458178000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u9infd50i9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715782000,"log":"[2026-02-17T13:10:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/115703950469983060.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rt3ar9cnyd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715783000,"log":"[2026-02-17T13:21:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9577041963190847000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8vebc8qxhm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715784000,"log":"[2026-02-17T15:11:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2901","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-umrunn92vh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715785000,"log":"[2026-02-17T14:28:42Z DEBUG sqlx::query] SELECT FROM streams returned 281 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-92215i26go","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715786000,"log":"[2026-02-17T12:59:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6n9uf6ex7e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715787000,"log":"[2026-02-17T15:13:51Z TRACE cache::redis] cache set key=auth:token:7184","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zk6wbms6cu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715788000,"log":"[2026-02-17T14:45:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=0qmypy1b","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-as313xn7fj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715789000,"log":"[2026-02-17T14:02:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 29ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1c2dvir3l7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715790000,"log":"[2026-02-17T13:54:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 468ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yok4or6jsf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715791000,"log":"[2026-02-17T12:56:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-syih7drcg9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715792000,"log":"[2026-02-17T13:59:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 472ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2jrmd089m8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715793000,"log":"[2026-02-17T13:11:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9907752143231113000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yxv3yy615a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715794000,"log":"[2026-02-17T13:48:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 300ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lczgy634uk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715795000,"log":"[2026-02-17T14:33:13Z INFO tonic::transport::server] grpc querier disconnected client_id=a6zsd3sy","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-29mq7isz95","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715796000,"log":"[2026-02-17T13:55:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 494ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vt076e4azs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715797000,"log":"[2026-02-17T13:06:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xgp78p3g61","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715798000,"log":"[2026-02-17T13:38:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6393915998791833000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b68i025tpc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715799000,"log":"[2026-02-17T12:42:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1i6wzh7zdw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715800000,"log":"[2026-02-17T13:37:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6772764848911051000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-50ocdzqxf9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715801000,"log":"[2026-02-17T13:59:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 307ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oniatnqwkr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715802000,"log":"[2026-02-17T13:04:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tnxms8g22w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715803000,"log":"[2026-02-17T15:07:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6748","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kwlun79cq9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715804000,"log":"[2026-02-17T15:13:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8553","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8bx8b9yh4e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715805000,"log":"[2026-02-17T14:52:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ix1w14cpyy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715806000,"log":"[2026-02-17T15:14:57Z TRACE cache::redis] cache set key=user:session:8496","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pephho1xb6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715807000,"log":"[2026-02-17T15:15:51Z TRACE cache::redis] cache set key=auth:token:7909","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wmhyovz3ql","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715808000,"log":"[2026-02-17T15:13:46Z TRACE cache::redis] cache delete key=auth:token:7438","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ozzq56m01","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715809000,"log":"[2026-02-17T12:54:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u9i6xlzsrf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715810000,"log":"[2026-02-17T14:02:45Z INFO actix_web::middleware::logger] GET /api/_search 204 309ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fnhpz49sq9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715811000,"log":"[2026-02-17T14:30:32Z DEBUG sqlx::query] SELECT FROM streams returned 136 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j501q3v153","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715812000,"log":"[2026-02-17T12:43:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kds2vdj91i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715813000,"log":"[2026-02-17T14:51:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-80b4jkw7ft","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715814000,"log":"[2026-02-17T13:28:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9490631751621173000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x2qsvqyki1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715815000,"log":"[2026-02-17T14:53:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-etxtb7dv0n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715816000,"log":"[2026-02-17T13:31:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1663564936624715300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rm72dsa35","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715817000,"log":"[2026-02-17T13:49:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 214ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rqo21v9prb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715818000,"log":"[2026-02-17T15:11:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6191","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uxmw0hyznh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715819000,"log":"[2026-02-17T13:45:00Z INFO actix_web::middleware::logger] POST /api/_search 503 173ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tw0lomtc87","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715820000,"log":"[2026-02-17T13:01:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6b5p9g6wzi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715821000,"log":"[2026-02-17T14:57:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cr8n18oext","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715822000,"log":"[2026-02-17T14:35:59Z INFO tonic::transport::server] grpc router request_received client_id=xfrxidg5","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p2noxlvkyk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715823000,"log":"[2026-02-17T13:05:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-29clgm9d0c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715824000,"log":"[2026-02-17T14:46:28Z INFO tonic::transport::server] grpc querier disconnected client_id=0rffbq0y","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wxyteyw189","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715825000,"log":"[2026-02-17T12:41:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f54wnspiqf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715826000,"log":"[2026-02-17T13:11:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7082924362340730000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p2sx764cfz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715827000,"log":"[2026-02-17T13:51:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 381ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1yroicrlc9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715828000,"log":"[2026-02-17T14:01:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 461ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cngs8q8ry0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715829000,"log":"[2026-02-17T13:54:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 392ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-272lyy989i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715830000,"log":"[2026-02-17T13:18:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1838825693353185500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7x58h1sj3j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715831000,"log":"[2026-02-17T13:00:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m5iasmyz65","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715832000,"log":"[2026-02-17T13:25:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1055528964492051800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bu6b7ayne2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715833000,"log":"[2026-02-17T15:14:11Z TRACE cache::redis] cache expire key=auth:token:6325","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6fwe2k82s4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715834000,"log":"[2026-02-17T12:54:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3tjcjr6lfv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715835000,"log":"[2026-02-17T14:17:22Z DEBUG sqlx::query] SELECT FROM streams returned 725 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jlbpwhszo6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715836000,"log":"[2026-02-17T14:00:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 496ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-twy58c60e3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715837000,"log":"[2026-02-17T12:49:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6s3wswpw9d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715838000,"log":"[2026-02-17T14:32:12Z INFO tonic::transport::server] grpc ingester connected client_id=vxsbd5gp","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ly7kqs5lmi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715839000,"log":"[2026-02-17T13:38:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3226929923538344000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wqdutfwgyo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715840000,"log":"[2026-02-17T14:17:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 722 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dj1wkmpzqi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715841000,"log":"[2026-02-17T13:45:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 302ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mhw7ezqs1g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715842000,"log":"[2026-02-17T15:02:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ix87un75oo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715843000,"log":"[2026-02-17T14:56:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l4dxvpxexp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715844000,"log":"[2026-02-17T13:32:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5927165273943222000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xpd4ttdzhc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715845000,"log":"[2026-02-17T14:54:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k9uzr4kjmo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715846000,"log":"[2026-02-17T14:38:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=9zh1jjbs","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4lf7n9ctt4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715847000,"log":"[2026-02-17T12:42:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nuxqy2aapq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715848000,"log":"[2026-02-17T13:41:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7403761798832502000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3zdqimzbl5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715849000,"log":"[2026-02-17T13:55:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 284ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ynjzaq5hm9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715850000,"log":"[2026-02-17T13:37:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8816586207998125000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-174m95gvrr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715851000,"log":"[2026-02-17T13:31:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6321580583028308000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1cibjqotqq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715852000,"log":"[2026-02-17T12:53:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-agublaml5c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715853000,"log":"[2026-02-17T13:59:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 244ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kb36a02nrj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715854000,"log":"[2026-02-17T13:45:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 174ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w2nopmoxt6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715855000,"log":"[2026-02-17T14:27:20Z DEBUG sqlx::query] DELETE FROM alerts returned 182 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bkpkzjnz8m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715856000,"log":"[2026-02-17T12:55:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ufmaididgc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715857000,"log":"[2026-02-17T14:16:03Z DEBUG sqlx::query] INSERT FROM users returned 98 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-diod92xcf6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715858000,"log":"[2026-02-17T13:26:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6414009519932084000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wy37xbjzao","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715859000,"log":"[2026-02-17T14:35:03Z INFO tonic::transport::server] grpc querier disconnected client_id=68pvkt5o","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1t0imxborc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715860000,"log":"[2026-02-17T14:57:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o8fmdf5rft","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715861000,"log":"[2026-02-17T13:20:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2582610274868990500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-117igix4nc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715862000,"log":"[2026-02-17T14:14:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 835 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vcmb1oipn9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715863000,"log":"[2026-02-17T13:59:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 393ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-em38zqqpbl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715864000,"log":"[2026-02-17T13:39:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3093957237879172600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-98piedakcq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715865000,"log":"[2026-02-17T12:54:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4r26srb9j5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715866000,"log":"[2026-02-17T12:43:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tgvv95odnw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715867000,"log":"[2026-02-17T14:38:09Z INFO tonic::transport::server] grpc router request_received client_id=ox7u5s2d","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vifjf8tsrb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715868000,"log":"[2026-02-17T13:13:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3278534374662627300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9oss69h6wo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715869000,"log":"[2026-02-17T14:53:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4g7ftn3yq7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715870000,"log":"[2026-02-17T15:00:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lw7z43r805","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715871000,"log":"[2026-02-17T13:16:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7623704268791796000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9r1tp8w0av","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715872000,"log":"[2026-02-17T13:19:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4185556666304966000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nyt5nx6bx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715873000,"log":"[2026-02-17T14:31:10Z DEBUG sqlx::query] DELETE FROM alerts returned 5 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aahx35bxho","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715874000,"log":"[2026-02-17T15:17:40Z TRACE cache::redis] cache delete key=query:result:4616","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w6jacqa3i5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715875000,"log":"[2026-02-17T13:14:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6497709996647723000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6a4kcrz0v2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715876000,"log":"[2026-02-17T14:32:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=x4xv6l54","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5p0rci6snv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715877000,"log":"[2026-02-17T14:06:00Z INFO actix_web::middleware::logger] POST /api/_search 503 385ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3u7qm1ch7q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715878000,"log":"[2026-02-17T15:17:00Z TRACE cache::redis] cache hit key=query:result:9896","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-evfan2ihmo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715879000,"log":"[2026-02-17T15:14:01Z TRACE cache::redis] cache miss key=auth:token:2226","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ul0o1u5btf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715880000,"log":"[2026-02-17T14:16:22Z DEBUG sqlx::query] SELECT FROM streams returned 864 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qanikumoac","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715881000,"log":"[2026-02-17T15:11:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7624","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rwidvkz0y6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715882000,"log":"[2026-02-17T13:25:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8977110223862885000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-27pb4aqodq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715883000,"log":"[2026-02-17T13:36:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1125395158664526100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k2fw0ytj0w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715884000,"log":"[2026-02-17T13:35:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6532410867324607000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hv4xff7erh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715885000,"log":"[2026-02-17T14:10:32Z DEBUG sqlx::query] SELECT FROM streams returned 441 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wkj9rt3m7p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715886000,"log":"[2026-02-17T15:05:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-87p02o5zzu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715887000,"log":"[2026-02-17T15:00:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ffd1oorjiq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715888000,"log":"[2026-02-17T13:46:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 376ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q20g6obr4j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715889000,"log":"[2026-02-17T14:45:47Z INFO tonic::transport::server] grpc ingester connected client_id=z1vp3l1r","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s41ul8gonx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715890000,"log":"[2026-02-17T12:43:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9yjs2pvy8h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715891000,"log":"[2026-02-17T14:36:39Z INFO tonic::transport::server] grpc router request_received client_id=zuy7frzi","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tvus0vpnhw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715892000,"log":"[2026-02-17T14:28:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 725 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mxzm7oiqmk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715893000,"log":"[2026-02-17T15:19:49Z TRACE cache::redis] cache miss key=stream:schema:5787","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zj2yc2t4c1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715894000,"log":"[2026-02-17T14:39:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=qjetub6l","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8axc64zjm1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715895000,"log":"[2026-02-17T13:13:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7581776391404822000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-43m71cbbpp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715896000,"log":"[2026-02-17T13:22:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/475679210632820030.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fb5ac90s2l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715897000,"log":"[2026-02-17T15:19:48Z TRACE cache::redis] cache hit key=org:config:7782","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8e4dzd63rz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715898000,"log":"[2026-02-17T13:42:40Z INFO actix_web::middleware::logger] POST /api/_search 400 306ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yxl038o2p5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715899000,"log":"[2026-02-17T14:36:03Z INFO tonic::transport::server] grpc querier disconnected client_id=wn96ejgn","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2qf6fpiveh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715900000,"log":"[2026-02-17T14:45:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=zmp7zw7m","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cag5l5480f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715901000,"log":"[2026-02-17T13:08:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cb313f0pci","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715902000,"log":"[2026-02-17T14:54:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1d7chiiva1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715903000,"log":"[2026-02-17T14:06:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 184ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2t08zxb100","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715904000,"log":"[2026-02-17T12:40:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i6pphnuc4i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715905000,"log":"[2026-02-17T15:15:14Z TRACE cache::redis] cache get key=stream:schema:2355","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k79wz3lyjh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715906000,"log":"[2026-02-17T13:27:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5184040945183628000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3a7lru2j9m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715907000,"log":"[2026-02-17T14:42:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=c4uj5925","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vxf7e7k5ev","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715908000,"log":"[2026-02-17T13:38:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4252687461139799000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sie0z0wgfk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715909000,"log":"[2026-02-17T14:13:35Z DEBUG sqlx::query] DELETE FROM alerts returned 262 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xecbejkr0c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715910000,"log":"[2026-02-17T15:03:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2388rlxqlf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715911000,"log":"[2026-02-17T14:08:00Z DEBUG sqlx::query] DELETE FROM alerts returned 216 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kw0l1ba3v5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715912000,"log":"[2026-02-17T13:12:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5294131464454095000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mjlcryhtfv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715913000,"log":"[2026-02-17T14:10:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 624 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4mhpi7zf1x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715914000,"log":"[2026-02-17T12:53:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f69rpecrx6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715915000,"log":"[2026-02-17T12:48:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nctcd74da6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715916000,"log":"[2026-02-17T14:32:32Z INFO tonic::transport::server] grpc ingester connected client_id=nrtiigmc","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-706niaexws","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715917000,"log":"[2026-02-17T14:17:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 894 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sq71brte6x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715918000,"log":"[2026-02-17T13:35:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7943770142391460000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iow1vkbszp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715919000,"log":"[2026-02-17T14:18:45Z DEBUG sqlx::query] DELETE FROM alerts returned 497 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7w04sibxsf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715920000,"log":"[2026-02-17T13:14:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/584189007473832100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5kwpcg53qz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715921000,"log":"[2026-02-17T15:01:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nluv6c33yq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715922000,"log":"[2026-02-17T14:29:08Z DEBUG sqlx::query] INSERT FROM users returned 90 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eky2p6bf98","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715923000,"log":"[2026-02-17T13:46:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 455ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n785978fz1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715924000,"log":"[2026-02-17T14:46:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=qlbxot9y","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6hbhwg9u5q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715925000,"log":"[2026-02-17T13:16:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9192290826207059000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-heh51vd68m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715926000,"log":"[2026-02-17T14:50:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wa7wd2o9as","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715927000,"log":"[2026-02-17T12:47:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tac0r86jm2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715928000,"log":"[2026-02-17T14:56:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eef5ewxhx2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715929000,"log":"[2026-02-17T14:01:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 345ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-th9rhws69f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715930000,"log":"[2026-02-17T13:29:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6951353826714144000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g54is61ugj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715931000,"log":"[2026-02-17T15:13:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9820","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dc08neir19","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715932000,"log":"[2026-02-17T13:46:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 269ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zxheny5bre","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715933000,"log":"[2026-02-17T12:56:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w7vfdl7kuc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715934000,"log":"[2026-02-17T13:39:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7642692813857097000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1mwv53jwqo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715935000,"log":"[2026-02-17T13:17:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/566863029850057660.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lbhtunsrbr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715936000,"log":"[2026-02-17T12:41:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bflgb3pgy6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715937000,"log":"[2026-02-17T14:30:57Z DEBUG sqlx::query] SELECT FROM streams returned 672 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h5nqnjl7i6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715938000,"log":"[2026-02-17T15:02:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q0er90r6zc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715939000,"log":"[2026-02-17T13:48:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 443ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s1z3k5kkjy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715940000,"log":"[2026-02-17T12:53:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6f4ju2tuuy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715941000,"log":"[2026-02-17T14:41:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=dy3qwws2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o0xw1lnv33","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715942000,"log":"[2026-02-17T13:38:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4455366151746860000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d8za64cfeu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715943000,"log":"[2026-02-17T13:05:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iu7w3xhzhs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715944000,"log":"[2026-02-17T14:09:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 724 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6z06344a66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715945000,"log":"[2026-02-17T13:08:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7421688678228167000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x72gmleu7k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715946000,"log":"[2026-02-17T14:39:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=www07q8u","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bptukitica","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715947000,"log":"[2026-02-17T14:33:07Z INFO tonic::transport::server] grpc ingester connected client_id=3t1i8ymx","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0hifpbi3w7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715948000,"log":"[2026-02-17T12:43:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mpe36oek0b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715949000,"log":"[2026-02-17T14:40:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=bqvov5ih","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ceyss4htn8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715950000,"log":"[2026-02-17T14:36:17Z INFO tonic::transport::server] grpc ingester connected client_id=vz4limui","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4wlv27kqtl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715951000,"log":"[2026-02-17T12:54:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dc88cdl7mq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715952000,"log":"[2026-02-17T14:21:07Z DEBUG sqlx::query] SELECT FROM streams returned 5 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7c249nu6oa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715953000,"log":"[2026-02-17T14:30:13Z DEBUG sqlx::query] INSERT FROM users returned 628 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9x38zsyc3t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715954000,"log":"[2026-02-17T14:57:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v1ersr3nrl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715955000,"log":"[2026-02-17T14:16:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 21 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3z7auxqikm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715956000,"log":"[2026-02-17T14:33:27Z INFO tonic::transport::server] grpc ingester connected client_id=8jgt0wny","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fatwy84mjk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715957000,"log":"[2026-02-17T14:16:38Z DEBUG sqlx::query] INSERT FROM users returned 28 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h6cnq5ae94","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715958000,"log":"[2026-02-17T15:10:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=9886","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zbmuqm1bk1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715959000,"log":"[2026-02-17T13:19:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5687843779056590000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ht4rxoozdc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715960000,"log":"[2026-02-17T13:36:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7780625077022592000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3zix9yk1s5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715961000,"log":"[2026-02-17T13:19:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9724941722280770000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fjq9qr1hoc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715962000,"log":"[2026-02-17T12:54:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tqbcj9j9gy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715963000,"log":"[2026-02-17T13:20:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5349001410787480000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-loguvyv65y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715964000,"log":"[2026-02-17T13:07:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1iaeb70prc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715965000,"log":"[2026-02-17T14:12:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 714 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9vjjotj2ve","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715966000,"log":"[2026-02-17T13:41:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5111081461438794000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9s94zl64nd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715967000,"log":"[2026-02-17T14:24:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 694 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nzusqcsbj8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715968000,"log":"[2026-02-17T14:15:43Z DEBUG sqlx::query] INSERT FROM users returned 534 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g105ikj0fy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715969000,"log":"[2026-02-17T14:26:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 946 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bg311bxto0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715970000,"log":"[2026-02-17T13:58:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 428ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m33hwfjn69","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715971000,"log":"[2026-02-17T14:53:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-alj7o5mea5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715972000,"log":"[2026-02-17T13:52:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 216ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ddmh799on3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715973000,"log":"[2026-02-17T15:14:38Z TRACE cache::redis] cache get key=org:config:2059","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q3fvt3u7k1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715974000,"log":"[2026-02-17T14:40:37Z INFO tonic::transport::server] grpc ingester connected client_id=7kzwtk5w","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tdjs231a54","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715975000,"log":"[2026-02-17T14:21:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 215 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sw8p1q6xpa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715976000,"log":"[2026-02-17T13:08:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ho70lvyce4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715977000,"log":"[2026-02-17T13:51:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 339ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h402blumht","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715978000,"log":"[2026-02-17T13:59:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 285ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hqtvivg6tj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715979000,"log":"[2026-02-17T13:58:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 409ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wmqmeweq4r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715980000,"log":"[2026-02-17T12:47:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sdnmzwcfto","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715981000,"log":"[2026-02-17T12:47:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7fueve3y7q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715982000,"log":"[2026-02-17T12:55:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ujtbpgxabo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715983000,"log":"[2026-02-17T15:13:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1080","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8s3g2yfdgq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715984000,"log":"[2026-02-17T12:46:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x9v72xyi1i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715985000,"log":"[2026-02-17T13:54:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 265ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6b456y99gv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715986000,"log":"[2026-02-17T13:13:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3687515279160913400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bj684d3g89","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715987000,"log":"[2026-02-17T14:10:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 848 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x91bvs2wfp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715988000,"log":"[2026-02-17T13:25:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6308731969469450000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tazqnjgmzs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715989000,"log":"[2026-02-17T14:44:04Z INFO tonic::transport::server] grpc router request_received client_id=g9gyctrl","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tnqjpfh7ig","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715990000,"log":"[2026-02-17T13:44:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 159ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dm664ttd6d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715991000,"log":"[2026-02-17T14:26:43Z DEBUG sqlx::query] INSERT FROM users returned 174 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xb4xxsp4qe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715992000,"log":"[2026-02-17T14:24:45Z DEBUG sqlx::query] DELETE FROM alerts returned 600 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hl731b6spa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715993000,"log":"[2026-02-17T14:18:15Z DEBUG sqlx::query] DELETE FROM alerts returned 398 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hmyn9tbqsz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715994000,"log":"[2026-02-17T14:06:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 59ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z8bnu8njm3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331715995000,"log":"[2026-02-17T14:55:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-krj28cwx8o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715996000,"log":"[2026-02-17T15:20:26Z TRACE cache::redis] cache get key=auth:token:7801","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sbgp5y2hr1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715997000,"log":"[2026-02-17T13:21:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8148124597916484000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4bwjy7xzjg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331715998000,"log":"[2026-02-17T12:37:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hl51qk5ips","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331715999000,"log":"[2026-02-17T14:49:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p4ybo6ypzr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716000000,"log":"[2026-02-17T12:44:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5t8ajxncf6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716001000,"log":"[2026-02-17T12:57:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h1g8m44mjh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716002000,"log":"[2026-02-17T15:11:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=3301","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kg6l4z3mkk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716003000,"log":"[2026-02-17T13:19:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7480657029724441000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ebrb97ez2h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716004000,"log":"[2026-02-17T15:10:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=977","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u41ssxg6ee","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716005000,"log":"[2026-02-17T14:38:37Z INFO tonic::transport::server] grpc ingester connected client_id=f5buvmbo","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1zyr80vqho","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716006000,"log":"[2026-02-17T14:33:03Z INFO tonic::transport::server] grpc querier disconnected client_id=axyne3k9","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-384qcdw5tg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716007000,"log":"[2026-02-17T12:49:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8s26kymbiq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716008000,"log":"[2026-02-17T12:51:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v7lw8nx4kx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716009000,"log":"[2026-02-17T14:13:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 544 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-evyys1vw71","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716010000,"log":"[2026-02-17T15:00:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rjt3rkrl6u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716011000,"log":"[2026-02-17T14:02:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 387ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-crdz4wa9t3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716012000,"log":"[2026-02-17T15:15:05Z TRACE cache::redis] cache expire key=query:result:6533","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iaj9xbbbgi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716013000,"log":"[2026-02-17T12:40:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xea2v4yzzx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716014000,"log":"[2026-02-17T12:57:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a0uqyta7dv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716015000,"log":"[2026-02-17T12:56:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ufkhl2b5w3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716016000,"log":"[2026-02-17T14:12:58Z DEBUG sqlx::query] INSERT FROM users returned 915 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wgbqy3cu23","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716017000,"log":"[2026-02-17T13:36:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6685005780753081000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9wnq4afhr0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716018000,"log":"[2026-02-17T13:35:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5711487902874053000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f0s3vxad34","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716019000,"log":"[2026-02-17T13:00:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8kvvcseauh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716020000,"log":"[2026-02-17T14:02:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 335ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tggrl680iw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716021000,"log":"[2026-02-17T14:29:17Z DEBUG sqlx::query] SELECT FROM streams returned 138 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wl5o39073y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716022000,"log":"[2026-02-17T13:04:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bzngtsl8ke","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716023000,"log":"[2026-02-17T13:25:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7940009163039501000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-25pruh0yer","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716024000,"log":"[2026-02-17T15:07:15Z DEBUG metrics::reporter] recorded metric active_connections value=4916","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8kg6h21klv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716025000,"log":"[2026-02-17T15:04:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-meqjvebpcl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716026000,"log":"[2026-02-17T15:19:20Z TRACE cache::redis] cache get key=query:result:1840","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wvwqvcqg1h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716027000,"log":"[2026-02-17T14:34:57Z INFO tonic::transport::server] grpc ingester connected client_id=woyph6rk","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mpmyb2ns25","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716028000,"log":"[2026-02-17T14:51:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9irb7gq45c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716029000,"log":"[2026-02-17T14:27:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 238 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p7zol7qked","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716030000,"log":"[2026-02-17T13:44:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 365ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hcjl784r02","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716031000,"log":"[2026-02-17T15:18:56Z TRACE cache::redis] cache get key=auth:token:1142","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mutwg2o545","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716032000,"log":"[2026-02-17T15:09:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=4550","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g3kocn8nlp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716033000,"log":"[2026-02-17T13:07:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pxmw5m2zah","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716034000,"log":"[2026-02-17T13:18:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5551427601481207000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ivp97yf95u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716035000,"log":"[2026-02-17T12:59:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-11qij9twxd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716036000,"log":"[2026-02-17T13:23:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5957207127951187000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h7hv0fppds","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716037000,"log":"[2026-02-17T12:36:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n969iyefm2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716038000,"log":"[2026-02-17T14:49:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-to5y45io9b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716039000,"log":"[2026-02-17T14:36:08Z INFO tonic::transport::server] grpc querier disconnected client_id=fvr38pbj","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vnysd67b90","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716040000,"log":"[2026-02-17T13:51:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 323ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tdvbx8huzu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716041000,"log":"[2026-02-17T15:00:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-icno5kyv48","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716042000,"log":"[2026-02-17T15:01:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4ip24crwp3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716043000,"log":"[2026-02-17T13:12:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9080796624191647000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tzzbmdjn94","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716044000,"log":"[2026-02-17T14:50:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dz778t32jg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716045000,"log":"[2026-02-17T15:21:20Z TRACE cache::redis] cache get key=query:result:6374","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xjca0p9sx1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716046000,"log":"[2026-02-17T14:34:13Z INFO tonic::transport::server] grpc querier disconnected client_id=0a7u37nt","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dnbcq3iics","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716047000,"log":"[2026-02-17T15:17:12Z TRACE cache::redis] cache hit key=user:session:2984","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vk6ungux3n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716048000,"log":"[2026-02-17T12:52:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8lln0ieqd7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716049000,"log":"[2026-02-17T12:36:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fvw63k081t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716050000,"log":"[2026-02-17T14:53:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rs9janyhjm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716051000,"log":"[2026-02-17T14:19:43Z DEBUG sqlx::query] INSERT FROM users returned 233 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qktusektfh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716052000,"log":"[2026-02-17T15:10:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=5637","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z2jhy4hqmi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716053000,"log":"[2026-02-17T14:45:34Z INFO tonic::transport::server] grpc router request_received client_id=n92x7ea9","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4pcvogj4tx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716054000,"log":"[2026-02-17T13:19:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3315071856753744000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j8qdlj25ke","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716055000,"log":"[2026-02-17T13:39:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6267710644659246000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i6xzo22gq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716056000,"log":"[2026-02-17T12:52:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zx1pj9bdsl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716057000,"log":"[2026-02-17T14:50:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n0qp1m20zo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716058000,"log":"[2026-02-17T14:12:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 927 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q1aqifzw9k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716059000,"log":"[2026-02-17T14:08:30Z DEBUG sqlx::query] DELETE FROM alerts returned 502 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5fv6cb7lfp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716060000,"log":"[2026-02-17T13:13:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5108587416725150000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1i8q248mpr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716061000,"log":"[2026-02-17T14:02:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 449ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fooxl8tuux","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716062000,"log":"[2026-02-17T13:16:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5448426925964420000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9wfoxh2utz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716063000,"log":"[2026-02-17T15:17:27Z TRACE cache::redis] cache set key=user:session:8058","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9dnx9m0sdz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716064000,"log":"[2026-02-17T14:33:43Z INFO tonic::transport::server] grpc querier disconnected client_id=c86lmfhq","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-55x0skdw7t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716065000,"log":"[2026-02-17T13:53:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 257ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u919tmmtha","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716066000,"log":"[2026-02-17T14:40:57Z INFO tonic::transport::server] grpc ingester connected client_id=5fys04jq","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-692rjn2d52","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716067000,"log":"[2026-02-17T13:38:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3645412648180936700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zyuhchwi8z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716068000,"log":"[2026-02-17T13:41:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 312ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n2itjcejd5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716069000,"log":"[2026-02-17T14:46:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=hxw2i2cz","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tkp8eqrkxd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716070000,"log":"[2026-02-17T13:51:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 114ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zvwmj0lubv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716071000,"log":"[2026-02-17T13:17:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7850942948574866000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yfadlvl9i2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716072000,"log":"[2026-02-17T14:39:03Z INFO tonic::transport::server] grpc querier disconnected client_id=ywpy0o7c","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-alz05v9bv2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716073000,"log":"[2026-02-17T14:24:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 580 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y82z4mai39","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716074000,"log":"[2026-02-17T13:24:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7109506275983009000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cwhlcdq97v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716075000,"log":"[2026-02-17T13:38:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1120688204747435400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mkr4usfuvt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716076000,"log":"[2026-02-17T14:50:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-juismbjr7f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716077000,"log":"[2026-02-17T13:22:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3457439976569993700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xr62vssiui","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716078000,"log":"[2026-02-17T14:57:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w1vtk0dcq8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716079000,"log":"[2026-02-17T14:50:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z7gddzruks","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716080000,"log":"[2026-02-17T14:58:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xmivzcw6ae","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716081000,"log":"[2026-02-17T15:06:00Z DEBUG metrics::reporter] recorded metric active_connections value=3786","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xuy69y02de","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716082000,"log":"[2026-02-17T13:35:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/678806096402278000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xi8x3z0w2i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716083000,"log":"[2026-02-17T15:00:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b4l3mgis96","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716084000,"log":"[2026-02-17T13:20:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2514913873420022000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2zlx2224kn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716085000,"log":"[2026-02-17T12:47:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-02r746xjq4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716086000,"log":"[2026-02-17T13:18:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8190125608534684000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y6aaoteua4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716087000,"log":"[2026-02-17T13:21:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5219944207942438000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ciihb1vzia","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716088000,"log":"[2026-02-17T14:09:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 17 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-43otkaoor8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716089000,"log":"[2026-02-17T14:29:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 145 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pk90kkdevd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716090000,"log":"[2026-02-17T14:47:19Z INFO tonic::transport::server] grpc router request_received client_id=leouflqw","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hzu9zy1n87","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716091000,"log":"[2026-02-17T15:02:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-62yvv7dt8y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716092000,"log":"[2026-02-17T14:17:37Z DEBUG sqlx::query] SELECT FROM streams returned 766 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-chq7iomdpo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716093000,"log":"[2026-02-17T13:47:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 158ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9evhwpiihn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716094000,"log":"[2026-02-17T15:14:21Z TRACE cache::redis] cache set key=auth:token:8601","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kv5xw6mdmn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716095000,"log":"[2026-02-17T14:24:43Z DEBUG sqlx::query] INSERT FROM users returned 60 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-caw5dxhl5q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716096000,"log":"[2026-02-17T14:05:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 198ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kxxx4psf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716097000,"log":"[2026-02-17T15:04:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bzq1x9tknf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716098000,"log":"[2026-02-17T14:09:57Z DEBUG sqlx::query] SELECT FROM streams returned 113 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9dqxy98f0i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716099000,"log":"[2026-02-17T13:10:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5084437528289099000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3ch2horld6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716100000,"log":"[2026-02-17T13:45:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 294ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gjo4ky378n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716101000,"log":"[2026-02-17T14:32:29Z INFO tonic::transport::server] grpc router request_received client_id=51ohui4o","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-20vtz1pkdh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716102000,"log":"[2026-02-17T13:55:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 450ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fcxl73a8va","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716103000,"log":"[2026-02-17T13:36:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8043438342929181000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-koug4hh4ys","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716104000,"log":"[2026-02-17T12:53:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tzqg9ikosf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716105000,"log":"[2026-02-17T14:22:48Z DEBUG sqlx::query] INSERT FROM users returned 712 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sf5pic2lvl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716106000,"log":"[2026-02-17T15:01:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8al70vjm63","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716107000,"log":"[2026-02-17T12:58:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u3avzo7lrd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716108000,"log":"[2026-02-17T13:00:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-549b4b8309","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716109000,"log":"[2026-02-17T13:17:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5724951855061129000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4eqbl16odk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716110000,"log":"[2026-02-17T13:27:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1625755806085223000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-87frbk40lc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716111000,"log":"[2026-02-17T14:56:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2n4j105bqz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716112000,"log":"[2026-02-17T12:38:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-99zlapalpm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716113000,"log":"[2026-02-17T12:47:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4swfk345wj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716114000,"log":"[2026-02-17T13:19:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6590775715035313000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5gjwm08gfk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716115000,"log":"[2026-02-17T14:03:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 37ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kvtq2gi6eg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716116000,"log":"[2026-02-17T12:52:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tam8loe48z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716117000,"log":"[2026-02-17T13:29:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/227677325026307550.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dlsyx976wy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716118000,"log":"[2026-02-17T12:54:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t2yoj7lun5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716119000,"log":"[2026-02-17T13:30:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7104815244915601000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a7z96pnjwl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716120000,"log":"[2026-02-17T14:48:14Z INFO tonic::transport::server] grpc router request_received client_id=wzl0dqcp","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3vnmhdj70g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716121000,"log":"[2026-02-17T13:29:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2901021570742132700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k1dr7eoa5e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716122000,"log":"[2026-02-17T13:52:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 319ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3b9ttc0jj8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716123000,"log":"[2026-02-17T13:47:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 182ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r6at690fmq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716124000,"log":"[2026-02-17T14:14:43Z DEBUG sqlx::query] INSERT FROM users returned 610 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6zbnwdqxu8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716125000,"log":"[2026-02-17T14:47:52Z INFO tonic::transport::server] grpc ingester connected client_id=rlmmk9uh","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ufna38n3c3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716126000,"log":"[2026-02-17T14:49:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r8s6ujeu94","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716127000,"log":"[2026-02-17T13:21:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3090941291655038500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j849ri73dr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716128000,"log":"[2026-02-17T13:26:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3376638354677503500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4n13fgndfw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716129000,"log":"[2026-02-17T13:24:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6759687898067535000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f9oce1ymnf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716130000,"log":"[2026-02-17T14:03:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 413ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7e7q1o6958","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716131000,"log":"[2026-02-17T12:59:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d5ngkhc2vb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716132000,"log":"[2026-02-17T14:11:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 692 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4g94cca5le","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716133000,"log":"[2026-02-17T14:20:13Z DEBUG sqlx::query] INSERT FROM users returned 926 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3pgk9ci60s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716134000,"log":"[2026-02-17T14:42:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=ln9yo7mz","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pn8yhfy7tw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716135000,"log":"[2026-02-17T14:15:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 373 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m9l7gsm9n3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716136000,"log":"[2026-02-17T14:21:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 305 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kbvsw8migd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716137000,"log":"[2026-02-17T13:24:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6531834648583419000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-17qxbkevv2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716138000,"log":"[2026-02-17T13:59:45Z INFO actix_web::middleware::logger] GET /api/_search 204 82ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pl8xn5ys8x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716139000,"log":"[2026-02-17T13:35:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4602522052538960000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rd6gg13apc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716140000,"log":"[2026-02-17T13:56:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 151ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y3xx6cdz3k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716141000,"log":"[2026-02-17T14:53:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-94d8vogo5z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716142000,"log":"[2026-02-17T13:12:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4896467012064792000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7a2ny7ojw2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716143000,"log":"[2026-02-17T13:58:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 243ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bnq9efbgqc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716144000,"log":"[2026-02-17T13:47:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 167ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u8ol6mvvqw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716145000,"log":"[2026-02-17T14:05:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 110ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rlks86qphe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716146000,"log":"[2026-02-17T13:54:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 295ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ubdt9hwf3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716147000,"log":"[2026-02-17T14:25:02Z DEBUG sqlx::query] SELECT FROM streams returned 299 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qa40euzmkw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716148000,"log":"[2026-02-17T13:58:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 419ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j1617x6um","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716149000,"log":"[2026-02-17T14:18:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 41 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p31890kw7d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716150000,"log":"[2026-02-17T14:40:43Z INFO tonic::transport::server] grpc querier disconnected client_id=ipz1h6t9","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1kc0cy7ygh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716151000,"log":"[2026-02-17T15:00:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mkb5ikum8m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716152000,"log":"[2026-02-17T14:27:37Z DEBUG sqlx::query] SELECT FROM streams returned 758 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0ecb1vrarl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716153000,"log":"[2026-02-17T14:33:09Z INFO tonic::transport::server] grpc router request_received client_id=ap3r8prc","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7hmtiy13na","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716154000,"log":"[2026-02-17T13:48:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 202ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qonja44emq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716155000,"log":"[2026-02-17T13:58:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 241ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h92chptr5c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716156000,"log":"[2026-02-17T13:12:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3440480619745074000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-utk1zfu0su","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716157000,"log":"[2026-02-17T13:05:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ffxrf90ng4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716158000,"log":"[2026-02-17T14:11:32Z DEBUG sqlx::query] SELECT FROM streams returned 516 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lgwozeixlr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716159000,"log":"[2026-02-17T13:43:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 399ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2d9l0i217g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716160000,"log":"[2026-02-17T14:23:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 455 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j9nzhay7td","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716161000,"log":"[2026-02-17T14:43:29Z INFO tonic::transport::server] grpc router request_received client_id=q521wafk","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7xiga8l8vz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716162000,"log":"[2026-02-17T13:51:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 461ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u6pigbv7ys","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716163000,"log":"[2026-02-17T12:59:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lsekoopuwy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716164000,"log":"[2026-02-17T14:26:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 640 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vmkafeuo2b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716165000,"log":"[2026-02-17T13:58:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 353ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-spd7qgr7vr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716166000,"log":"[2026-02-17T13:21:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8585144896463255000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yspvg7okw4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716167000,"log":"[2026-02-17T13:24:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/618322948369105500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v82cdvzpae","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716168000,"log":"[2026-02-17T13:38:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4382291551234847000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-131cawzoys","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716169000,"log":"[2026-02-17T14:27:25Z DEBUG sqlx::query] DELETE FROM alerts returned 192 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6r6s3qwfnv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716170000,"log":"[2026-02-17T14:20:25Z DEBUG sqlx::query] DELETE FROM alerts returned 752 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i4ndn2l4sk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716171000,"log":"[2026-02-17T13:34:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9495767411772043000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b67akpd87r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716172000,"log":"[2026-02-17T14:36:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=beuynr84","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z4npnsk1xz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716173000,"log":"[2026-02-17T12:35:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-imzacx233d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716174000,"log":"[2026-02-17T14:57:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-syhyctreku","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716175000,"log":"[2026-02-17T14:15:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 346 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t4ll2jsq4y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716176000,"log":"[2026-02-17T15:10:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1249","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w27s9dtkq1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716177000,"log":"[2026-02-17T12:53:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-emm9ewt0np","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716178000,"log":"[2026-02-17T14:28:17Z DEBUG sqlx::query] SELECT FROM streams returned 428 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7iq8smpwc3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716179000,"log":"[2026-02-17T14:23:22Z DEBUG sqlx::query] SELECT FROM streams returned 303 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hm859nmi2d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716180000,"log":"[2026-02-17T13:54:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vyfi4fjpzt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716181000,"log":"[2026-02-17T13:28:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1202481639351436000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c4r45keyrr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716182000,"log":"[2026-02-17T14:50:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zv9yqvpxgj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716183000,"log":"[2026-02-17T14:05:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 469ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uqxkrzjiba","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716184000,"log":"[2026-02-17T15:12:40Z DEBUG metrics::reporter] recorded metric active_connections value=5445","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w8uf7dexmk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716185000,"log":"[2026-02-17T14:55:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1z2ez1vpno","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716186000,"log":"[2026-02-17T13:50:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 340ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yfa28bnqhi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716187000,"log":"[2026-02-17T13:37:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7745808058549650000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b07v03vuvd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716188000,"log":"[2026-02-17T12:46:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-elup163imz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716189000,"log":"[2026-02-17T13:49:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 441ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nczt1ybnlz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716190000,"log":"[2026-02-17T13:59:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 414ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jipnjzqg3l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716191000,"log":"[2026-02-17T14:39:34Z INFO tonic::transport::server] grpc router request_received client_id=xt6sxyos","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2214im81ji","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716192000,"log":"[2026-02-17T14:34:39Z INFO tonic::transport::server] grpc router request_received client_id=j547ke7t","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uair42ok3o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716193000,"log":"[2026-02-17T14:24:02Z DEBUG sqlx::query] SELECT FROM streams returned 237 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lz9xkplovq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716194000,"log":"[2026-02-17T14:52:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jsbm5t5s0g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716195000,"log":"[2026-02-17T13:37:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9428480113272435000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aqktp2pnd4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716196000,"log":"[2026-02-17T13:04:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4k0y2nuk91","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716197000,"log":"[2026-02-17T13:25:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9260068994760690000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3l5h0rxy33","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716198000,"log":"[2026-02-17T12:57:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-53xjicqh2r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716199000,"log":"[2026-02-17T14:34:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=04b4gl0z","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k67ym75hfo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716200000,"log":"[2026-02-17T12:44:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1gtfunfzy8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716201000,"log":"[2026-02-17T14:53:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mb6z2bpto1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716202000,"log":"[2026-02-17T13:32:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5185596805050148000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gseq1v54uk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716203000,"log":"[2026-02-17T15:19:04Z TRACE cache::redis] cache delete key=stream:schema:4799","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-osiqwx249g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716204000,"log":"[2026-02-17T14:01:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 468ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jmx7qpw5um","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716205000,"log":"[2026-02-17T14:54:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rw773epp02","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716206000,"log":"[2026-02-17T15:00:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hr9gwerr58","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716207000,"log":"[2026-02-17T14:38:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=m2cvgs7j","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-25qem3crbh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716208000,"log":"[2026-02-17T14:44:28Z INFO tonic::transport::server] grpc querier disconnected client_id=iaf3y3zs","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k72fxx83jl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716209000,"log":"[2026-02-17T12:45:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nzyxdno5hx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716210000,"log":"[2026-02-17T13:35:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1117149938036090500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-crqvt0oym6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716211000,"log":"[2026-02-17T13:32:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4450492967116577300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mym95f8w2o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716212000,"log":"[2026-02-17T13:26:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1619771632902451500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ljq2qgphwr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716213000,"log":"[2026-02-17T14:32:27Z INFO tonic::transport::server] grpc ingester connected client_id=28qc60of","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uedagxfvy3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716214000,"log":"[2026-02-17T12:41:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w4cq440z52","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716215000,"log":"[2026-02-17T14:18:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 941 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tfi76t9i9f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716216000,"log":"[2026-02-17T15:13:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5854","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4nbmaqoptm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716217000,"log":"[2026-02-17T13:23:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7750302814147971000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1xc6nfxis9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716218000,"log":"[2026-02-17T13:56:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 381ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6vqqykgcei","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716219000,"log":"[2026-02-17T14:16:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 916 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vlvzcsarmx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716220000,"log":"[2026-02-17T15:13:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5532","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kaqjjvz7lu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716221000,"log":"[2026-02-17T13:10:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1371556018500348400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d3bu144dkw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716222000,"log":"[2026-02-17T13:20:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4492581908662274000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lqf4pvn8ve","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716223000,"log":"[2026-02-17T13:54:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 331ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w8rc7cyjk3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716224000,"log":"[2026-02-17T14:24:05Z DEBUG sqlx::query] DELETE FROM alerts returned 578 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nmt5tyy4km","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716225000,"log":"[2026-02-17T14:07:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 941 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2eiz8fjdmk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716226000,"log":"[2026-02-17T13:07:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ku7r32nn9h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716227000,"log":"[2026-02-17T13:59:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 211ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yrrn0yklqx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716228000,"log":"[2026-02-17T12:43:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mj6tfssc1b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716229000,"log":"[2026-02-17T14:13:17Z DEBUG sqlx::query] SELECT FROM streams returned 941 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2a04spkql1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716230000,"log":"[2026-02-17T14:09:20Z DEBUG sqlx::query] DELETE FROM alerts returned 79 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bmlqtisblf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716231000,"log":"[2026-02-17T13:33:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7310629907791226000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jzl4tg9mgf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716232000,"log":"[2026-02-17T13:21:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4208820600702081000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tbar7y4kiz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716233000,"log":"[2026-02-17T13:49:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 298ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hjwx0mgc51","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716234000,"log":"[2026-02-17T14:51:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lhxt4hffjz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716235000,"log":"[2026-02-17T13:32:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7875141008676603000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p3c8jcppzr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716236000,"log":"[2026-02-17T13:50:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 374ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oa7fr5rpl7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716237000,"log":"[2026-02-17T13:39:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5047316352176259000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-srcbvaltxl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716238000,"log":"[2026-02-17T13:41:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4902792223718104000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9j1bqrvdlg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716239000,"log":"[2026-02-17T13:54:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 97ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y2vgqre9tf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716240000,"log":"[2026-02-17T14:23:42Z DEBUG sqlx::query] SELECT FROM streams returned 282 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-86a32st312","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716241000,"log":"[2026-02-17T14:38:54Z INFO tonic::transport::server] grpc router request_received client_id=sptamzpv","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9eejj4uq7h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716242000,"log":"[2026-02-17T13:37:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1174242967128647000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-03qytsu08z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716243000,"log":"[2026-02-17T14:42:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=bumu4dce","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ydzqldsdkk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716244000,"log":"[2026-02-17T13:11:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7790464641989190000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-klqpasgqzd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716245000,"log":"[2026-02-17T13:39:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1200323965642080500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-33rqj2hxhu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716246000,"log":"[2026-02-17T13:04:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xdgf5fcd8i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716247000,"log":"[2026-02-17T14:45:29Z INFO tonic::transport::server] grpc router request_received client_id=4e40jhjf","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dt5pkd5tab","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716248000,"log":"[2026-02-17T13:23:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2959696193170960000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o6bwxoxu7y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716249000,"log":"[2026-02-17T15:21:19Z TRACE cache::redis] cache miss key=stream:schema:5620","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dg4gvfez8x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716250000,"log":"[2026-02-17T14:08:37Z DEBUG sqlx::query] SELECT FROM streams returned 962 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6brmhwfkpn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716251000,"log":"[2026-02-17T15:09:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4485","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tlujuf33h1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716252000,"log":"[2026-02-17T13:32:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/148919205728486050.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-17uoj1o503","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716253000,"log":"[2026-02-17T13:57:40Z INFO actix_web::middleware::logger] POST /api/_search 400 74ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wldmp4jfe4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716254000,"log":"[2026-02-17T14:48:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dsbxrmike8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716255000,"log":"[2026-02-17T15:16:48Z TRACE cache::redis] cache hit key=org:config:7287","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wlosp7chjj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716256000,"log":"[2026-02-17T13:25:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8344144388997900000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uo7og0ddx1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716257000,"log":"[2026-02-17T14:27:57Z DEBUG sqlx::query] SELECT FROM streams returned 189 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pqia64vn9h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716258000,"log":"[2026-02-17T12:48:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ap6z07q5kf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716259000,"log":"[2026-02-17T15:00:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-joilo237nc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716260000,"log":"[2026-02-17T13:14:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2897405651615152600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9k182kw9jm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716261000,"log":"[2026-02-17T14:20:23Z DEBUG sqlx::query] INSERT FROM users returned 315 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r7f49x51ct","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716262000,"log":"[2026-02-17T14:43:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=s88ymx7o","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kxedg92ddj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716263000,"log":"[2026-02-17T14:03:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 92ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xcjqx1zmlm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716264000,"log":"[2026-02-17T13:56:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 91ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m2bu5rhaff","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716265000,"log":"[2026-02-17T13:26:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9430457039835752000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mkf34eex3m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716266000,"log":"[2026-02-17T14:59:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h8i3pzeler","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716267000,"log":"[2026-02-17T13:51:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 390ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j6pvizqenf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716268000,"log":"[2026-02-17T13:31:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8155274625116973000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v98nkh01w5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716269000,"log":"[2026-02-17T14:28:00Z DEBUG sqlx::query] DELETE FROM alerts returned 188 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-66qtoic5g9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716270000,"log":"[2026-02-17T14:12:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 226 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yx9weyrgy2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716271000,"log":"[2026-02-17T15:10:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=9853","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n9rbrpjfr4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716272000,"log":"[2026-02-17T13:07:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4xprs6f2fp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716273000,"log":"[2026-02-17T13:27:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3774950010214697000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i6b7c9t4o7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716274000,"log":"[2026-02-17T14:11:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 516 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-my1mjsekc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716275000,"log":"[2026-02-17T13:13:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9266445728952805000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dfv39g13ln","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716276000,"log":"[2026-02-17T14:17:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 345 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-thgld2rpq2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716277000,"log":"[2026-02-17T14:27:58Z DEBUG sqlx::query] INSERT FROM users returned 332 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-18gi8ic0qi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716278000,"log":"[2026-02-17T12:51:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ltkhq9j8p1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716279000,"log":"[2026-02-17T12:50:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jpfa6yn1n7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716280000,"log":"[2026-02-17T15:13:10Z DEBUG metrics::reporter] recorded metric active_connections value=9614","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-er7ld93mdm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716281000,"log":"[2026-02-17T15:05:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4069","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jjl7n19nfc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716282000,"log":"[2026-02-17T14:22:20Z DEBUG sqlx::query] DELETE FROM alerts returned 478 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pslxf8dzre","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716283000,"log":"[2026-02-17T14:03:05Z INFO actix_web::middleware::logger] GET /api/_search 401 316ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4gbzb2u9p0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716284000,"log":"[2026-02-17T13:54:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 294ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h0c2x5haev","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716285000,"log":"[2026-02-17T15:02:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7d0oyxj3vz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716286000,"log":"[2026-02-17T13:46:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 372ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6duhixd0kf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716287000,"log":"[2026-02-17T14:31:18Z DEBUG sqlx::query] INSERT FROM users returned 677 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mhu9r400w4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716288000,"log":"[2026-02-17T13:45:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 134ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oin4jpr8d1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716289000,"log":"[2026-02-17T13:52:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 280ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g46s6ze6t9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716290000,"log":"[2026-02-17T13:25:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3163444668048720400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-efckickcp9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716291000,"log":"[2026-02-17T13:45:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 74ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0s4aymv1n3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716292000,"log":"[2026-02-17T14:59:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mlgaucvqze","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716293000,"log":"[2026-02-17T13:51:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 145ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-96oc6vrfbq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716294000,"log":"[2026-02-17T13:35:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4418619390381897000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bbaen9jbvz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716295000,"log":"[2026-02-17T13:08:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5033689652361062000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-05fc2avtmn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716296000,"log":"[2026-02-17T14:18:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 815 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-psvu7x06fn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716297000,"log":"[2026-02-17T14:02:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 169ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xjfama0hze","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716298000,"log":"[2026-02-17T13:26:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2201628032345055500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l2t474jbmj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716299000,"log":"[2026-02-17T13:46:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 286ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mws4flvx9x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716300000,"log":"[2026-02-17T14:59:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4rj7mgv4jh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716301000,"log":"[2026-02-17T14:51:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k5gbonp1td","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716302000,"log":"[2026-02-17T14:50:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dychjx9yco","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716303000,"log":"[2026-02-17T14:06:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 68ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3w7halqsaj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716304000,"log":"[2026-02-17T14:17:25Z DEBUG sqlx::query] DELETE FROM alerts returned 693 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cuqniedq9e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716305000,"log":"[2026-02-17T12:48:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qoeih9g4dn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716306000,"log":"[2026-02-17T14:37:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=qdafc1hn","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-etmxb6ayr5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716307000,"log":"[2026-02-17T14:15:52Z DEBUG sqlx::query] SELECT FROM streams returned 155 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hgg5ej3247","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716308000,"log":"[2026-02-17T13:12:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6417253619067444000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-znx5y5ei5t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716309000,"log":"[2026-02-17T12:39:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uiou3te9l5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716310000,"log":"[2026-02-17T13:57:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 469ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iqbgb1w59y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716311000,"log":"[2026-02-17T13:24:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7674156584841894000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ngfp67757m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716312000,"log":"[2026-02-17T13:44:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 470ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ssczrpa75f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716313000,"log":"[2026-02-17T13:56:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 477ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u36w3bdxqb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716314000,"log":"[2026-02-17T15:03:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c9sy2ya1i6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716315000,"log":"[2026-02-17T14:28:12Z DEBUG sqlx::query] SELECT FROM streams returned 833 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-piidphf3dm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716316000,"log":"[2026-02-17T15:08:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=6185","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-upm8jmgy3c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716317000,"log":"[2026-02-17T14:38:43Z INFO tonic::transport::server] grpc querier disconnected client_id=d3ywdkdo","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ubrbui716t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716318000,"log":"[2026-02-17T13:49:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 288ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ybiq6raoi9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716319000,"log":"[2026-02-17T12:59:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lh3lrof6yw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716320000,"log":"[2026-02-17T13:25:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6826817423291609000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kjj8qm2g6g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716321000,"log":"[2026-02-17T14:25:05Z DEBUG sqlx::query] DELETE FROM alerts returned 208 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z1r56xvwhu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716322000,"log":"[2026-02-17T15:19:58Z TRACE cache::redis] cache delete key=org:config:4545","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-372dnop6iv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716323000,"log":"[2026-02-17T13:25:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3283315601186802700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o37zyph9pl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716324000,"log":"[2026-02-17T13:35:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/960348970498388100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-48x28870ev","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716325000,"log":"[2026-02-17T14:10:10Z DEBUG sqlx::query] DELETE FROM alerts returned 706 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5z49r8whu6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716326000,"log":"[2026-02-17T13:03:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b4fe0spb9j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716327000,"log":"[2026-02-17T12:53:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i3nor6ze7f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716328000,"log":"[2026-02-17T13:02:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b5v46336yz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716329000,"log":"[2026-02-17T14:33:23Z INFO tonic::transport::server] grpc querier disconnected client_id=6uxoh8m2","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-87va1qpej8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716330000,"log":"[2026-02-17T14:52:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a6q6t9tscx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716331000,"log":"[2026-02-17T14:02:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 200ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f0v0rhiqhm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716332000,"log":"[2026-02-17T13:16:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7555107491510795000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9606gcscr1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716333000,"log":"[2026-02-17T14:31:07Z DEBUG sqlx::query] SELECT FROM streams returned 704 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v70c6jqu7c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716334000,"log":"[2026-02-17T14:27:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 38 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rtqmxctfot","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716335000,"log":"[2026-02-17T15:16:53Z TRACE cache::redis] cache expire key=org:config:6543","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cac75ile38","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716336000,"log":"[2026-02-17T15:02:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4amepu34ol","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716337000,"log":"[2026-02-17T13:09:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8010870342393937000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u5rgin4v3t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716338000,"log":"[2026-02-17T14:34:32Z INFO tonic::transport::server] grpc ingester connected client_id=tzxkc5i3","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tgbyab5o5t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716339000,"log":"[2026-02-17T14:31:32Z DEBUG sqlx::query] SELECT FROM streams returned 236 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0fhgp3o2j2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716340000,"log":"[2026-02-17T14:42:27Z INFO tonic::transport::server] grpc ingester connected client_id=9y9hf3hf","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lroh92r3af","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716341000,"log":"[2026-02-17T12:50:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-edj35e0sw7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716342000,"log":"[2026-02-17T13:29:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3647415754159060000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lm4qk8l29n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716343000,"log":"[2026-02-17T14:34:53Z INFO tonic::transport::server] grpc querier disconnected client_id=bhvq3wzg","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kozhbsdbfa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716344000,"log":"[2026-02-17T14:23:18Z DEBUG sqlx::query] INSERT FROM users returned 705 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8mx20h92ye","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716345000,"log":"[2026-02-17T14:57:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bxma4ioh4f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716346000,"log":"[2026-02-17T12:58:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d441zpypnx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716347000,"log":"[2026-02-17T13:03:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bq8pppe7wh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716348000,"log":"[2026-02-17T14:47:34Z INFO tonic::transport::server] grpc router request_received client_id=3sf8iglm","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wzfxuqq6w8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716349000,"log":"[2026-02-17T15:00:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8hqqi1z5or","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716350000,"log":"[2026-02-17T13:53:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 84ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-axjmy3kkqi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716351000,"log":"[2026-02-17T14:52:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-auseu1gvg6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716352000,"log":"[2026-02-17T15:14:14Z TRACE cache::redis] cache get key=stream:schema:8756","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-npz6hlmnlz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716353000,"log":"[2026-02-17T12:47:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2xgi3ixa4d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716354000,"log":"[2026-02-17T15:00:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vs3gwvi0bp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716355000,"log":"[2026-02-17T13:10:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8289647493906980000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6mvv3c1w3a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716356000,"log":"[2026-02-17T14:40:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ujihv8fg","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2w4mt0j5c4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716357000,"log":"[2026-02-17T13:13:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7662795510656171000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gwzgendnx1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716358000,"log":"[2026-02-17T14:06:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 182ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j20rpb2ng8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716359000,"log":"[2026-02-17T12:49:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ozg7oracst","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716360000,"log":"[2026-02-17T14:01:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 431ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c52sjkk7q3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716361000,"log":"[2026-02-17T13:16:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/359638966144752960.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2vtg233ydp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716362000,"log":"[2026-02-17T14:14:33Z DEBUG sqlx::query] INSERT FROM users returned 990 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mhvdfj0l3v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716363000,"log":"[2026-02-17T13:51:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 81ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6d7wc3posj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716364000,"log":"[2026-02-17T13:12:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/398446022178160500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d5nuzxwm0o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716365000,"log":"[2026-02-17T13:29:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6061577551319301000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u80pirainl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716366000,"log":"[2026-02-17T13:20:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6459346781474491000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r87dmu1rjb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716367000,"log":"[2026-02-17T14:33:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=57tk1p7v","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-39dnbqoqop","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716368000,"log":"[2026-02-17T13:35:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5053824169945131000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-if19gzsqsi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716369000,"log":"[2026-02-17T13:40:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3763944062630644000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5qdbekmz2t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716370000,"log":"[2026-02-17T12:42:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-362nhv4po0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716371000,"log":"[2026-02-17T13:25:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3505656364447402500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h33q5g6su7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716372000,"log":"[2026-02-17T14:11:30Z DEBUG sqlx::query] DELETE FROM alerts returned 195 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gep3vwdfqe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716373000,"log":"[2026-02-17T12:58:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-28rnepzxs8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716374000,"log":"[2026-02-17T13:58:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 490ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-66rpm0lkdk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716375000,"log":"[2026-02-17T14:41:44Z INFO tonic::transport::server] grpc router request_received client_id=lbxyp6mu","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wy5lnf4dfb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716376000,"log":"[2026-02-17T15:04:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wepabmrhcj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716377000,"log":"[2026-02-17T12:49:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ni4nt7g4ln","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716378000,"log":"[2026-02-17T13:16:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9360685688684106000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v11se4fhbz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716379000,"log":"[2026-02-17T13:47:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-atszeeocfa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716380000,"log":"[2026-02-17T14:50:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iu3n8zoegj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716381000,"log":"[2026-02-17T13:30:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/544314626412996100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ukobxzjp8q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716382000,"log":"[2026-02-17T12:36:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eho3rpr3b6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716383000,"log":"[2026-02-17T12:58:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1k3s78uw1w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716384000,"log":"[2026-02-17T15:12:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7677","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f3yxb1ywun","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716385000,"log":"[2026-02-17T14:16:15Z DEBUG sqlx::query] DELETE FROM alerts returned 607 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g2lrzo0m2b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716386000,"log":"[2026-02-17T12:57:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0k1dmytrqv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716387000,"log":"[2026-02-17T13:32:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9488736971210550000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2f1dm3sq0k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716388000,"log":"[2026-02-17T13:44:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 330ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-89fi91gkel","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716389000,"log":"[2026-02-17T14:36:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=gh5vhps2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8c3ah91xo6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716390000,"log":"[2026-02-17T15:03:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wissk274je","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716391000,"log":"[2026-02-17T15:04:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-95qkcj1zxr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716392000,"log":"[2026-02-17T12:52:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-klc17e6uch","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716393000,"log":"[2026-02-17T14:59:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ybprpt7gr8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716394000,"log":"[2026-02-17T12:48:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7jo1gf2xo6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716395000,"log":"[2026-02-17T12:37:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-luo9yply68","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716396000,"log":"[2026-02-17T13:55:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 145ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rm893f3o5y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716397000,"log":"[2026-02-17T14:48:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2ep3gars","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qjgqvrvt3h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716398000,"log":"[2026-02-17T13:17:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/640016860804688500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9ru2anew3l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716399000,"log":"[2026-02-17T14:04:05Z INFO actix_web::middleware::logger] GET /api/_search 201 129ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vv447exjda","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716400000,"log":"[2026-02-17T14:01:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 486ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gdmyot00tq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716401000,"log":"[2026-02-17T13:25:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8201936256278013000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4pg00690h3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716402000,"log":"[2026-02-17T14:26:37Z DEBUG sqlx::query] SELECT FROM streams returned 451 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ynp2x82ejk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716403000,"log":"[2026-02-17T12:42:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d5x001b6j2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716404000,"log":"[2026-02-17T14:34:49Z INFO tonic::transport::server] grpc router request_received client_id=clnfbbwq","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n1nl9r8dgl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716405000,"log":"[2026-02-17T13:55:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 21ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g2802jin5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716406000,"log":"[2026-02-17T14:01:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 405ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vb2j58ehu7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716407000,"log":"[2026-02-17T13:34:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7221973064214824000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zgib4jlg30","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716408000,"log":"[2026-02-17T13:54:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 205ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mmxeqd7ffv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716409000,"log":"[2026-02-17T14:24:37Z DEBUG sqlx::query] SELECT FROM streams returned 11 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q639hy99q5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716410000,"log":"[2026-02-17T14:54:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fp8il5kqfl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716411000,"log":"[2026-02-17T13:51:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 79ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t36ok33bpb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716412000,"log":"[2026-02-17T13:22:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6515992826580593000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-guw15qcwbc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716413000,"log":"[2026-02-17T12:42:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g2i1c8eten","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716414000,"log":"[2026-02-17T14:17:05Z DEBUG sqlx::query] DELETE FROM alerts returned 281 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ca1xkylbnu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716415000,"log":"[2026-02-17T14:01:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 328ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nive4aos1a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716416000,"log":"[2026-02-17T12:37:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cqygov11y2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716417000,"log":"[2026-02-17T12:46:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-exhkamcmig","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716418000,"log":"[2026-02-17T14:50:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4jd0evkjwv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716419000,"log":"[2026-02-17T13:16:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8049730863761795000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ulcsa8yhm0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716420000,"log":"[2026-02-17T14:01:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 264ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-had5v1z1wa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716421000,"log":"[2026-02-17T14:13:20Z DEBUG sqlx::query] DELETE FROM alerts returned 430 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3uco54ht5q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716422000,"log":"[2026-02-17T14:56:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wozfm4nv6j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716423000,"log":"[2026-02-17T13:52:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 134ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zlejsif93x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716424000,"log":"[2026-02-17T14:25:20Z DEBUG sqlx::query] DELETE FROM alerts returned 244 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p747xiy1xi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716425000,"log":"[2026-02-17T14:36:27Z INFO tonic::transport::server] grpc ingester connected client_id=kh3j65bg","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a8yv7allmx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716426000,"log":"[2026-02-17T14:19:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 458 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u1j4h9o1vu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716427000,"log":"[2026-02-17T13:31:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7456650216659130000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6gc1efkd5n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716428000,"log":"[2026-02-17T14:57:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5815daxb53","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716429000,"log":"[2026-02-17T13:42:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 309ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2prsua4a6r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716430000,"log":"[2026-02-17T14:23:58Z DEBUG sqlx::query] INSERT FROM users returned 436 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-endlzthtjb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716431000,"log":"[2026-02-17T14:53:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j5597wtl28","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716432000,"log":"[2026-02-17T13:50:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 297ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r4szqdkjic","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716433000,"log":"[2026-02-17T13:45:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 216ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-un6ku4iwb6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716434000,"log":"[2026-02-17T14:23:13Z DEBUG sqlx::query] INSERT FROM users returned 666 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2t9toob979","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716435000,"log":"[2026-02-17T14:26:00Z DEBUG sqlx::query] DELETE FROM alerts returned 266 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5j20zohc69","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716436000,"log":"[2026-02-17T14:18:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 309 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3zuzaamgv2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716437000,"log":"[2026-02-17T14:34:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=gfozdzgj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dw0u4v50hw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716438000,"log":"[2026-02-17T13:38:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3969244628541795000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kv8e3tzgnq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716439000,"log":"[2026-02-17T15:17:15Z TRACE cache::redis] cache set key=query:result:9501","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7tj4tv61pd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716440000,"log":"[2026-02-17T14:37:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=5rkeq1z6","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g5o0z0efay","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716441000,"log":"[2026-02-17T13:06:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0xostgv63s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716442000,"log":"[2026-02-17T13:07:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fmy5wm1wpg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716443000,"log":"[2026-02-17T13:18:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1343976625799405600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zwx6slscdz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716444000,"log":"[2026-02-17T14:19:00Z DEBUG sqlx::query] DELETE FROM alerts returned 804 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w4wg4b9bqt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716445000,"log":"[2026-02-17T12:44:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ohyb7aka6h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716446000,"log":"[2026-02-17T14:20:35Z DEBUG sqlx::query] DELETE FROM alerts returned 127 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tytyvg59e2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716447000,"log":"[2026-02-17T14:40:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=9z8rwtvo","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kia97jqoda","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716448000,"log":"[2026-02-17T13:11:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9298515929773724000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c8j5i31o0q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716449000,"log":"[2026-02-17T13:43:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 20ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ge8uhtlqew","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716450000,"log":"[2026-02-17T15:07:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7524","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ph8ydr84vk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716451000,"log":"[2026-02-17T14:17:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 721 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1racvb2r3p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716452000,"log":"[2026-02-17T14:08:53Z DEBUG sqlx::query] INSERT FROM users returned 261 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kz66ynlvbg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716453000,"log":"[2026-02-17T14:00:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 391ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-66yauzt6d0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716454000,"log":"[2026-02-17T15:06:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3470","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-29gikewbu3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716455000,"log":"[2026-02-17T13:04:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5gtvw8gasy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716456000,"log":"[2026-02-17T14:10:25Z DEBUG sqlx::query] DELETE FROM alerts returned 925 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mwmeu7anb1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716457000,"log":"[2026-02-17T14:43:02Z INFO tonic::transport::server] grpc ingester connected client_id=n9l6yu7n","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zfrepnkzrs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716458000,"log":"[2026-02-17T13:28:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1681186135156738000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a6889wgw4l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716459000,"log":"[2026-02-17T14:52:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-visgn7nwc4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716460000,"log":"[2026-02-17T13:19:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6947510344818954000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-joy3q859c1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716461000,"log":"[2026-02-17T14:29:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 553 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-19tw28s23d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716462000,"log":"[2026-02-17T14:07:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 474 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xaapd6iuxs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716463000,"log":"[2026-02-17T14:47:39Z INFO tonic::transport::server] grpc router request_received client_id=f9dn4haq","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-955z8j5jhu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716464000,"log":"[2026-02-17T14:19:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 690 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ylwsfai8sv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716465000,"log":"[2026-02-17T14:12:57Z DEBUG sqlx::query] SELECT FROM streams returned 888 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uvzg400qqz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716466000,"log":"[2026-02-17T13:05:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l3qc50re2o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716467000,"log":"[2026-02-17T14:37:37Z INFO tonic::transport::server] grpc ingester connected client_id=6nq3xpqt","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ogdwjq25ae","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716468000,"log":"[2026-02-17T12:38:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y3u19szomk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716469000,"log":"[2026-02-17T14:58:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rh7co9xcx9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716470000,"log":"[2026-02-17T12:35:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5bn4h7e7k8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716471000,"log":"[2026-02-17T13:39:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5549149232607393000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n1y1xjhj9d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716472000,"log":"[2026-02-17T14:40:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=jna06wj0","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6rsqeupi9g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716473000,"log":"[2026-02-17T15:00:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qkxe7aohtp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716474000,"log":"[2026-02-17T12:50:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p3ut88ocr3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716475000,"log":"[2026-02-17T15:06:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1287","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kmm4366qx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716476000,"log":"[2026-02-17T15:07:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9557","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8bcdg2qyan","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716477000,"log":"[2026-02-17T14:12:02Z DEBUG sqlx::query] SELECT FROM streams returned 355 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tt6ykxmbgu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716478000,"log":"[2026-02-17T12:36:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mtds2i6gbu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716479000,"log":"[2026-02-17T14:13:53Z DEBUG sqlx::query] INSERT FROM users returned 990 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2i1t6cp176","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716480000,"log":"[2026-02-17T13:33:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9036130772045487000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p3sbqp8d7c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716481000,"log":"[2026-02-17T15:10:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1055","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7lj8dvfcl5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716482000,"log":"[2026-02-17T14:07:52Z DEBUG sqlx::query] SELECT FROM streams returned 369 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kqe5foe1nk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716483000,"log":"[2026-02-17T14:01:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 203ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tr57y126pq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716484000,"log":"[2026-02-17T14:44:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=73p6yxgh","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1k85wtrvdb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716485000,"log":"[2026-02-17T15:15:43Z TRACE cache::redis] cache miss key=org:config:1666","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b0c860q4ab","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716486000,"log":"[2026-02-17T14:46:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=jrqzndnc","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-28xjzmwkvb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716487000,"log":"[2026-02-17T14:17:23Z DEBUG sqlx::query] INSERT FROM users returned 553 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t0i4zntid","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716488000,"log":"[2026-02-17T14:59:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-99pwel189p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716489000,"log":"[2026-02-17T15:17:01Z TRACE cache::redis] cache miss key=auth:token:5102","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-078wrytfca","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716490000,"log":"[2026-02-17T14:50:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vbwui46n9v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716491000,"log":"[2026-02-17T15:14:52Z TRACE cache::redis] cache delete key=user:session:3746","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e9upa0lowq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716492000,"log":"[2026-02-17T14:45:52Z INFO tonic::transport::server] grpc ingester connected client_id=i60kj3xs","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eg3dksmioy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716493000,"log":"[2026-02-17T15:17:39Z TRACE cache::redis] cache set key=stream:schema:3037","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vxh45hvkip","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716494000,"log":"[2026-02-17T14:44:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=kmjl0ef4","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b82uf0s9j1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716495000,"log":"[2026-02-17T12:53:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t93owcazsi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716496000,"log":"[2026-02-17T12:56:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v7lrxk0cd6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716497000,"log":"[2026-02-17T14:46:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=q2u0pvtm","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5y05g4c6nk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716498000,"log":"[2026-02-17T14:20:32Z DEBUG sqlx::query] SELECT FROM streams returned 55 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p7yn3c60cs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716499000,"log":"[2026-02-17T13:30:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6051810250942684000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j1wp0q104z","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716500000,"log":"[2026-02-17T15:12:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6101","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-761pm2tlik","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716501000,"log":"[2026-02-17T13:14:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2884186244017937000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q4d3dhfqfk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716502000,"log":"[2026-02-17T14:46:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=xphlrspj","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7yrdaskl4m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716503000,"log":"[2026-02-17T13:19:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3699882548594700000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y9tfqditby","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716504000,"log":"[2026-02-17T14:51:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mchgcrli5o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716505000,"log":"[2026-02-17T13:25:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3975429316130137000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6o474750g7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716506000,"log":"[2026-02-17T14:19:17Z DEBUG sqlx::query] SELECT FROM streams returned 547 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j7b7kvoyg2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716507000,"log":"[2026-02-17T13:50:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 123ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vbv3scrbza","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716508000,"log":"[2026-02-17T13:38:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/292350314252576300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uma4ir8zc7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716509000,"log":"[2026-02-17T15:08:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=3323","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-knz8juw2hg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716510000,"log":"[2026-02-17T13:32:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3578390781373404000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t7jwkl8qms","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716511000,"log":"[2026-02-17T13:40:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/979148658906146000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mqkzkf2wfk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716512000,"log":"[2026-02-17T13:30:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5241602517054591000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uh3qq48zdx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716513000,"log":"[2026-02-17T14:20:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 472 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mkd5dos4pj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716514000,"log":"[2026-02-17T12:40:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nzqawpu13q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716515000,"log":"[2026-02-17T14:23:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 991 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8djmytbmpc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716516000,"log":"[2026-02-17T13:08:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3911917445646247000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wwglv4fhzq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716517000,"log":"[2026-02-17T15:19:45Z TRACE cache::redis] cache set key=query:result:9843","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ibdbzirv6z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716518000,"log":"[2026-02-17T13:21:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7118131863767765000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bcxh26p8jp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716519000,"log":"[2026-02-17T14:52:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t4galwyzm9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716520000,"log":"[2026-02-17T14:13:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 57 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-glabspbn03","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716521000,"log":"[2026-02-17T12:46:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ak3y4jly19","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716522000,"log":"[2026-02-17T13:42:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 411ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y15kc5qdm7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716523000,"log":"[2026-02-17T12:51:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qm858qje2o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716524000,"log":"[2026-02-17T13:34:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6360923851653540000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wyohxk84b3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716525000,"log":"[2026-02-17T12:36:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jo5wx7ngiw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716526000,"log":"[2026-02-17T14:28:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 772 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x2z4az0ovf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716527000,"log":"[2026-02-17T14:14:07Z DEBUG sqlx::query] SELECT FROM streams returned 999 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jkarasotrj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716528000,"log":"[2026-02-17T15:06:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7173","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-th6ii12yey","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716529000,"log":"[2026-02-17T14:55:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8bk5riehct","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716530000,"log":"[2026-02-17T12:40:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7kig5180uq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716531000,"log":"[2026-02-17T13:55:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 123ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-94qfycomps","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716532000,"log":"[2026-02-17T13:54:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 66ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n8aer5attf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716533000,"log":"[2026-02-17T14:54:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tzc12862tj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716534000,"log":"[2026-02-17T13:40:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3851168811422694000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xbrlnq3v3q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716535000,"log":"[2026-02-17T14:37:54Z INFO tonic::transport::server] grpc router request_received client_id=71zmgchw","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5wv22otsyp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716536000,"log":"[2026-02-17T14:46:59Z INFO tonic::transport::server] grpc router request_received client_id=dd755un8","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pwinxxykga","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716537000,"log":"[2026-02-17T13:28:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7270021767914665000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t10xzyso7l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716538000,"log":"[2026-02-17T12:42:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lagkxz1bzp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716539000,"log":"[2026-02-17T13:56:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 407ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-onjiewqwoy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716540000,"log":"[2026-02-17T14:27:00Z DEBUG sqlx::query] DELETE FROM alerts returned 493 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wxbxecy80q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716541000,"log":"[2026-02-17T13:17:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7743746380697402000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fn6a8twa6c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716542000,"log":"[2026-02-17T14:44:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=b654sirl","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bvbc31olkk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716543000,"log":"[2026-02-17T13:46:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 346ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3g6qw1822o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716544000,"log":"[2026-02-17T12:38:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vt50a7at9u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716545000,"log":"[2026-02-17T13:52:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 417ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vdnv6gqci3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716546000,"log":"[2026-02-17T12:41:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kg9d6zn5ss","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716547000,"log":"[2026-02-17T14:01:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 284ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-csi6w40ycn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716548000,"log":"[2026-02-17T13:13:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4149518595995667000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qat9wffwc2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716549000,"log":"[2026-02-17T14:51:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ex3inqhsgu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716550000,"log":"[2026-02-17T15:17:41Z TRACE cache::redis] cache expire key=auth:token:8860","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zj7lnkz5d7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716551000,"log":"[2026-02-17T13:33:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2216419696299873800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h0nraqqzul","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716552000,"log":"[2026-02-17T15:15:08Z TRACE cache::redis] cache get key=org:config:5493","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wzx2r9n03","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716553000,"log":"[2026-02-17T14:38:04Z INFO tonic::transport::server] grpc router request_received client_id=5mhv1tr8","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vozjtz84as","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716554000,"log":"[2026-02-17T13:02:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vr47rnj2ks","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716555000,"log":"[2026-02-17T14:53:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0s8objw5d5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716556000,"log":"[2026-02-17T15:05:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4546","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ib7trg09wy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716557000,"log":"[2026-02-17T14:40:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=4l1mvu68","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ngor2dxnj5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716558000,"log":"[2026-02-17T14:00:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 36ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-szmm1zd20y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716559000,"log":"[2026-02-17T14:45:19Z INFO tonic::transport::server] grpc router request_received client_id=lu5qz5v1","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gzhcxum3mb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716560000,"log":"[2026-02-17T13:15:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8975403226539132000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o8qx44yryu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716561000,"log":"[2026-02-17T12:43:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o9u0qwgooz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716562000,"log":"[2026-02-17T15:17:22Z TRACE cache::redis] cache delete key=user:session:3746","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vh6kpgah37","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716563000,"log":"[2026-02-17T13:08:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oh3cqq4qys","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716564000,"log":"[2026-02-17T13:08:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/186458638617956740.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-caz6m6wuye","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716565000,"log":"[2026-02-17T12:47:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n0n24jv9gn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716566000,"log":"[2026-02-17T14:08:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 46 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8mdinphp78","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716567000,"log":"[2026-02-17T12:54:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-js1g29duqo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716568000,"log":"[2026-02-17T13:48:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 127ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4f9fnkbkrq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716569000,"log":"[2026-02-17T14:45:13Z INFO tonic::transport::server] grpc querier disconnected client_id=m4ove1q9","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nl02m3ng9l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716570000,"log":"[2026-02-17T14:44:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=psmrkvkg","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f3orkvxy6a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716571000,"log":"[2026-02-17T13:07:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ji7z7axhz6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716572000,"log":"[2026-02-17T14:47:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pfnm1gfl","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9f8lz2za3n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716573000,"log":"[2026-02-17T14:49:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2aa1e5ebth","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716574000,"log":"[2026-02-17T13:07:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-un0x9it4go","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716575000,"log":"[2026-02-17T13:07:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eavxw6c8xv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716576000,"log":"[2026-02-17T14:30:42Z DEBUG sqlx::query] SELECT FROM streams returned 199 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5cil2jizkr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716577000,"log":"[2026-02-17T14:58:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-crpc8frmio","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716578000,"log":"[2026-02-17T14:42:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=7pdel22k","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-enh3079quj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716579000,"log":"[2026-02-17T13:03:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wo2sjnuiw8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716580000,"log":"[2026-02-17T13:21:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1937907209215888000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t3b1ucep3p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716581000,"log":"[2026-02-17T12:45:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gpdcyp3ey5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716582000,"log":"[2026-02-17T15:20:03Z TRACE cache::redis] cache set key=org:config:2843","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jko3yadic2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716583000,"log":"[2026-02-17T13:31:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1088911932443772400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j23007zyxj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716584000,"log":"[2026-02-17T13:25:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6138749614668273000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ex5ejy451z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716585000,"log":"[2026-02-17T15:14:28Z TRACE cache::redis] cache delete key=org:config:7344","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j5skr4olos","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716586000,"log":"[2026-02-17T12:49:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ehpfap93m7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716587000,"log":"[2026-02-17T14:51:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ut9taw7w2w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716588000,"log":"[2026-02-17T14:21:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 948 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6dge8oh9ok","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716589000,"log":"[2026-02-17T14:42:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=3h8lhubx","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1jyhdk1exq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716590000,"log":"[2026-02-17T14:31:43Z DEBUG sqlx::query] INSERT FROM users returned 212 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ymodjojmfo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716591000,"log":"[2026-02-17T15:14:31Z TRACE cache::redis] cache miss key=auth:token:8138","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sbgklku7a8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716592000,"log":"[2026-02-17T15:21:51Z TRACE cache::redis] cache set key=auth:token:5821","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mqvmv23cxg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716593000,"log":"[2026-02-17T15:17:49Z TRACE cache::redis] cache miss key=stream:schema:1994","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-108ioo9y8k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716594000,"log":"[2026-02-17T14:29:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 950 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wnbt1ixezq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716595000,"log":"[2026-02-17T12:45:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sjeobmk6rd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716596000,"log":"[2026-02-17T14:30:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 699 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4dg9wxuy3f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716597000,"log":"[2026-02-17T13:08:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9621833001291835000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ngaos67kdw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716598000,"log":"[2026-02-17T14:08:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 228 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c8h8m55sat","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716599000,"log":"[2026-02-17T14:06:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 195ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m2qcsjhh3n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716600000,"log":"[2026-02-17T15:04:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i49074hh5q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716601000,"log":"[2026-02-17T13:53:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 488ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-btq4j77bdr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716602000,"log":"[2026-02-17T13:10:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6611078811801879000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-necmj9chmz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716603000,"log":"[2026-02-17T14:00:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 319ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jmxwjdr3cd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716604000,"log":"[2026-02-17T14:35:12Z INFO tonic::transport::server] grpc ingester connected client_id=nacszp4h","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qqbe5hf67i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716605000,"log":"[2026-02-17T14:40:39Z INFO tonic::transport::server] grpc router request_received client_id=lcu8ajsw","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sfgff3la9z","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716606000,"log":"[2026-02-17T14:59:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cimi3q8ap3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716607000,"log":"[2026-02-17T14:01:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 356ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ggynjeb73z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716608000,"log":"[2026-02-17T13:06:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r092axvdb3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716609000,"log":"[2026-02-17T14:37:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=eah92jcd","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-21kdemnswe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716610000,"log":"[2026-02-17T12:38:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zi8xukzjyj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716611000,"log":"[2026-02-17T15:09:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8700","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bu6cw1hngi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716612000,"log":"[2026-02-17T14:42:39Z INFO tonic::transport::server] grpc router request_received client_id=fdgxwszo","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i1gv40tjoi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716613000,"log":"[2026-02-17T13:57:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 355ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jg3j09q4yj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716614000,"log":"[2026-02-17T14:05:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 196ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-064sq7tom7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716615000,"log":"[2026-02-17T14:18:47Z DEBUG sqlx::query] SELECT FROM streams returned 297 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6psrvmmscp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716616000,"log":"[2026-02-17T12:54:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6una02gkvy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716617000,"log":"[2026-02-17T12:52:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w0eozx00ku","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716618000,"log":"[2026-02-17T12:51:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6p24zklh3m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716619000,"log":"[2026-02-17T13:03:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aid30x14co","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716620000,"log":"[2026-02-17T12:45:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-26g1qagy47","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716621000,"log":"[2026-02-17T14:49:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0v1kpreq3n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716622000,"log":"[2026-02-17T14:28:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 621 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2ya2snivkv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716623000,"log":"[2026-02-17T13:40:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7105662479057168000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ygue5m4j8d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716624000,"log":"[2026-02-17T14:22:50Z DEBUG sqlx::query] DELETE FROM alerts returned 603 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n4kve44a5q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716625000,"log":"[2026-02-17T13:13:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7022488183827766000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w3vsqmrku6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716626000,"log":"[2026-02-17T14:48:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dh98z7cazs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716627000,"log":"[2026-02-17T14:51:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-beot65hi1k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716628000,"log":"[2026-02-17T12:46:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eyfwcw3iy5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716629000,"log":"[2026-02-17T13:17:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9407972611980935000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rug9eaxxcp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716630000,"log":"[2026-02-17T15:03:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r4rr0aqasj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716631000,"log":"[2026-02-17T13:26:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7245537448455869000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o4ox4orhbk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716632000,"log":"[2026-02-17T14:25:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 17 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-krdatf86on","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716633000,"log":"[2026-02-17T13:37:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9883682536641251000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kiw0fc4ga1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716634000,"log":"[2026-02-17T15:21:38Z TRACE cache::redis] cache get key=org:config:5832","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ll3suxjrjt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716635000,"log":"[2026-02-17T12:40:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qx2hwfmgv1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716636000,"log":"[2026-02-17T14:22:28Z DEBUG sqlx::query] INSERT FROM users returned 517 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1sc3nuy510","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716637000,"log":"[2026-02-17T14:00:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 345ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jlc6sc2tzf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716638000,"log":"[2026-02-17T13:05:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hby2bzgasv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716639000,"log":"[2026-02-17T13:34:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9772344479550430000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iqurhb94rq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716640000,"log":"[2026-02-17T13:02:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fukmyhkml9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716641000,"log":"[2026-02-17T13:34:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7875793596872945000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sksfr1lyrb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716642000,"log":"[2026-02-17T14:54:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z9nwqyhhe8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716643000,"log":"[2026-02-17T14:25:52Z DEBUG sqlx::query] SELECT FROM streams returned 446 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1y3jofoilb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716644000,"log":"[2026-02-17T14:02:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 380ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-aa5qyewz2u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716645000,"log":"[2026-02-17T13:59:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 243ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ij3x6yb24p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716646000,"log":"[2026-02-17T14:01:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 350ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-09c6ntpgdx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716647000,"log":"[2026-02-17T13:58:40Z INFO actix_web::middleware::logger] POST /api/_search 200 211ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-epk58fkix2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716648000,"log":"[2026-02-17T13:20:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8404534269314088000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6v36qspm39","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716649000,"log":"[2026-02-17T12:43:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ujeimyb46l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716650000,"log":"[2026-02-17T15:19:12Z TRACE cache::redis] cache hit key=user:session:5349","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-omwl8bfykf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716651000,"log":"[2026-02-17T12:49:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fulifnonfa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716652000,"log":"[2026-02-17T13:03:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r1qd0nsjfh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716653000,"log":"[2026-02-17T15:07:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7469","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l6kjww80im","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716654000,"log":"[2026-02-17T14:53:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yf11rew39e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716655000,"log":"[2026-02-17T13:39:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/675435678428006900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fu12yg2ohr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716656000,"log":"[2026-02-17T13:53:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 498ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3thnnrgfr0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716657000,"log":"[2026-02-17T14:18:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 212 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qfsc2986en","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716658000,"log":"[2026-02-17T15:21:10Z TRACE cache::redis] cache delete key=query:result:6187","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rw0s0ol69s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716659000,"log":"[2026-02-17T13:45:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 375ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s6qamq98yj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716660000,"log":"[2026-02-17T12:39:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hkuzys37r3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716661000,"log":"[2026-02-17T13:54:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 191ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5h7z1rotwj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716662000,"log":"[2026-02-17T12:50:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i5ilulrq5r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716663000,"log":"[2026-02-17T15:11:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1721","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1uwkmcxwhz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716664000,"log":"[2026-02-17T14:43:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=9wkpiqvo","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c56hou44ei","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716665000,"log":"[2026-02-17T12:57:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fdkv74emz5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716666000,"log":"[2026-02-17T15:11:20Z DEBUG metrics::reporter] recorded metric active_connections value=7879","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5bt3ku0o5o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716667000,"log":"[2026-02-17T13:01:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gbgbgeb4pn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716668000,"log":"[2026-02-17T12:40:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tj1unltupq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716669000,"log":"[2026-02-17T13:51:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ab4gtsoqsx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716670000,"log":"[2026-02-17T14:16:52Z DEBUG sqlx::query] SELECT FROM streams returned 164 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i8l11qcy6m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716671000,"log":"[2026-02-17T12:43:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-05b8ixpqpc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716672000,"log":"[2026-02-17T14:14:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 675 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5rfks1rjw6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716673000,"log":"[2026-02-17T12:35:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8eecec0vph","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716674000,"log":"[2026-02-17T12:45:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zzgdl2g5ob","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716675000,"log":"[2026-02-17T15:19:02Z TRACE cache::redis] cache get key=user:session:9491","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wln1bacv0f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716676000,"log":"[2026-02-17T14:01:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 137ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9kwheajpge","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716677000,"log":"[2026-02-17T15:07:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=4933","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r5azcc81km","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716678000,"log":"[2026-02-17T13:04:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ppx616ftpc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716679000,"log":"[2026-02-17T13:23:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9440474362592266000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-op31xeu7ip","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716680000,"log":"[2026-02-17T13:50:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 374ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5pwmvek2so","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716681000,"log":"[2026-02-17T13:23:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6358247962243735000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o8ckrz87b9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716682000,"log":"[2026-02-17T15:06:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=265","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-79vyc85kkj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716683000,"log":"[2026-02-17T14:33:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=l8r5g4q3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rqae1frofo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716684000,"log":"[2026-02-17T14:16:35Z DEBUG sqlx::query] DELETE FROM alerts returned 23 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-46g3n1au2c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716685000,"log":"[2026-02-17T13:04:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-stl3zwjt29","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716686000,"log":"[2026-02-17T15:03:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cfondtivkh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716687000,"log":"[2026-02-17T14:38:13Z INFO tonic::transport::server] grpc querier disconnected client_id=amz28a2h","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d7sa1bsui9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716688000,"log":"[2026-02-17T14:43:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=b86dx586","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3hpgw9c1so","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716689000,"log":"[2026-02-17T13:55:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 280ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ckvdxtjhm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716690000,"log":"[2026-02-17T14:58:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-68wtvmux81","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716691000,"log":"[2026-02-17T12:36:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-33f6dzwy3j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716692000,"log":"[2026-02-17T14:19:25Z DEBUG sqlx::query] DELETE FROM alerts returned 45 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zjvge64lmt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716693000,"log":"[2026-02-17T14:50:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-42p6e8470d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716694000,"log":"[2026-02-17T13:59:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 266ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-amirv32ib3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716695000,"log":"[2026-02-17T13:14:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9523366251591895000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7738nd6qet","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716696000,"log":"[2026-02-17T13:20:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6848194497079361000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7l15f6toj8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716697000,"log":"[2026-02-17T13:55:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 15ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qo53yvvf6a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716698000,"log":"[2026-02-17T13:00:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tnilxvn0yb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716699000,"log":"[2026-02-17T13:19:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9979135965893011000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f9zd7a4h85","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716700000,"log":"[2026-02-17T13:33:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4388062780170168000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9khc42dmen","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716701000,"log":"[2026-02-17T13:44:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 167ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ejfdfxac5t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716702000,"log":"[2026-02-17T13:44:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 99ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lni99ohjzs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716703000,"log":"[2026-02-17T13:03:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tko6rizgg9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716704000,"log":"[2026-02-17T12:53:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u8nojzsum9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716705000,"log":"[2026-02-17T14:19:35Z DEBUG sqlx::query] DELETE FROM alerts returned 358 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i0vlm8de9t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716706000,"log":"[2026-02-17T13:54:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 160ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2fh0klxdm2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716707000,"log":"[2026-02-17T13:59:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 255ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4d5jqnucci","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716708000,"log":"[2026-02-17T12:50:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vrxwpn7inl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716709000,"log":"[2026-02-17T12:59:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o66h530983","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716710000,"log":"[2026-02-17T14:33:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=1pqjfb2b","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8u71ixtskp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716711000,"log":"[2026-02-17T14:21:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 600 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nl7c2gzw57","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716712000,"log":"[2026-02-17T13:48:00Z INFO actix_web::middleware::logger] POST /api/_search 503 465ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5u4uz1a9gi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716713000,"log":"[2026-02-17T14:35:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=eu1isigj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1q4zykacq0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716714000,"log":"[2026-02-17T14:03:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 253ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qivybmyf6h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716715000,"log":"[2026-02-17T14:55:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2vp9f1uqjv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716716000,"log":"[2026-02-17T13:37:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2197078247813253400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-or74e12hgx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716717000,"log":"[2026-02-17T13:23:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9538794798740560000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wzerv0sphd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716718000,"log":"[2026-02-17T13:19:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/160375842656499400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iii985k234","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716719000,"log":"[2026-02-17T12:55:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d6mzb1pw0u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716720000,"log":"[2026-02-17T13:47:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 485ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-o0zxcpbpiv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716721000,"log":"[2026-02-17T14:28:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 716 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2l9xyjhkr2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716722000,"log":"[2026-02-17T13:23:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2157508841921520400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iu8jo2qxnq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716723000,"log":"[2026-02-17T14:33:18Z INFO tonic::transport::server] grpc querier disconnected client_id=g9b23qqm","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tuw7fd81ot","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716724000,"log":"[2026-02-17T13:14:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8484380689008928000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qf3qnqipch","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716725000,"log":"[2026-02-17T14:59:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2fl3gr0eh8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716726000,"log":"[2026-02-17T15:05:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r4qt387pbn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716727000,"log":"[2026-02-17T13:32:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3842100763598930000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fp1rzgtxbn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716728000,"log":"[2026-02-17T13:33:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9001275959465454000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ztv8htjj4n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716729000,"log":"[2026-02-17T14:32:52Z INFO tonic::transport::server] grpc ingester connected client_id=c8tlbedv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qac0komzst","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716730000,"log":"[2026-02-17T12:44:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c2j9xhs9yo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716731000,"log":"[2026-02-17T13:27:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7712976995176719000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n2trwf69sf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716732000,"log":"[2026-02-17T14:59:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kjbijupjnt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716733000,"log":"[2026-02-17T13:23:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4020759173863246300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8jqlqadet5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716734000,"log":"[2026-02-17T14:47:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=f2x0k3df","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zuoeoo6v8l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716735000,"log":"[2026-02-17T12:41:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-texm4b3uig","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716736000,"log":"[2026-02-17T14:47:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=1iyni3p7","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wcyxvg71ge","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716737000,"log":"[2026-02-17T13:17:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2384434501744084000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pa4tyrio2o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716738000,"log":"[2026-02-17T13:08:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5627287636553327000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m9891zsxnt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716739000,"log":"[2026-02-17T14:39:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=hauu9zws","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7ysbge0eej","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716740000,"log":"[2026-02-17T15:09:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3298","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p735c58w2a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716741000,"log":"[2026-02-17T14:44:43Z INFO tonic::transport::server] grpc querier disconnected client_id=ev9p064p","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3xpuigp91e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716742000,"log":"[2026-02-17T13:21:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/551381334098548500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g1a8yvgvaa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716743000,"log":"[2026-02-17T12:47:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bzfny4ai76","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716744000,"log":"[2026-02-17T14:42:43Z INFO tonic::transport::server] grpc querier disconnected client_id=kfputx02","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rr2jdcp3i9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716745000,"log":"[2026-02-17T12:51:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rsb0pmds0l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716746000,"log":"[2026-02-17T14:21:38Z DEBUG sqlx::query] INSERT FROM users returned 8 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k9xbyfounk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716747000,"log":"[2026-02-17T15:09:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3845","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4fpvc9xj0s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716748000,"log":"[2026-02-17T14:50:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1lnr2103ww","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716749000,"log":"[2026-02-17T13:53:20Z INFO actix_web::middleware::logger] POST /api/_search 401 455ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xxn6pa7o7c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716750000,"log":"[2026-02-17T14:33:28Z INFO tonic::transport::server] grpc querier disconnected client_id=ixhvxiuh","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-39t6ekghu3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716751000,"log":"[2026-02-17T15:11:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=5744","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vxdfybz2rt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716752000,"log":"[2026-02-17T13:36:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1704665477280857900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qqzw80n67d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716753000,"log":"[2026-02-17T14:37:03Z INFO tonic::transport::server] grpc querier disconnected client_id=vqyj4bwg","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sxmos3g61w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716754000,"log":"[2026-02-17T15:16:55Z TRACE cache::redis] cache miss key=query:result:1047","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1rqfnewsd5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716755000,"log":"[2026-02-17T14:44:34Z INFO tonic::transport::server] grpc router request_received client_id=uhsmtikb","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b8nevk481y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716756000,"log":"[2026-02-17T15:20:07Z TRACE cache::redis] cache miss key=user:session:5153","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fld6d8f6k4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716757000,"log":"[2026-02-17T12:41:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0uvuij5yc6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716758000,"log":"[2026-02-17T12:43:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l7alar6qnz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716759000,"log":"[2026-02-17T14:46:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=ttokyn1z","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pcv13yfclf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716760000,"log":"[2026-02-17T13:30:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4446506720522367500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-els33xob1h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716761000,"log":"[2026-02-17T13:13:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3155174923852773000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5jsaxrnm6h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716762000,"log":"[2026-02-17T14:41:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rtz6az7c","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-keu9jp6wvn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716763000,"log":"[2026-02-17T13:24:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4779991995502075000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wf9nhwg726","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716764000,"log":"[2026-02-17T13:42:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 100ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-smgnaqbf3g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716765000,"log":"[2026-02-17T14:00:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 151ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ncvy62icq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716766000,"log":"[2026-02-17T14:51:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r0iu55kaej","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716767000,"log":"[2026-02-17T14:40:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=bf8wmban","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p1m4fesjsj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716768000,"log":"[2026-02-17T14:28:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 861 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ader8nhlgf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716769000,"log":"[2026-02-17T15:11:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=6093","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qvqcq1n5go","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716770000,"log":"[2026-02-17T15:20:46Z TRACE cache::redis] cache delete key=auth:token:249","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9w5wvgiuz0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716771000,"log":"[2026-02-17T13:41:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 288ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f1dxnyo6oc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716772000,"log":"[2026-02-17T13:37:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9630939362789683000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qgjor1quns","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716773000,"log":"[2026-02-17T15:04:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d380fpw4q3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716774000,"log":"[2026-02-17T14:26:45Z DEBUG sqlx::query] DELETE FROM alerts returned 43 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n9pqj105gc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716775000,"log":"[2026-02-17T15:14:24Z TRACE cache::redis] cache hit key=stream:schema:848","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tw2r3vwzev","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716776000,"log":"[2026-02-17T13:07:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nsuvhf2pse","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716777000,"log":"[2026-02-17T12:58:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wme8o00htz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716778000,"log":"[2026-02-17T14:55:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xqhn5vhr9q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716779000,"log":"[2026-02-17T13:02:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c5xlcrotx8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716780000,"log":"[2026-02-17T12:51:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ypr9amev8r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716781000,"log":"[2026-02-17T13:58:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 226ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sgfvcylrev","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716782000,"log":"[2026-02-17T13:21:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4767040402093372000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0aong1eecs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716783000,"log":"[2026-02-17T12:52:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pashfs4upd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716784000,"log":"[2026-02-17T15:15:56Z TRACE cache::redis] cache get key=auth:token:7648","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tqyx2b9chz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716785000,"log":"[2026-02-17T13:09:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7436723390129291000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ybn0pn2wko","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716786000,"log":"[2026-02-17T12:52:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mkekut86xe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716787000,"log":"[2026-02-17T13:05:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ox6e6g5g0p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716788000,"log":"[2026-02-17T14:23:35Z DEBUG sqlx::query] DELETE FROM alerts returned 243 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cyd87t6ek2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716789000,"log":"[2026-02-17T13:35:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8269840428767787000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sgj3xs2in8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716790000,"log":"[2026-02-17T14:30:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 642 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-an8u8ka3po","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716791000,"log":"[2026-02-17T12:40:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gyuobwrdya","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716792000,"log":"[2026-02-17T14:34:48Z INFO tonic::transport::server] grpc querier disconnected client_id=2cnpmcrg","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ae5oc4np2o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716793000,"log":"[2026-02-17T15:09:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9198","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g0nsg6hggd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716794000,"log":"[2026-02-17T14:40:38Z INFO tonic::transport::server] grpc querier disconnected client_id=iqi8mcf2","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cgj2n0b4ms","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716795000,"log":"[2026-02-17T13:34:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1523925668703576300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kfuo5nmoa7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716796000,"log":"[2026-02-17T15:17:31Z TRACE cache::redis] cache miss key=auth:token:471","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-brdewrxr60","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716797000,"log":"[2026-02-17T15:08:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=4551","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p9u2n2a4f5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716798000,"log":"[2026-02-17T14:34:28Z INFO tonic::transport::server] grpc querier disconnected client_id=l8lwgurf","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rzrfxd5ia3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716799000,"log":"[2026-02-17T14:04:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 333ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1u4gri4r67","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716800000,"log":"[2026-02-17T14:40:48Z INFO tonic::transport::server] grpc querier disconnected client_id=dh991kib","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-btbh1tk73d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716801000,"log":"[2026-02-17T14:18:27Z DEBUG sqlx::query] SELECT FROM streams returned 527 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bre3r44c30","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716802000,"log":"[2026-02-17T13:17:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8933110645366809000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mel9bpsnyc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716803000,"log":"[2026-02-17T14:54:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-id6msqo9mt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716804000,"log":"[2026-02-17T13:14:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6687347808548059000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7y4btsfls3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716805000,"log":"[2026-02-17T13:03:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ko1iojb32s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716806000,"log":"[2026-02-17T15:11:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=9255","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8rycnxv12c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716807000,"log":"[2026-02-17T13:07:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tl2d5djwng","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716808000,"log":"[2026-02-17T12:50:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lpqlbe2cn3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716809000,"log":"[2026-02-17T14:07:43Z DEBUG sqlx::query] INSERT FROM users returned 589 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mg7fxefk5e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716810000,"log":"[2026-02-17T13:29:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5415013764723795000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qgqim82a0g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716811000,"log":"[2026-02-17T13:04:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ysckdh8gy2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716812000,"log":"[2026-02-17T15:06:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=2431","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rvauyrotg9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716813000,"log":"[2026-02-17T15:19:59Z TRACE cache::redis] cache expire key=stream:schema:9643","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ynvxw0ukd1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716814000,"log":"[2026-02-17T13:31:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9019870592556738000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-poc0a8mpfs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716815000,"log":"[2026-02-17T13:23:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2986484883346345500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-52hj2eu5hm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716816000,"log":"[2026-02-17T13:10:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2051624527542013200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qjjjbtzxsd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716817000,"log":"[2026-02-17T14:20:03Z DEBUG sqlx::query] INSERT FROM users returned 987 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1inqmw44xx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716818000,"log":"[2026-02-17T14:41:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=m46mc3wu","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v1rtxqj4oy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716819000,"log":"[2026-02-17T12:37:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sjgeqe9e13","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716820000,"log":"[2026-02-17T13:09:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7094391456226996000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uvra4b81at","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716821000,"log":"[2026-02-17T12:52:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8d1x2t5ov2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716822000,"log":"[2026-02-17T13:00:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vyu7ac8dfg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716823000,"log":"[2026-02-17T13:28:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6032950813064423000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-79hv4qczcx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716824000,"log":"[2026-02-17T13:28:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5104821901290649000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cyzoeooqg2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716825000,"log":"[2026-02-17T12:57:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-86gobdwl4o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716826000,"log":"[2026-02-17T13:54:45Z INFO actix_web::middleware::logger] GET /api/_search 503 445ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7m6raifoz1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716827000,"log":"[2026-02-17T13:40:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9270346571340833000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fkbblvg1iz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716828000,"log":"[2026-02-17T15:04:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bhytffpriv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716829000,"log":"[2026-02-17T14:32:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=jf780lvd","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fi176ri4h8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716830000,"log":"[2026-02-17T14:00:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 28ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ncl293szcw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716831000,"log":"[2026-02-17T13:13:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6303332456291908000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dgai84h8kn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716832000,"log":"[2026-02-17T13:01:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aba550tr7u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716833000,"log":"[2026-02-17T14:31:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 621 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rfjl8uu0j6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716834000,"log":"[2026-02-17T13:01:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-npmwcbprwl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716835000,"log":"[2026-02-17T13:06:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-km59qn4h0a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716836000,"log":"[2026-02-17T14:22:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 615 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-013nsenc5d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716837000,"log":"[2026-02-17T14:55:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f5g8lonaqm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716838000,"log":"[2026-02-17T14:53:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fnryytwbyg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716839000,"log":"[2026-02-17T13:03:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0cnue36mgk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716840000,"log":"[2026-02-17T14:47:24Z INFO tonic::transport::server] grpc router request_received client_id=4byxybnt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b45bh69aff","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716841000,"log":"[2026-02-17T13:35:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9660845709232806000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mqcc2eixwl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716842000,"log":"[2026-02-17T13:38:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9126828033054865000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wqlsucvwsp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716843000,"log":"[2026-02-17T14:40:49Z INFO tonic::transport::server] grpc router request_received client_id=f7q1axot","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rozepswwif","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716844000,"log":"[2026-02-17T13:10:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2358781918342672000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oqumy70o9s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716845000,"log":"[2026-02-17T12:50:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zxx0xfpbd6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716846000,"log":"[2026-02-17T13:17:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1706230688751488800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lp830dw2de","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716847000,"log":"[2026-02-17T15:00:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9brx25wx5q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716848000,"log":"[2026-02-17T15:16:44Z TRACE cache::redis] cache get key=stream:schema:757","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qnug0irion","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716849000,"log":"[2026-02-17T14:11:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 829 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v34hmi5lx8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716850000,"log":"[2026-02-17T13:24:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8461307077651408000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wmxcg5r1pv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716851000,"log":"[2026-02-17T15:19:35Z TRACE cache::redis] cache expire key=query:result:6437","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e4ee0qq8v9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716852000,"log":"[2026-02-17T14:42:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=zmym6idf","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t59udz3kc3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716853000,"log":"[2026-02-17T14:30:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 717 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yfxcjc0tph","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716854000,"log":"[2026-02-17T14:39:48Z INFO tonic::transport::server] grpc querier disconnected client_id=jes39z93","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qfdpwrcc04","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716855000,"log":"[2026-02-17T13:04:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-302azpgzbk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716856000,"log":"[2026-02-17T12:49:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ww2ygz5td7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716857000,"log":"[2026-02-17T14:52:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lzwfeb276f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716858000,"log":"[2026-02-17T14:25:53Z DEBUG sqlx::query] INSERT FROM users returned 670 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-92l81dlagy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716859000,"log":"[2026-02-17T12:51:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0fn2neshoc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716860000,"log":"[2026-02-17T15:02:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w81e14zbfr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716861000,"log":"[2026-02-17T13:08:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eppv42fy53","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716862000,"log":"[2026-02-17T12:46:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g4j3l7mflw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716863000,"log":"[2026-02-17T15:01:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n4jo8aoo5w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716864000,"log":"[2026-02-17T14:50:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2umw1zuu5h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716865000,"log":"[2026-02-17T13:15:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8873450988800125000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-in957521ym","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716866000,"log":"[2026-02-17T14:19:10Z DEBUG sqlx::query] DELETE FROM alerts returned 321 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x5ur6bg92w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716867000,"log":"[2026-02-17T12:58:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-42o1vbuw3q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716868000,"log":"[2026-02-17T15:18:07Z TRACE cache::redis] cache miss key=user:session:8649","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cqgg63j6gn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716869000,"log":"[2026-02-17T13:32:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6167219562506783000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2rb6adrd83","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716870000,"log":"[2026-02-17T14:33:08Z INFO tonic::transport::server] grpc querier disconnected client_id=bnmz971z","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i0gnn1ggio","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716871000,"log":"[2026-02-17T14:18:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 297 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-27km5mopla","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716872000,"log":"[2026-02-17T15:18:11Z TRACE cache::redis] cache expire key=auth:token:6832","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h2j6theabv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716873000,"log":"[2026-02-17T13:06:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m8ahg8j3f7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716874000,"log":"[2026-02-17T13:29:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7371411104685648000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x7e65lfjwv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716875000,"log":"[2026-02-17T12:44:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tolm6pms69","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716876000,"log":"[2026-02-17T15:00:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ftfnv35zwq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716877000,"log":"[2026-02-17T15:19:23Z TRACE cache::redis] cache expire key=org:config:4223","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v0melxxiv6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716878000,"log":"[2026-02-17T15:08:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=6176","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4n4v9soa9q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716879000,"log":"[2026-02-17T14:17:55Z DEBUG sqlx::query] DELETE FROM alerts returned 751 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vocvmgxc7r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716880000,"log":"[2026-02-17T14:53:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-31qtzty686","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716881000,"log":"[2026-02-17T13:38:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1840189746670888200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eav6xcsjrt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716882000,"log":"[2026-02-17T13:33:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4901765385214720000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-npv6chrzsw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716883000,"log":"[2026-02-17T13:32:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6229029816600771000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n1zksg7zs4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716884000,"log":"[2026-02-17T13:26:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7754519686760648000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jttrowgvai","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716885000,"log":"[2026-02-17T12:38:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vbnr7w7rh2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716886000,"log":"[2026-02-17T13:10:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1586613345335534600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b2y21f6j7p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716887000,"log":"[2026-02-17T14:20:17Z DEBUG sqlx::query] SELECT FROM streams returned 598 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ju9ldjyro","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716888000,"log":"[2026-02-17T13:42:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 90ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wxeskzaszm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716889000,"log":"[2026-02-17T14:52:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-22bgomw984","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716890000,"log":"[2026-02-17T12:57:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vme31bo2x2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716891000,"log":"[2026-02-17T12:52:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3e2nvcaf0w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716892000,"log":"[2026-02-17T14:27:08Z DEBUG sqlx::query] INSERT FROM users returned 360 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8s26wldzvq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716893000,"log":"[2026-02-17T14:31:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 899 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-46n5lzp0dl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716894000,"log":"[2026-02-17T15:09:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7946","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y04odoyfnn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716895000,"log":"[2026-02-17T14:25:03Z DEBUG sqlx::query] INSERT FROM users returned 480 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ygy70sh5if","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716896000,"log":"[2026-02-17T13:45:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f3m428fm5d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716897000,"log":"[2026-02-17T14:37:44Z INFO tonic::transport::server] grpc router request_received client_id=9v5vbxyv","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zuza91932v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716898000,"log":"[2026-02-17T14:37:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=he4sizgt","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-660aogc532","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716899000,"log":"[2026-02-17T14:24:12Z DEBUG sqlx::query] SELECT FROM streams returned 447 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wiiy47twcf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716900000,"log":"[2026-02-17T15:02:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iblw4hl968","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716901000,"log":"[2026-02-17T13:53:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 235ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x3y6gvph9l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716902000,"log":"[2026-02-17T14:42:09Z INFO tonic::transport::server] grpc router request_received client_id=jqikul6d","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p3ab9l8oa5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716903000,"log":"[2026-02-17T13:49:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 481ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k0yuwb5e4g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716904000,"log":"[2026-02-17T12:57:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qfjfbgogv1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716905000,"log":"[2026-02-17T14:17:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 971 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5c2838ltu8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716906000,"log":"[2026-02-17T14:44:53Z INFO tonic::transport::server] grpc querier disconnected client_id=8rpi0thr","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qjrmdei1gb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716907000,"log":"[2026-02-17T13:26:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4833536231422761000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sci16uggtm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716908000,"log":"[2026-02-17T12:46:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p5n67cpn3c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716909000,"log":"[2026-02-17T14:23:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 698 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ezklzfmbq2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716910000,"log":"[2026-02-17T12:38:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k9pfk7cmny","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716911000,"log":"[2026-02-17T15:21:39Z TRACE cache::redis] cache set key=stream:schema:7613","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fm9z4y392y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716912000,"log":"[2026-02-17T13:57:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 482ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t7fexkgelx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716913000,"log":"[2026-02-17T13:32:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7337940721482585000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0w2ktz24si","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716914000,"log":"[2026-02-17T15:00:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b0vc44fvv7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716915000,"log":"[2026-02-17T12:48:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uokmfjcbc1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716916000,"log":"[2026-02-17T12:39:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3upt2gbqoc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716917000,"log":"[2026-02-17T14:35:42Z INFO tonic::transport::server] grpc ingester connected client_id=quao2fd5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f8ohd834ow","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716918000,"log":"[2026-02-17T12:41:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vg4395drsq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716919000,"log":"[2026-02-17T14:38:12Z INFO tonic::transport::server] grpc ingester connected client_id=nxmr6t8c","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r50516724l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716920000,"log":"[2026-02-17T15:15:15Z TRACE cache::redis] cache set key=query:result:5151","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bq6p71q9wr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716921000,"log":"[2026-02-17T15:01:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qgomzs3pd2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716922000,"log":"[2026-02-17T13:37:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6323015631359865000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zjnu3nv7r2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716923000,"log":"[2026-02-17T14:57:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5ysu8em9eg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716924000,"log":"[2026-02-17T13:15:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4912843948200327000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-08iazzji2x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716925000,"log":"[2026-02-17T14:53:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6t7smfm6d2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716926000,"log":"[2026-02-17T14:38:49Z INFO tonic::transport::server] grpc router request_received client_id=7p8o1omo","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5qhzz08uqw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716927000,"log":"[2026-02-17T13:36:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6676454651393808000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hqz2lbgiu2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716928000,"log":"[2026-02-17T14:05:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 483ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-23nfcleipw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716929000,"log":"[2026-02-17T12:59:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o3ivzaiktr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716930000,"log":"[2026-02-17T14:19:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 78 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s9mporeoqj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716931000,"log":"[2026-02-17T15:13:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5098","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j6croj31yi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716932000,"log":"[2026-02-17T14:39:53Z INFO tonic::transport::server] grpc querier disconnected client_id=e4v9vzpf","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bdv5vvrfqo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716933000,"log":"[2026-02-17T15:00:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2zkw0uapjk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716934000,"log":"[2026-02-17T14:54:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-50p7rfmh3a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716935000,"log":"[2026-02-17T14:57:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6kvyd80tyl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716936000,"log":"[2026-02-17T14:00:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 462ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v8kmf15w1e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716937000,"log":"[2026-02-17T12:36:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gdb9g2kssn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716938000,"log":"[2026-02-17T13:06:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gln5uqf3n4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716939000,"log":"[2026-02-17T14:03:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 277ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-he1h4vgtg4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716940000,"log":"[2026-02-17T15:04:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-azy2vymdbq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716941000,"log":"[2026-02-17T14:21:03Z DEBUG sqlx::query] INSERT FROM users returned 973 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zi6n2telfq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716942000,"log":"[2026-02-17T14:14:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 374 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-93vaxkiz0n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716943000,"log":"[2026-02-17T14:06:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 408ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-72r4coch1k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716944000,"log":"[2026-02-17T14:24:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 821 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ohter6q6sv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716945000,"log":"[2026-02-17T13:20:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3107272462267323000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jxe312hdsb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716946000,"log":"[2026-02-17T14:47:44Z INFO tonic::transport::server] grpc router request_received client_id=drphh78m","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rzp86q96wj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716947000,"log":"[2026-02-17T14:42:08Z INFO tonic::transport::server] grpc querier disconnected client_id=femexcln","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ppv0opsccl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716948000,"log":"[2026-02-17T12:55:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vu1ooemc9o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716949000,"log":"[2026-02-17T15:18:25Z TRACE cache::redis] cache miss key=query:result:601","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ui7onnwpmw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716950000,"log":"[2026-02-17T14:31:25Z DEBUG sqlx::query] DELETE FROM alerts returned 18 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kwdphni5bi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716951000,"log":"[2026-02-17T14:35:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=xelvs546","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ipl95b393f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716952000,"log":"[2026-02-17T14:29:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 628 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a72rdiduvr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716953000,"log":"[2026-02-17T15:16:21Z TRACE cache::redis] cache set key=auth:token:8537","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zrlpn5asyv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716954000,"log":"[2026-02-17T13:02:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r7b5nylkfe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716955000,"log":"[2026-02-17T12:39:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-37m0yob2vx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716956000,"log":"[2026-02-17T13:13:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/904026397988659800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-auel8wxus0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716957000,"log":"[2026-02-17T13:10:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9725448004724220000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x9m1etlt6t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716958000,"log":"[2026-02-17T14:05:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 72ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-40w84imjdo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716959000,"log":"[2026-02-17T13:29:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/433949658437118300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ufqxqhkmls","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716960000,"log":"[2026-02-17T14:00:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 368ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0fn1oq2sdx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716961000,"log":"[2026-02-17T12:53:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gtnvei80xr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716962000,"log":"[2026-02-17T14:33:48Z INFO tonic::transport::server] grpc querier disconnected client_id=c3oy68kw","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3afxp196lb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716963000,"log":"[2026-02-17T13:35:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2863355944943921700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kcr7lbv5c9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716964000,"log":"[2026-02-17T13:44:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 415ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2d588jp0z2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716965000,"log":"[2026-02-17T15:12:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=447","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s0m58rlaj0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716966000,"log":"[2026-02-17T12:49:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1jxiyn6zk0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716967000,"log":"[2026-02-17T12:53:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gpmf7exi4h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716968000,"log":"[2026-02-17T14:42:03Z INFO tonic::transport::server] grpc querier disconnected client_id=98oe5pwd","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k4za158jij","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716969000,"log":"[2026-02-17T14:41:03Z INFO tonic::transport::server] grpc querier disconnected client_id=e0n2er13","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ihrmkk7r0r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716970000,"log":"[2026-02-17T12:42:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2kcdzlxu11","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716971000,"log":"[2026-02-17T15:10:10Z DEBUG metrics::reporter] recorded metric active_connections value=7676","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lthnil1ws8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716972000,"log":"[2026-02-17T14:48:27Z INFO tonic::transport::server] grpc ingester connected client_id=bvn5hrcl","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-du4un99qtu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716973000,"log":"[2026-02-17T12:55:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hlknjrq5l7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716974000,"log":"[2026-02-17T15:02:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2e4wwevhxz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716975000,"log":"[2026-02-17T14:41:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ob8bdjhd","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tcdkjt8jib","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716976000,"log":"[2026-02-17T13:56:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 279ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tjck0f2y57","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716977000,"log":"[2026-02-17T13:20:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/141631058327287280.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-11z006gnjq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716978000,"log":"[2026-02-17T12:52:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-97npr3rnm9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716979000,"log":"[2026-02-17T13:42:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kejm56py46","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716980000,"log":"[2026-02-17T12:37:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bqw6drdct8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716981000,"log":"[2026-02-17T13:48:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 329ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r6yjc55ph1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716982000,"log":"[2026-02-17T13:18:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/174650174509962700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5dwoikdbiw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716983000,"log":"[2026-02-17T14:20:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 942 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ui3ww8uwpp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716984000,"log":"[2026-02-17T13:15:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/897240127255016800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e9arv0xyfg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716985000,"log":"[2026-02-17T13:30:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/287717331261136960.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9q3v662poc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716986000,"log":"[2026-02-17T15:11:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=7779","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qjs0a2lpev","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716987000,"log":"[2026-02-17T12:40:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-01u9bvib3s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716988000,"log":"[2026-02-17T13:18:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9650043620886174000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e4w92yp17c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716989000,"log":"[2026-02-17T13:24:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4233292269407851000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f1i3pe1wvs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716990000,"log":"[2026-02-17T12:35:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w3i3v28736","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716991000,"log":"[2026-02-17T14:32:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=krxy1q07","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rrweesiyee","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716992000,"log":"[2026-02-17T15:11:25Z DEBUG metrics::reporter] recorded metric active_connections value=5824","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c438i4h29s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716993000,"log":"[2026-02-17T14:53:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zwr33eheri","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331716994000,"log":"[2026-02-17T14:57:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5alge5vh3e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716995000,"log":"[2026-02-17T14:34:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=tb4cnuii","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uzuesi0pnk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716996000,"log":"[2026-02-17T14:40:07Z INFO tonic::transport::server] grpc ingester connected client_id=ura09bkw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-84cr1h5wh4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716997000,"log":"[2026-02-17T13:13:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5239744559783092000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aw9warsopm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331716998000,"log":"[2026-02-17T14:00:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 316ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-atxse9klp0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331716999000,"log":"[2026-02-17T15:12:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2158","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qmpi64lvui","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717000000,"log":"[2026-02-17T13:06:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hlpgkd50z5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717001000,"log":"[2026-02-17T13:17:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5056354062226233000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q28rysj0zy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717002000,"log":"[2026-02-17T14:43:48Z INFO tonic::transport::server] grpc querier disconnected client_id=w2tj4c41","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-84b133qarz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717003000,"log":"[2026-02-17T13:14:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3246199003702006300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s67njq640k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717004000,"log":"[2026-02-17T13:15:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2688607128106684000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qjy8k8c57h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717005000,"log":"[2026-02-17T12:55:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-okmhzso6qe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717006000,"log":"[2026-02-17T13:24:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2707486338767859000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-142pcv685n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717007000,"log":"[2026-02-17T12:39:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-699mmufwzf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717008000,"log":"[2026-02-17T13:17:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1014518050259236600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rlk3x9yc2n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717009000,"log":"[2026-02-17T14:29:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 838 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3rtz52o86g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717010000,"log":"[2026-02-17T14:35:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=1ue2tvpj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qvj8cqd32x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717011000,"log":"[2026-02-17T14:53:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wsrgxh09tj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717012000,"log":"[2026-02-17T14:23:17Z DEBUG sqlx::query] SELECT FROM streams returned 458 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eli6s2uv88","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717013000,"log":"[2026-02-17T13:24:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5591471561848993000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2kt997okz0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717014000,"log":"[2026-02-17T13:19:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7397116288797238000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-js3m789q94","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717015000,"log":"[2026-02-17T13:16:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8629925176098913000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t44tukrpvx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717016000,"log":"[2026-02-17T14:48:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=fu7tlxi3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yn2htowysn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717017000,"log":"[2026-02-17T15:19:54Z TRACE cache::redis] cache hit key=stream:schema:1135","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q10xnaxfwg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717018000,"log":"[2026-02-17T13:04:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m3q7grq1b3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717019000,"log":"[2026-02-17T12:59:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xx6v6i3f0y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717020000,"log":"[2026-02-17T14:14:32Z DEBUG sqlx::query] SELECT FROM streams returned 438 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q0b5ee5s6b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717021000,"log":"[2026-02-17T14:52:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1roblp83tc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717022000,"log":"[2026-02-17T14:12:05Z DEBUG sqlx::query] DELETE FROM alerts returned 323 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yrl5bygmmu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717023000,"log":"[2026-02-17T13:09:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/330083144638442240.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lafirzpbuq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717024000,"log":"[2026-02-17T13:29:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7416603299772599000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8gravsfc2u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717025000,"log":"[2026-02-17T14:58:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ull229z13g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717026000,"log":"[2026-02-17T13:13:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8013791350741679000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s48wbk6i5a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717027000,"log":"[2026-02-17T14:56:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1ombvxdeka","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717028000,"log":"[2026-02-17T14:45:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=byp48jjp","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-13jw0igrrc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717029000,"log":"[2026-02-17T12:53:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l9yvmj3kf6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717030000,"log":"[2026-02-17T12:56:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1bnra5y76r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717031000,"log":"[2026-02-17T13:40:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9088832004403462000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5qbreb9jry","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717032000,"log":"[2026-02-17T13:35:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4644134732850074000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fjwoc25jx9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717033000,"log":"[2026-02-17T15:16:58Z TRACE cache::redis] cache delete key=org:config:5608","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1t5w65gbqn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717034000,"log":"[2026-02-17T13:12:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3142229734200461300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o4mnomebqo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717035000,"log":"[2026-02-17T13:21:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3970198727166212000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6kfnefw64u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717036000,"log":"[2026-02-17T13:50:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 487ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xey70jmyqh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717037000,"log":"[2026-02-17T14:53:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-povyvweck3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717038000,"log":"[2026-02-17T14:28:58Z DEBUG sqlx::query] INSERT FROM users returned 502 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-exkjmm07iz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717039000,"log":"[2026-02-17T14:06:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 127ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-isx38zollh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717040000,"log":"[2026-02-17T14:42:58Z INFO tonic::transport::server] grpc querier disconnected client_id=9791x1oy","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y3piutrmwc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717041000,"log":"[2026-02-17T12:58:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-021k3gmk61","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717042000,"log":"[2026-02-17T14:57:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b5j20zq81g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717043000,"log":"[2026-02-17T15:06:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7038","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jcr7w5qjoa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717044000,"log":"[2026-02-17T13:11:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1701380136825925000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aar7zht2cc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717045000,"log":"[2026-02-17T13:36:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9562785839450253000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ouxh9h8uq8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717046000,"log":"[2026-02-17T15:07:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1249","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y0hyraxvfo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717047000,"log":"[2026-02-17T12:38:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rj409digxj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717048000,"log":"[2026-02-17T15:12:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1828","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rcjejzoi54","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717049000,"log":"[2026-02-17T12:54:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lxze70gq2j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717050000,"log":"[2026-02-17T12:54:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pysewdgcmu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717051000,"log":"[2026-02-17T13:00:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kx5b0kb2n0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717052000,"log":"[2026-02-17T13:56:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 390ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m0a975z6wj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717053000,"log":"[2026-02-17T13:53:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 253ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y3ux9whlq6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717054000,"log":"[2026-02-17T15:12:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1869","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kn7qs43bi7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717055000,"log":"[2026-02-17T13:19:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3920707588870921700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e7h5t44buh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717056000,"log":"[2026-02-17T15:18:58Z TRACE cache::redis] cache delete key=org:config:3970","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bg545dweso","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717057000,"log":"[2026-02-17T12:38:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1d537kj1a1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717058000,"log":"[2026-02-17T14:39:43Z INFO tonic::transport::server] grpc querier disconnected client_id=sxrwuv1s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bsr1yqnqfw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717059000,"log":"[2026-02-17T15:17:58Z TRACE cache::redis] cache delete key=org:config:4812","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zoooe8jcyq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717060000,"log":"[2026-02-17T13:41:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9323635877785040000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-if2nes3rqb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717061000,"log":"[2026-02-17T13:07:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1pgchg27ha","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717062000,"log":"[2026-02-17T15:20:59Z TRACE cache::redis] cache expire key=stream:schema:4253","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ipipj6w593","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717063000,"log":"[2026-02-17T13:31:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/942387348418565200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xlnhfkx7tx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717064000,"log":"[2026-02-17T13:36:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8162692262214864000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8prf9eai42","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717065000,"log":"[2026-02-17T14:44:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=aizblcrg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tk4iwuwdqm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717066000,"log":"[2026-02-17T13:56:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 86ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tk4znawm79","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717067000,"log":"[2026-02-17T14:49:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l1gb6a77dr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717068000,"log":"[2026-02-17T12:51:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v1r189q2ga","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717069000,"log":"[2026-02-17T13:44:00Z INFO actix_web::middleware::logger] POST /api/_search 204 494ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r8vgehfrc5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717070000,"log":"[2026-02-17T15:06:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1327","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aolxkbkn8f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717071000,"log":"[2026-02-17T15:10:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5081","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wsifzxe82r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717072000,"log":"[2026-02-17T13:51:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 441ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fsw1yostgj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717073000,"log":"[2026-02-17T14:41:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=med4re8f","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7yq9nsqq2d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717074000,"log":"[2026-02-17T13:01:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-33l4t4waxb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717075000,"log":"[2026-02-17T13:13:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6371358119340713000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wxbtjbnj6w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717076000,"log":"[2026-02-17T15:01:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yp2ys3rsie","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717077000,"log":"[2026-02-17T12:42:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o8w7e01n82","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717078000,"log":"[2026-02-17T14:10:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 123 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fax7jjslsb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717079000,"log":"[2026-02-17T13:00:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f7eux9u8jz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717080000,"log":"[2026-02-17T14:34:08Z INFO tonic::transport::server] grpc querier disconnected client_id=4kl5kbs8","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ovm7svm15f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717081000,"log":"[2026-02-17T14:04:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 265ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f4j2uvl127","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717082000,"log":"[2026-02-17T14:12:32Z DEBUG sqlx::query] SELECT FROM streams returned 863 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q30w776idd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717083000,"log":"[2026-02-17T13:42:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 74ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lv8kmrzv7g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717084000,"log":"[2026-02-17T13:32:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5697763112204601000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vr4pj9uqh6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717085000,"log":"[2026-02-17T15:18:26Z TRACE cache::redis] cache get key=auth:token:3306","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-szbg3krh2d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717086000,"log":"[2026-02-17T12:51:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a76ea4ngqa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717087000,"log":"[2026-02-17T14:28:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 445 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x39j5fn923","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717088000,"log":"[2026-02-17T14:38:34Z INFO tonic::transport::server] grpc router request_received client_id=15xa30mm","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nxwkhlmf2x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717089000,"log":"[2026-02-17T14:14:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 569 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-neh6l6oaub","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717090000,"log":"[2026-02-17T15:11:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1172","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pyjkfmcnas","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717091000,"log":"[2026-02-17T14:08:47Z DEBUG sqlx::query] SELECT FROM streams returned 793 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6c8b22c3k5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717092000,"log":"[2026-02-17T14:04:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 187ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jy8qa00m9t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717093000,"log":"[2026-02-17T14:01:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 467ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qo3mitf8j4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717094000,"log":"[2026-02-17T13:58:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 390ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dqq84nfjem","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717095000,"log":"[2026-02-17T14:00:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 423ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-metdw33g9j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717096000,"log":"[2026-02-17T13:43:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 147ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7ykcbkbqgc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717097000,"log":"[2026-02-17T13:15:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4620871591463796000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-75hrfheg6y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717098000,"log":"[2026-02-17T12:43:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a952tozowz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717099000,"log":"[2026-02-17T14:13:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 886 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eoulnwr2k3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717100000,"log":"[2026-02-17T14:30:40Z DEBUG sqlx::query] DELETE FROM alerts returned 985 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uslkaetzfk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717101000,"log":"[2026-02-17T14:05:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 26ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-trzwt50t12","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717102000,"log":"[2026-02-17T14:01:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 456ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-baynr19jfc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717103000,"log":"[2026-02-17T14:39:18Z INFO tonic::transport::server] grpc querier disconnected client_id=whof7s7j","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yqq36ao8fc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717104000,"log":"[2026-02-17T12:52:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tz3bdib9ce","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717105000,"log":"[2026-02-17T15:07:00Z DEBUG metrics::reporter] recorded metric active_connections value=885","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j4hy9dv4hv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717106000,"log":"[2026-02-17T15:15:20Z TRACE cache::redis] cache get key=query:result:9283","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jtia9kluej","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717107000,"log":"[2026-02-17T14:06:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 34ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hend2xo09w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717108000,"log":"[2026-02-17T13:25:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8611192601379262000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mnlb71ryne","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717109000,"log":"[2026-02-17T13:08:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6824519227659391000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-92lgbox8ch","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717110000,"log":"[2026-02-17T12:40:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mfdq22aba6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717111000,"log":"[2026-02-17T13:28:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3234560095460425700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vz0c1ubr3x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717112000,"log":"[2026-02-17T14:21:15Z DEBUG sqlx::query] DELETE FROM alerts returned 832 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eyl6f8ckkw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717113000,"log":"[2026-02-17T13:18:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/668174504643881300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dlritmhhf7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717114000,"log":"[2026-02-17T12:38:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9c10yw2yln","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717115000,"log":"[2026-02-17T14:09:40Z DEBUG sqlx::query] DELETE FROM alerts returned 270 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r7xkwqsge5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717116000,"log":"[2026-02-17T14:16:37Z DEBUG sqlx::query] SELECT FROM streams returned 967 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c75pvo2luf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717117000,"log":"[2026-02-17T14:42:18Z INFO tonic::transport::server] grpc querier disconnected client_id=g8mqrvr3","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hj0gtmawc0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717118000,"log":"[2026-02-17T13:31:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7519955928636269000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eozqry4d51","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717119000,"log":"[2026-02-17T13:53:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dcct9s3j2y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717120000,"log":"[2026-02-17T14:52:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1ne4taibqe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717121000,"log":"[2026-02-17T13:03:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-16p3fglvdh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717122000,"log":"[2026-02-17T14:34:34Z INFO tonic::transport::server] grpc router request_received client_id=r0aph2y9","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0wajszyv6m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717123000,"log":"[2026-02-17T13:46:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 431ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zbyel4kyvu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717124000,"log":"[2026-02-17T13:02:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yusznylsnv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717125000,"log":"[2026-02-17T14:02:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 174ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j4htsph1k1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717126000,"log":"[2026-02-17T14:31:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 289 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vxrk0nc5ce","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717127000,"log":"[2026-02-17T14:51:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qy49i9ebue","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717128000,"log":"[2026-02-17T13:41:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 115ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-97jbsi7ie9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717129000,"log":"[2026-02-17T13:39:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1595619817869376000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fq59j7f4qb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717130000,"log":"[2026-02-17T13:24:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5213561183387221000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u1by98u9r0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717131000,"log":"[2026-02-17T14:19:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 301 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2p9lminryk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717132000,"log":"[2026-02-17T13:26:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/483574647375018000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ooggkxboqs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717133000,"log":"[2026-02-17T13:17:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8528767578481829000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-puerr4suql","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717134000,"log":"[2026-02-17T15:11:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=833","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-53pynhe1n0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717135000,"log":"[2026-02-17T13:10:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2748973475341374000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yjvswwelmv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717136000,"log":"[2026-02-17T13:12:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3226288032696269300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hvjmmnka4q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717137000,"log":"[2026-02-17T14:36:44Z INFO tonic::transport::server] grpc router request_received client_id=uzlbmqfj","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ygnn7lanr4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717138000,"log":"[2026-02-17T14:13:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 614 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y8x7h7gi3b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717139000,"log":"[2026-02-17T14:34:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=mz9jpfss","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8qow8q9zx2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717140000,"log":"[2026-02-17T13:35:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8996896899693337000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mw0a6c2zsu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717141000,"log":"[2026-02-17T12:54:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4i41gecl0b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717142000,"log":"[2026-02-17T14:03:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 427ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oua7qc411m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717143000,"log":"[2026-02-17T12:58:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k6ldbkihrd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717144000,"log":"[2026-02-17T14:14:27Z DEBUG sqlx::query] SELECT FROM streams returned 181 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y1tgiadm1f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717145000,"log":"[2026-02-17T15:15:59Z TRACE cache::redis] cache expire key=stream:schema:2506","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-289sii0t94","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717146000,"log":"[2026-02-17T14:20:43Z DEBUG sqlx::query] INSERT FROM users returned 153 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rlbaxaiwrj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717147000,"log":"[2026-02-17T13:41:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6437746210181731000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zkq7c83gqm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717148000,"log":"[2026-02-17T13:33:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1586011635599562500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aspbqludpa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717149000,"log":"[2026-02-17T13:27:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6817555688424797000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kgph9gmoq4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717150000,"log":"[2026-02-17T13:55:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 251ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uckhzc4thc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717151000,"log":"[2026-02-17T14:53:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-52aeu51jvj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717152000,"log":"[2026-02-17T14:50:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hxlflqegxv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717153000,"log":"[2026-02-17T13:21:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4690414709192883000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6g873w51ym","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717154000,"log":"[2026-02-17T13:23:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5648733068677005000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1qmqo0184l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717155000,"log":"[2026-02-17T13:40:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6358558893521818000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ugnchpvnik","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717156000,"log":"[2026-02-17T14:40:19Z INFO tonic::transport::server] grpc router request_received client_id=lorpwsm1","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dhwf5gdl28","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717157000,"log":"[2026-02-17T13:54:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 61ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m30i4hhgyo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717158000,"log":"[2026-02-17T13:23:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2884852931159429600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pu3m1jv2ns","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717159000,"log":"[2026-02-17T13:25:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5822521589770770000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1590s4higz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717160000,"log":"[2026-02-17T13:20:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9569683746645019000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-szm7oiic1o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717161000,"log":"[2026-02-17T15:07:45Z DEBUG metrics::reporter] recorded metric active_connections value=301","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l1s9gncuiv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717162000,"log":"[2026-02-17T14:12:07Z DEBUG sqlx::query] SELECT FROM streams returned 96 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xqk8gpf5mt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717163000,"log":"[2026-02-17T14:17:20Z DEBUG sqlx::query] DELETE FROM alerts returned 467 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bzi7kxrb2q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717164000,"log":"[2026-02-17T13:13:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/605849908484241700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nlrepp74ts","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717165000,"log":"[2026-02-17T13:43:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 270ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yj17tlrdue","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717166000,"log":"[2026-02-17T14:33:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=f8q1guyw","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tp7uqc4h9v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717167000,"log":"[2026-02-17T14:53:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xc7vvij42b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717168000,"log":"[2026-02-17T15:09:25Z DEBUG metrics::reporter] recorded metric active_connections value=1844","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uynuddou32","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717169000,"log":"[2026-02-17T15:03:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ry29fjijh7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717170000,"log":"[2026-02-17T13:10:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2836487511353176000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q11pqwfe6w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717171000,"log":"[2026-02-17T15:12:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3172","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nb95azylgj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717172000,"log":"[2026-02-17T15:01:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ijupyprufc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717173000,"log":"[2026-02-17T13:28:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3477481317977371000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f6aibzt4ow","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717174000,"log":"[2026-02-17T14:16:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 956 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v14rn7zpal","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717175000,"log":"[2026-02-17T14:10:07Z DEBUG sqlx::query] SELECT FROM streams returned 706 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-71sqdswbak","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717176000,"log":"[2026-02-17T14:07:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 874 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w27xsqiizk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717177000,"log":"[2026-02-17T15:11:55Z DEBUG metrics::reporter] recorded metric active_connections value=966","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-82sqnw542s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717178000,"log":"[2026-02-17T14:28:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 626 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w437gqmicv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717179000,"log":"[2026-02-17T12:48:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ojxfnn2c2f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717180000,"log":"[2026-02-17T14:51:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-45ulv2qdrd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717181000,"log":"[2026-02-17T13:21:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4209632379465686500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uthvurfpfp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717182000,"log":"[2026-02-17T12:54:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nqfyskvjzf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717183000,"log":"[2026-02-17T14:30:02Z DEBUG sqlx::query] SELECT FROM streams returned 40 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q94inv96oz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717184000,"log":"[2026-02-17T15:15:52Z TRACE cache::redis] cache delete key=user:session:6055","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f18v67c59s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717185000,"log":"[2026-02-17T15:04:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fsvpqkhfzv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717186000,"log":"[2026-02-17T13:36:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9273444158584627000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1ggiljs79n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717187000,"log":"[2026-02-17T12:41:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5mqlxz4jwz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717188000,"log":"[2026-02-17T12:48:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s4rxbud1eo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717189000,"log":"[2026-02-17T15:21:49Z TRACE cache::redis] cache miss key=stream:schema:2942","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1jaqqtvm8p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717190000,"log":"[2026-02-17T14:51:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vw4jqt15lo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717191000,"log":"[2026-02-17T12:57:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6s3imuin6a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717192000,"log":"[2026-02-17T13:41:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7162017732280370000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xugp3sgv3c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717193000,"log":"[2026-02-17T13:08:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9356466226317883000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7wk4leqrci","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717194000,"log":"[2026-02-17T15:05:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pjp84yplyi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717195000,"log":"[2026-02-17T13:39:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6100095780070122000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qw3fut48vn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717196000,"log":"[2026-02-17T14:30:58Z DEBUG sqlx::query] INSERT FROM users returned 473 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eatkpukhmr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717197000,"log":"[2026-02-17T14:00:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 235ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t2j0ia75fr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717198000,"log":"[2026-02-17T12:56:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s6v8pznznv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717199000,"log":"[2026-02-17T15:06:25Z DEBUG metrics::reporter] recorded metric active_connections value=4315","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uej88uc4b2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717200000,"log":"[2026-02-17T13:49:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 20ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yjyajmgvu0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717201000,"log":"[2026-02-17T13:34:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9386600342067182000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-blsgveqkmr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717202000,"log":"[2026-02-17T14:26:55Z DEBUG sqlx::query] DELETE FROM alerts returned 846 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3w72yyaf7d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717203000,"log":"[2026-02-17T13:23:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2829444913808090000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zcavy4amvo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717204000,"log":"[2026-02-17T13:11:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3622760676533518300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-melimhqqoz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717205000,"log":"[2026-02-17T13:01:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6lkigiy55e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717206000,"log":"[2026-02-17T12:36:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m3uz6c73m5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717207000,"log":"[2026-02-17T14:01:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 89ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ne4101fimv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717208000,"log":"[2026-02-17T14:39:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=8h9xxiaa","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nl02c5mpig","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717209000,"log":"[2026-02-17T15:15:35Z TRACE cache::redis] cache expire key=query:result:1686","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jutghdis76","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717210000,"log":"[2026-02-17T15:05:50Z DEBUG metrics::reporter] recorded metric active_connections value=1272","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f9q4h0h7hx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717211000,"log":"[2026-02-17T12:51:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o7g2gzzko0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717212000,"log":"[2026-02-17T15:11:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=2319","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x8d0kn6476","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717213000,"log":"[2026-02-17T13:43:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 73ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b9e37vk5ok","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717214000,"log":"[2026-02-17T15:17:38Z TRACE cache::redis] cache get key=org:config:7915","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0woohbbdgw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717215000,"log":"[2026-02-17T15:06:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=2214","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8uri4um7li","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717216000,"log":"[2026-02-17T14:14:55Z DEBUG sqlx::query] DELETE FROM alerts returned 170 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c5d6qjthgx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717217000,"log":"[2026-02-17T15:01:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-76s93xvong","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717218000,"log":"[2026-02-17T14:49:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dtseo7umek","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717219000,"log":"[2026-02-17T13:51:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 464ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b9e9nj4w0n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717220000,"log":"[2026-02-17T13:05:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kyu76mw5sg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717221000,"log":"[2026-02-17T13:49:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 448ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eu0gnu5qrx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717222000,"log":"[2026-02-17T13:05:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q3ydkftuyo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717223000,"log":"[2026-02-17T12:54:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f99h9rrwrf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717224000,"log":"[2026-02-17T14:42:54Z INFO tonic::transport::server] grpc router request_received client_id=87rdfug5","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q1n0gn45pw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717225000,"log":"[2026-02-17T13:24:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6602347433649682000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-arx4w8knmd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717226000,"log":"[2026-02-17T14:14:40Z DEBUG sqlx::query] DELETE FROM alerts returned 316 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hf2h0hu6jk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717227000,"log":"[2026-02-17T13:16:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8339563926158145000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j8bruc6ot7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717228000,"log":"[2026-02-17T12:46:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f2h5z34tj2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717229000,"log":"[2026-02-17T12:44:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ha491e8po","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717230000,"log":"[2026-02-17T14:01:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 8ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-66psq7111r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717231000,"log":"[2026-02-17T12:50:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6m6en4dmxs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717232000,"log":"[2026-02-17T14:02:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 81ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fwnog2ix1t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717233000,"log":"[2026-02-17T14:56:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-efsz1e95ru","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717234000,"log":"[2026-02-17T14:28:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 108 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oz853lun39","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717235000,"log":"[2026-02-17T12:44:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1cj5sk3kl6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717236000,"log":"[2026-02-17T13:43:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 284ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sh511qa3d5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717237000,"log":"[2026-02-17T13:32:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3554577795858000400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k02482nt5m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717238000,"log":"[2026-02-17T14:57:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l2ssq4d9c7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717239000,"log":"[2026-02-17T13:54:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 266ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-og3jywarud","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717240000,"log":"[2026-02-17T15:04:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pmg91pxuut","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717241000,"log":"[2026-02-17T12:35:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bzkpbo62ci","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717242000,"log":"[2026-02-17T14:11:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 883 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-162o0a55gv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717243000,"log":"[2026-02-17T12:56:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-we8f3ekou9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717244000,"log":"[2026-02-17T13:03:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bsxtlos0wo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717245000,"log":"[2026-02-17T12:46:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m3bba1qun6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717246000,"log":"[2026-02-17T12:53:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xex3z2u2df","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717247000,"log":"[2026-02-17T15:14:23Z TRACE cache::redis] cache expire key=org:config:6346","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4mjjms86in","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717248000,"log":"[2026-02-17T12:37:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nxavckgndh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717249000,"log":"[2026-02-17T12:36:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fgjl1v4119","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717250000,"log":"[2026-02-17T13:11:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/651356811145422600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vp8f0yxqpi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717251000,"log":"[2026-02-17T14:48:17Z INFO tonic::transport::server] grpc ingester connected client_id=srxddgt6","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rkfkzrpicu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717252000,"log":"[2026-02-17T14:20:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 401 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3x8q4hzrm2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717253000,"log":"[2026-02-17T14:40:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=vdo8919l","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m3vx5j6v3a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717254000,"log":"[2026-02-17T13:06:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zwfzz8fhvn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717255000,"log":"[2026-02-17T13:38:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3691484725173930000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-16y448djcz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717256000,"log":"[2026-02-17T13:58:25Z INFO actix_web::middleware::logger] GET /api/_search 400 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mjb44r5ove","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717257000,"log":"[2026-02-17T14:11:37Z DEBUG sqlx::query] SELECT FROM streams returned 636 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wvxeew456c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717258000,"log":"[2026-02-17T12:50:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vd6gfkboiy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717259000,"log":"[2026-02-17T14:50:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9b85vzjts5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717260000,"log":"[2026-02-17T14:50:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yla1m33lqj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717261000,"log":"[2026-02-17T14:35:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=e9kptmhm","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pdtw9tyzqr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717262000,"log":"[2026-02-17T14:13:25Z DEBUG sqlx::query] DELETE FROM alerts returned 868 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e78stps5uy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717263000,"log":"[2026-02-17T14:03:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 440ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-64kkyqjwp6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717264000,"log":"[2026-02-17T13:01:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h40bz6cr3g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717265000,"log":"[2026-02-17T14:27:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 66 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-47e8kehij6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717266000,"log":"[2026-02-17T14:44:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=hlh3m9c6","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1v9b2zeevg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717267000,"log":"[2026-02-17T14:04:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 88ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-689ln14n6i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717268000,"log":"[2026-02-17T12:40:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-174rw8qak7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717269000,"log":"[2026-02-17T13:35:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9067071549939174000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ekwhmkdeta","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717270000,"log":"[2026-02-17T14:14:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 892 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iwa2tuql4u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717271000,"log":"[2026-02-17T15:07:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=5394","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-donmcz348r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717272000,"log":"[2026-02-17T13:42:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 496ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d6ew3x6r4t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717273000,"log":"[2026-02-17T14:57:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-utew3yx69l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717274000,"log":"[2026-02-17T14:17:30Z DEBUG sqlx::query] DELETE FROM alerts returned 917 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-728pgo8t96","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717275000,"log":"[2026-02-17T13:24:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8198840026413206000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6qcnjc0nqk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717276000,"log":"[2026-02-17T14:50:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-41zoqbzie4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717277000,"log":"[2026-02-17T13:44:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 313ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f8sh003fbv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717278000,"log":"[2026-02-17T14:14:10Z DEBUG sqlx::query] DELETE FROM alerts returned 154 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6pi6jc2llw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717279000,"log":"[2026-02-17T13:38:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6078707390616418000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wxvsu68s53","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717280000,"log":"[2026-02-17T13:54:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 350ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n5vobjm45z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717281000,"log":"[2026-02-17T13:12:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7290848194301407000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w9yvbkeph1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717282000,"log":"[2026-02-17T13:45:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 333ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g6e24jl3ct","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717283000,"log":"[2026-02-17T13:20:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2567172575669769000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-egri751bqq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717284000,"log":"[2026-02-17T13:52:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 402ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h9cmmd8fge","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717285000,"log":"[2026-02-17T14:07:22Z DEBUG sqlx::query] SELECT FROM streams returned 34 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n9i06i1b6y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717286000,"log":"[2026-02-17T13:30:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/223448939298016580.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hi61gogp2z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717287000,"log":"[2026-02-17T13:00:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7un68gaky7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717288000,"log":"[2026-02-17T13:40:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/459001129462897100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e59wik698b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717289000,"log":"[2026-02-17T14:48:13Z INFO tonic::transport::server] grpc querier disconnected client_id=w3rf9v8h","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9vsa0an3m9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717290000,"log":"[2026-02-17T14:16:07Z DEBUG sqlx::query] SELECT FROM streams returned 925 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2u2lo0tar7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717291000,"log":"[2026-02-17T12:55:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b3son21kt9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717292000,"log":"[2026-02-17T15:14:47Z TRACE cache::redis] cache expire key=user:session:7858","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lim69twx49","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717293000,"log":"[2026-02-17T15:15:48Z TRACE cache::redis] cache hit key=org:config:5469","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4jgoxcotpv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717294000,"log":"[2026-02-17T12:49:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ttthgrt9x2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717295000,"log":"[2026-02-17T13:21:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5373446900348451000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lzuwrbi8d6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717296000,"log":"[2026-02-17T13:05:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2uijm5bnef","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717297000,"log":"[2026-02-17T15:17:08Z TRACE cache::redis] cache get key=org:config:6221","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r496asnp5s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717298000,"log":"[2026-02-17T14:55:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r1wb04jlez","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717299000,"log":"[2026-02-17T14:01:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 63ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-scnetlkowi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717300000,"log":"[2026-02-17T13:27:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9433801011503403000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9ggx2diqrn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717301000,"log":"[2026-02-17T13:06:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5gtx8u15lu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717302000,"log":"[2026-02-17T15:09:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1249","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vgvzzx5cai","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717303000,"log":"[2026-02-17T14:00:25Z INFO actix_web::middleware::logger] GET /api/_search 404 106ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ugoo93w7q8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717304000,"log":"[2026-02-17T14:49:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0fhfx5dod5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717305000,"log":"[2026-02-17T13:10:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9477848370315071000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g3oakwl5fm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717306000,"log":"[2026-02-17T13:21:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/830652568792820100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x3mbzklz13","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717307000,"log":"[2026-02-17T15:11:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=830","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2jdlihflc4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717308000,"log":"[2026-02-17T12:43:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2x3y6qpxjx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717309000,"log":"[2026-02-17T14:04:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 276ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oomt5ayr8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717310000,"log":"[2026-02-17T14:46:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=df2pbs0f","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bgqn3vir5t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717311000,"log":"[2026-02-17T13:30:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6468102689607540000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q7g0jgjjzc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717312000,"log":"[2026-02-17T13:35:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4143045242771963400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rnc4dsuyvw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717313000,"log":"[2026-02-17T14:43:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=dv5clo6z","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7m5kq0tvxh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717314000,"log":"[2026-02-17T13:17:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/349969807422652000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iweqx3vyjd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717315000,"log":"[2026-02-17T14:05:25Z INFO actix_web::middleware::logger] GET /api/_search 200 209ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xfevhlei8y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717316000,"log":"[2026-02-17T14:49:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vo0xas40me","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717317000,"log":"[2026-02-17T15:08:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4322","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ayjb82tutn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717318000,"log":"[2026-02-17T14:44:49Z INFO tonic::transport::server] grpc router request_received client_id=mq9w14mx","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qdk0hlx7ij","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717319000,"log":"[2026-02-17T14:37:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=a9wsmq39","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o5kr6za66g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717320000,"log":"[2026-02-17T14:54:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8qs178zuos","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717321000,"log":"[2026-02-17T13:22:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5279471647327371000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tydtgjpkkp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717322000,"log":"[2026-02-17T14:13:30Z DEBUG sqlx::query] DELETE FROM alerts returned 363 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-crxvm5jgrp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717323000,"log":"[2026-02-17T14:58:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cwt55bfp2y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717324000,"log":"[2026-02-17T13:17:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6897937178544167000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e7nygmjk47","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717325000,"log":"[2026-02-17T13:10:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5859900741068320000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0ftmvk5u7d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717326000,"log":"[2026-02-17T14:48:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=7v5696sv","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ruixk4afjr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717327000,"log":"[2026-02-17T13:02:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iy455zcqvq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717328000,"log":"[2026-02-17T14:57:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t8cs97ahlm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717329000,"log":"[2026-02-17T14:24:38Z DEBUG sqlx::query] INSERT FROM users returned 392 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mwgq35c7wx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717330000,"log":"[2026-02-17T15:02:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jv7fkmoln0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717331000,"log":"[2026-02-17T14:07:25Z DEBUG sqlx::query] DELETE FROM alerts returned 896 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q5b9ztb3oa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717332000,"log":"[2026-02-17T14:32:22Z INFO tonic::transport::server] grpc ingester connected client_id=sxrd0eho","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-00qqh7w2h5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717333000,"log":"[2026-02-17T14:33:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=8tuv4b17","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ho503qighz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717334000,"log":"[2026-02-17T13:21:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1413571489457976600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ez817bnx0q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717335000,"log":"[2026-02-17T12:48:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-icxmjoif2f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717336000,"log":"[2026-02-17T13:12:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/831168387798972800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u4niswruoh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717337000,"log":"[2026-02-17T14:42:59Z INFO tonic::transport::server] grpc router request_received client_id=8qhclj46","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9nwhg0uu8a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717338000,"log":"[2026-02-17T14:28:37Z DEBUG sqlx::query] SELECT FROM streams returned 594 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uxh3z1k6w9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717339000,"log":"[2026-02-17T12:59:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mfpya1h30h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717340000,"log":"[2026-02-17T13:18:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2621375075292769000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d8ocs0ezzq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717341000,"log":"[2026-02-17T12:42:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-brs3aqp128","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717342000,"log":"[2026-02-17T12:41:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ib0tqul5bt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717343000,"log":"[2026-02-17T14:04:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 346ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4j192b7wcy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717344000,"log":"[2026-02-17T14:14:13Z DEBUG sqlx::query] INSERT FROM users returned 932 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xi2hmfivsn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717345000,"log":"[2026-02-17T14:22:47Z DEBUG sqlx::query] SELECT FROM streams returned 866 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e6s0a4cocl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717346000,"log":"[2026-02-17T15:10:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1467","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xq8gqu92g4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717347000,"log":"[2026-02-17T13:53:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 287ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ckp4qhp37x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717348000,"log":"[2026-02-17T14:49:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9p6o77l3wx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717349000,"log":"[2026-02-17T14:53:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ivlzuwsw1u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717350000,"log":"[2026-02-17T13:30:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3026605633913100000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9c1gvzsw1h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717351000,"log":"[2026-02-17T13:22:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9165567292875614000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fd9gu8ejiz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717352000,"log":"[2026-02-17T12:50:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tgbttjyryv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717353000,"log":"[2026-02-17T14:14:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 676 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-musig5btyn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717354000,"log":"[2026-02-17T14:17:48Z DEBUG sqlx::query] INSERT FROM users returned 822 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-frm6b1eteg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717355000,"log":"[2026-02-17T14:13:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 945 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ubbq4gpfop","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717356000,"log":"[2026-02-17T14:18:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 584 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ul93mx54oz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717357000,"log":"[2026-02-17T13:11:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4630977024641449000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3xh9sbexr8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717358000,"log":"[2026-02-17T12:49:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ll8w7ht8gg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717359000,"log":"[2026-02-17T12:55:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vxw448fan8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717360000,"log":"[2026-02-17T14:50:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fv2nxb90ku","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717361000,"log":"[2026-02-17T14:05:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 47ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-752etz2py7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717362000,"log":"[2026-02-17T13:49:20Z INFO actix_web::middleware::logger] POST /api/_search 500 267ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zm8dhx25u2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717363000,"log":"[2026-02-17T14:52:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g0x1pwfpgx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717364000,"log":"[2026-02-17T12:44:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2mu49s3a11","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717365000,"log":"[2026-02-17T15:13:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=780","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-verpucfkfs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717366000,"log":"[2026-02-17T14:41:02Z INFO tonic::transport::server] grpc ingester connected client_id=at6p3b7d","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vit6zehy3z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717367000,"log":"[2026-02-17T13:11:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5744318765507297000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dmwquynpp3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717368000,"log":"[2026-02-17T12:39:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hu4p1ovkb1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717369000,"log":"[2026-02-17T14:56:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s93cqgkl06","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717370000,"log":"[2026-02-17T15:08:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=696","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f2es87knpg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717371000,"log":"[2026-02-17T13:05:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i7a5j8j0zd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717372000,"log":"[2026-02-17T13:16:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2508212465514830000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-totgqse1k8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717373000,"log":"[2026-02-17T14:22:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 135 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-go48d0ihtw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717374000,"log":"[2026-02-17T13:50:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 9ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xmde47n4y9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717375000,"log":"[2026-02-17T14:49:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7abyrtn7ni","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717376000,"log":"[2026-02-17T14:35:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=d1t4bo92","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-juopf4a0ly","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717377000,"log":"[2026-02-17T13:36:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3389092414539498500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v0xrtzu3i1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717378000,"log":"[2026-02-17T13:22:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8028016380845438000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-41gix9gpsa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717379000,"log":"[2026-02-17T13:33:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/637094895337904100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-223wlojg70","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717380000,"log":"[2026-02-17T15:00:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a44myuymks","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717381000,"log":"[2026-02-17T15:06:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=7481","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eo47uduw56","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717382000,"log":"[2026-02-17T14:23:15Z DEBUG sqlx::query] DELETE FROM alerts returned 906 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-19ajqqlmmw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717383000,"log":"[2026-02-17T13:27:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5066423147644414000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l5vk7z1av6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717384000,"log":"[2026-02-17T13:30:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2953193422700479500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ih9bmxh29v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717385000,"log":"[2026-02-17T13:02:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-np4ospu4rz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717386000,"log":"[2026-02-17T12:58:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hqzz7ot7en","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717387000,"log":"[2026-02-17T14:48:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ye4yidk5","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mkm0cdomne","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717388000,"log":"[2026-02-17T14:27:10Z DEBUG sqlx::query] DELETE FROM alerts returned 700 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3cggfabi83","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717389000,"log":"[2026-02-17T12:53:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-heudoew7rc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717390000,"log":"[2026-02-17T14:52:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-on5yzc2izb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717391000,"log":"[2026-02-17T13:39:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2951896031584742000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-92apnoqyiq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717392000,"log":"[2026-02-17T14:42:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=82nispjq","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hl9uppuhfo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717393000,"log":"[2026-02-17T13:25:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/604821553077412600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hm11byrx2u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717394000,"log":"[2026-02-17T14:12:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 326 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yekgilgje2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717395000,"log":"[2026-02-17T13:30:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5640864127215306000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-03dsqvpmi0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717396000,"log":"[2026-02-17T14:11:08Z DEBUG sqlx::query] INSERT FROM users returned 652 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vt2lhbwomr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717397000,"log":"[2026-02-17T12:50:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b8og4ht56x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717398000,"log":"[2026-02-17T13:26:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3407758263060865000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yhrg7hepwg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717399000,"log":"[2026-02-17T14:47:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=hpbklzbz","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d5ed9naxjg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717400000,"log":"[2026-02-17T15:14:56Z TRACE cache::redis] cache get key=auth:token:92","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9hhpmb4et3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717401000,"log":"[2026-02-17T12:39:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fmwyntuyd0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717402000,"log":"[2026-02-17T15:02:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ig00gwlmhm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717403000,"log":"[2026-02-17T14:47:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ckr3lso5","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-khxeqdacw7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717404000,"log":"[2026-02-17T13:55:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 352ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3624u2z7u4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717405000,"log":"[2026-02-17T15:09:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=591","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-btq6peswo9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717406000,"log":"[2026-02-17T14:09:45Z DEBUG sqlx::query] DELETE FROM alerts returned 859 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rj2j0wqlr4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717407000,"log":"[2026-02-17T13:49:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 331ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vi4pnd8ib","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717408000,"log":"[2026-02-17T12:55:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k3irsem0f7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717409000,"log":"[2026-02-17T14:40:28Z INFO tonic::transport::server] grpc querier disconnected client_id=gqu7254q","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3vjs9wxpgz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717410000,"log":"[2026-02-17T13:50:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 258ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-roslkvv0k9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717411000,"log":"[2026-02-17T15:07:40Z DEBUG metrics::reporter] recorded metric active_connections value=9193","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-02xbjge6tp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717412000,"log":"[2026-02-17T14:33:34Z INFO tonic::transport::server] grpc router request_received client_id=9bs5fvc7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bpro66op1k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717413000,"log":"[2026-02-17T14:31:45Z DEBUG sqlx::query] DELETE FROM alerts returned 814 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1y0zmxy2qs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717414000,"log":"[2026-02-17T12:47:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-voax7fulwj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717415000,"log":"[2026-02-17T15:06:05Z DEBUG metrics::reporter] recorded metric active_connections value=879","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-03y17qrmtf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717416000,"log":"[2026-02-17T14:53:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bkbb9kirkj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717417000,"log":"[2026-02-17T15:06:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6094","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tanp63jgib","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717418000,"log":"[2026-02-17T14:15:53Z DEBUG sqlx::query] INSERT FROM users returned 229 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pexufx4h9c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717419000,"log":"[2026-02-17T15:11:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=2553","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ty5f6oh59j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717420000,"log":"[2026-02-17T13:46:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 467ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o4qa6mxchz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717421000,"log":"[2026-02-17T14:58:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lyvdi5qn6y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717422000,"log":"[2026-02-17T13:08:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5237920069027269000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5ihahorxz9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717423000,"log":"[2026-02-17T12:40:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4dm345xbgb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717424000,"log":"[2026-02-17T14:12:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 486 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-icw975totv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717425000,"log":"[2026-02-17T13:28:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5761904221228920000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7cndhluyuj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717426000,"log":"[2026-02-17T13:30:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6577866102505654000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fpo3o64yh9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717427000,"log":"[2026-02-17T14:34:09Z INFO tonic::transport::server] grpc router request_received client_id=5aj08pu2","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tjvsd3gz5k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717428000,"log":"[2026-02-17T14:30:47Z DEBUG sqlx::query] SELECT FROM streams returned 853 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ucd9lj0b2k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717429000,"log":"[2026-02-17T12:39:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mo8ov0odzl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717430000,"log":"[2026-02-17T15:21:12Z TRACE cache::redis] cache hit key=user:session:9165","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3zwzzyer8v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717431000,"log":"[2026-02-17T14:53:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qjdwcf4rro","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717432000,"log":"[2026-02-17T14:10:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 518 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-63ce91wbab","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717433000,"log":"[2026-02-17T14:19:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 903 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-md2s2kwnna","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717434000,"log":"[2026-02-17T12:48:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gdeqw7dmsw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717435000,"log":"[2026-02-17T14:14:00Z DEBUG sqlx::query] DELETE FROM alerts returned 361 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dolmpa1t7a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717436000,"log":"[2026-02-17T13:37:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6150442400595766000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pnzhatawyy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717437000,"log":"[2026-02-17T13:07:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7grb1qr3d9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717438000,"log":"[2026-02-17T14:28:57Z DEBUG sqlx::query] SELECT FROM streams returned 739 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ar5496lp8y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717439000,"log":"[2026-02-17T12:47:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2v86lxs08i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717440000,"log":"[2026-02-17T12:42:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zohvcwwd2b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717441000,"log":"[2026-02-17T12:57:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-75q3yuorpu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717442000,"log":"[2026-02-17T13:00:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xlt7fr7o8x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717443000,"log":"[2026-02-17T15:01:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-evqcrvoxe5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717444000,"log":"[2026-02-17T14:37:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=cwzuvdy5","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e5nm3nq6wn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717445000,"log":"[2026-02-17T14:30:17Z DEBUG sqlx::query] SELECT FROM streams returned 278 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bh5e86nq17","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717446000,"log":"[2026-02-17T13:28:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3189424310255786000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2che73aib5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717447000,"log":"[2026-02-17T14:55:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4bvt4e7bfq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717448000,"log":"[2026-02-17T14:38:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=irz1ymak","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z8jxhwwgwb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717449000,"log":"[2026-02-17T14:00:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 49ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ntmb87hew4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717450000,"log":"[2026-02-17T15:13:56Z TRACE cache::redis] cache get key=auth:token:2145","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-krbtu3trfj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717451000,"log":"[2026-02-17T14:50:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4prc5h3mtv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717452000,"log":"[2026-02-17T15:05:35Z DEBUG metrics::reporter] recorded metric active_connections value=6713","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n79upbrmwn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717453000,"log":"[2026-02-17T14:30:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 513 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-emxqsl0djm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717454000,"log":"[2026-02-17T14:06:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 188ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z0j7wdszia","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717455000,"log":"[2026-02-17T14:47:13Z INFO tonic::transport::server] grpc querier disconnected client_id=ezhpi3j1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xxet4swy5x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717456000,"log":"[2026-02-17T14:45:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=7jrknl8l","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8nawygz8h8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717457000,"log":"[2026-02-17T14:56:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9sd5uvxa7k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717458000,"log":"[2026-02-17T14:32:18Z INFO tonic::transport::server] grpc querier disconnected client_id=95g547sa","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2rcft0b4pj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717459000,"log":"[2026-02-17T14:36:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=2z1jzpgk","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4lm6tljup6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717460000,"log":"[2026-02-17T13:30:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2902148787528364500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dudp4mp7q8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717461000,"log":"[2026-02-17T13:16:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6932272213785025000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kdhwg9v7qi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717462000,"log":"[2026-02-17T13:59:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 179ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5xrh0wagvi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717463000,"log":"[2026-02-17T14:44:57Z INFO tonic::transport::server] grpc ingester connected client_id=62o6icxu","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4axrx6xx8e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717464000,"log":"[2026-02-17T13:01:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w29wb4kdok","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717465000,"log":"[2026-02-17T12:57:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p4eqfymd95","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717466000,"log":"[2026-02-17T13:40:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/187678826668251650.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cv8nfyymmy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717467000,"log":"[2026-02-17T15:04:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tnk7rtdf4f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717468000,"log":"[2026-02-17T14:05:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rmwypd2nuz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717469000,"log":"[2026-02-17T13:31:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4163416328020699000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j9ez3dck56","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717470000,"log":"[2026-02-17T14:30:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 659 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lr8rn9mlfr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717471000,"log":"[2026-02-17T13:36:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2162840708935673900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2guihoom4l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717472000,"log":"[2026-02-17T12:50:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lcnzb4ymzy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717473000,"log":"[2026-02-17T12:41:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-389p4up4h9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717474000,"log":"[2026-02-17T14:19:58Z DEBUG sqlx::query] INSERT FROM users returned 466 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-isx2ke8954","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717475000,"log":"[2026-02-17T13:49:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 65ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9i5gdwg7nu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717476000,"log":"[2026-02-17T15:12:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1852","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fqcq67ih0f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717477000,"log":"[2026-02-17T14:16:08Z DEBUG sqlx::query] INSERT FROM users returned 786 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n7kb3qvveg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717478000,"log":"[2026-02-17T13:57:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 187ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xy091pbe8w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717479000,"log":"[2026-02-17T12:59:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hhiokvjsme","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717480000,"log":"[2026-02-17T14:22:17Z DEBUG sqlx::query] SELECT FROM streams returned 996 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jw9y1jmpnw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717481000,"log":"[2026-02-17T14:48:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a6u66rz3i4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717482000,"log":"[2026-02-17T13:04:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-acd1wk503b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717483000,"log":"[2026-02-17T13:22:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3119123344189867000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d4y412ustv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717484000,"log":"[2026-02-17T13:25:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7576795018984391000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z0xsfa4qwh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717485000,"log":"[2026-02-17T15:08:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1564","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qqgy4p2nk5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717486000,"log":"[2026-02-17T12:42:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1j27z7o16v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717487000,"log":"[2026-02-17T12:39:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c1o6c80ipv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717488000,"log":"[2026-02-17T13:31:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8816255342114785000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hq6nucvphu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717489000,"log":"[2026-02-17T14:10:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 203 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h73cidl2rz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717490000,"log":"[2026-02-17T14:41:33Z INFO tonic::transport::server] grpc querier disconnected client_id=dxzskt14","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-477xrcqqkh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717491000,"log":"[2026-02-17T14:31:33Z DEBUG sqlx::query] INSERT FROM users returned 181 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-04aupaytlt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717492000,"log":"[2026-02-17T13:09:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2649259253412224000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-96jmo0h517","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717493000,"log":"[2026-02-17T12:50:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a04jnigcw3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717494000,"log":"[2026-02-17T14:56:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1r7gh0mfvr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717495000,"log":"[2026-02-17T14:03:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 309ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m3cjyjtk4s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717496000,"log":"[2026-02-17T13:23:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9503743911954463000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-70bjl3nae7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717497000,"log":"[2026-02-17T14:48:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=di170cq4","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aj603ji723","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717498000,"log":"[2026-02-17T14:32:59Z INFO tonic::transport::server] grpc router request_received client_id=e3qkzole","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r99w8nri0c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717499000,"log":"[2026-02-17T13:42:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 490ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-roj87u66dx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717500000,"log":"[2026-02-17T15:09:10Z DEBUG metrics::reporter] recorded metric active_connections value=6504","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sjqd0j1hdz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717501000,"log":"[2026-02-17T14:09:15Z DEBUG sqlx::query] DELETE FROM alerts returned 951 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zwk66xodew","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717502000,"log":"[2026-02-17T12:50:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jqoeb5f0t3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717503000,"log":"[2026-02-17T12:38:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ez0xfxzhd8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717504000,"log":"[2026-02-17T12:48:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l9ifr2veie","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717505000,"log":"[2026-02-17T13:17:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1006699389429344100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2oulwpa2b4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717506000,"log":"[2026-02-17T14:09:28Z DEBUG sqlx::query] INSERT FROM users returned 570 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0nwk5hxoyy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717507000,"log":"[2026-02-17T13:27:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5112555174876908000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yttwl4dge2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717508000,"log":"[2026-02-17T13:01:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jcxf7d2x59","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717509000,"log":"[2026-02-17T13:30:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3283716087014268000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o4184z8vvt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717510000,"log":"[2026-02-17T13:11:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5008644404832443000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xcvn64390p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717511000,"log":"[2026-02-17T13:17:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3405214305057242600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2or7dbx3dq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717512000,"log":"[2026-02-17T14:55:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-htjqle8o48","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717513000,"log":"[2026-02-17T14:38:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=khpfcmiq","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xsb654zu9u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717514000,"log":"[2026-02-17T13:43:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b31e9en96g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717515000,"log":"[2026-02-17T13:05:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-paertyq9id","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717516000,"log":"[2026-02-17T14:24:35Z DEBUG sqlx::query] DELETE FROM alerts returned 611 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7557y8k4ub","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717517000,"log":"[2026-02-17T13:52:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 257ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2v1nh0jyst","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717518000,"log":"[2026-02-17T15:04:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1how98bevj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717519000,"log":"[2026-02-17T13:34:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/265835763408026620.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7eqbgt20s5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717520000,"log":"[2026-02-17T14:27:05Z DEBUG sqlx::query] DELETE FROM alerts returned 172 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ulq29k3owh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717521000,"log":"[2026-02-17T12:40:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zqure25grj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717522000,"log":"[2026-02-17T13:00:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fedibplofw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717523000,"log":"[2026-02-17T13:36:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1092440551863356200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4z1ep362lj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717524000,"log":"[2026-02-17T14:50:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6zn4lfojjt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717525000,"log":"[2026-02-17T14:25:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 447 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-odfyn76w8g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717526000,"log":"[2026-02-17T15:09:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1665","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uw9b76gmjs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717527000,"log":"[2026-02-17T13:44:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 240ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wnz26vxnat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717528000,"log":"[2026-02-17T15:17:36Z TRACE cache::redis] cache hit key=auth:token:1333","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-654hip9ytm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717529000,"log":"[2026-02-17T14:32:02Z INFO tonic::transport::server] grpc ingester connected client_id=fpyhhusg","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6pd2q4bc4b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717530000,"log":"[2026-02-17T12:37:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nk4wnc1kbv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717531000,"log":"[2026-02-17T13:06:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9hpm0rc3ij","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717532000,"log":"[2026-02-17T14:59:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mlbjn6gal4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717533000,"log":"[2026-02-17T14:15:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 461 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bv50astgzk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717534000,"log":"[2026-02-17T15:13:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=876","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vp2lj95b94","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717535000,"log":"[2026-02-17T14:21:52Z DEBUG sqlx::query] SELECT FROM streams returned 125 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j84biygg48","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717536000,"log":"[2026-02-17T15:02:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fk3m6gwxl9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717537000,"log":"[2026-02-17T13:24:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/444456730013835800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ysycc4j7jj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717538000,"log":"[2026-02-17T14:12:38Z DEBUG sqlx::query] INSERT FROM users returned 854 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rkaqce584u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717539000,"log":"[2026-02-17T13:15:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2173975143600060000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a6eb4jhifo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717540000,"log":"[2026-02-17T13:03:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-za40e4a0ee","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717541000,"log":"[2026-02-17T14:46:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=7w06t2me","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c9gza6yhfx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717542000,"log":"[2026-02-17T12:59:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bltfp8an1x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717543000,"log":"[2026-02-17T13:56:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 414ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wbl4xderl5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717544000,"log":"[2026-02-17T13:00:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-txb0swvn4p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717545000,"log":"[2026-02-17T14:40:27Z INFO tonic::transport::server] grpc ingester connected client_id=78h9ox0f","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3ipiiuo8zk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717546000,"log":"[2026-02-17T14:52:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4y3r68to7c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717547000,"log":"[2026-02-17T14:51:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-db5c759f1h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717548000,"log":"[2026-02-17T14:32:17Z INFO tonic::transport::server] grpc ingester connected client_id=4w6m191d","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mj40gqpji7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717549000,"log":"[2026-02-17T13:32:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/248000424981003040.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-22s3e7cgcl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717550000,"log":"[2026-02-17T12:59:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yppqz0op40","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717551000,"log":"[2026-02-17T13:08:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3070282328066254000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a73u81fn8y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717552000,"log":"[2026-02-17T13:47:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dbxixqcop1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717553000,"log":"[2026-02-17T13:18:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2865926530559917000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8cgwko497k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717554000,"log":"[2026-02-17T14:36:53Z INFO tonic::transport::server] grpc querier disconnected client_id=x5j5rbu5","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5r5e25sozs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717555000,"log":"[2026-02-17T15:01:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p6clgohwu8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717556000,"log":"[2026-02-17T13:16:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9489764788615901000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zgm57mzlfk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717557000,"log":"[2026-02-17T13:27:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6088547282264713000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3uhu20jbyj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717558000,"log":"[2026-02-17T15:17:44Z TRACE cache::redis] cache get key=stream:schema:6936","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nugiqzcb1m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717559000,"log":"[2026-02-17T13:43:05Z INFO actix_web::middleware::logger] GET /api/_search 201 421ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d2oh5hf9ws","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717560000,"log":"[2026-02-17T15:12:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9781","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x91gxsjzqm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717561000,"log":"[2026-02-17T15:19:50Z TRACE cache::redis] cache get key=query:result:2265","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lprasgw6py","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717562000,"log":"[2026-02-17T13:47:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 285ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-24eetra9r6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717563000,"log":"[2026-02-17T14:56:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wnsocivvwm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717564000,"log":"[2026-02-17T13:53:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wy1t3fc1xl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717565000,"log":"[2026-02-17T14:12:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 540 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gzle246wke","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717566000,"log":"[2026-02-17T14:13:12Z DEBUG sqlx::query] SELECT FROM streams returned 218 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7gvnv8glmd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717567000,"log":"[2026-02-17T15:10:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=766","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6o2l6nlc3y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717568000,"log":"[2026-02-17T14:41:29Z INFO tonic::transport::server] grpc router request_received client_id=ddrxxr7t","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c4tzwj1qq3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717569000,"log":"[2026-02-17T15:18:10Z TRACE cache::redis] cache delete key=query:result:3988","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eh0l69mmpl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717570000,"log":"[2026-02-17T12:52:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wco9sm9zej","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717571000,"log":"[2026-02-17T14:53:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0wirg55hz8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717572000,"log":"[2026-02-17T13:46:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 93ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n4cfw97bi9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717573000,"log":"[2026-02-17T13:52:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 143ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sypxydgh6v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717574000,"log":"[2026-02-17T14:43:18Z INFO tonic::transport::server] grpc querier disconnected client_id=chwl8q4o","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hky7qxfv0f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717575000,"log":"[2026-02-17T15:08:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=9725","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tf2doieq41","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717576000,"log":"[2026-02-17T15:19:29Z TRACE cache::redis] cache expire key=stream:schema:612","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dmt7y3micu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717577000,"log":"[2026-02-17T13:13:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3422570378408487000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9fznewef63","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717578000,"log":"[2026-02-17T15:21:28Z TRACE cache::redis] cache delete key=org:config:3752","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iuwvsye89","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717579000,"log":"[2026-02-17T14:06:05Z INFO actix_web::middleware::logger] GET /api/_search 401 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g2rbec5q9i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717580000,"log":"[2026-02-17T14:59:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zyislnf84r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717581000,"log":"[2026-02-17T13:14:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7791731840705758000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b8buukhow4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717582000,"log":"[2026-02-17T15:17:24Z TRACE cache::redis] cache hit key=stream:schema:8993","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fxj6jparzt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717583000,"log":"[2026-02-17T14:52:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m542177nxi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717584000,"log":"[2026-02-17T14:56:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nkh0bya053","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717585000,"log":"[2026-02-17T13:08:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/125417452765317340.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tsfls3rpsa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717586000,"log":"[2026-02-17T13:50:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 18ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8zduf7padm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717587000,"log":"[2026-02-17T14:20:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 935 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lypioldq1l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717588000,"log":"[2026-02-17T13:59:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 393ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-acgz9q8848","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717589000,"log":"[2026-02-17T14:36:42Z INFO tonic::transport::server] grpc ingester connected client_id=28npomgh","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wgy7122ueh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717590000,"log":"[2026-02-17T13:36:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5858092018404570000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9ws1s5wn62","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717591000,"log":"[2026-02-17T13:40:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6372314997325277000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s0bhehm8is","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717592000,"log":"[2026-02-17T13:09:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4647827252161263000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8ja4hagj75","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717593000,"log":"[2026-02-17T13:08:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-swxr1ilwom","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717594000,"log":"[2026-02-17T15:07:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=4595","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e6dqa1q6t4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717595000,"log":"[2026-02-17T13:20:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7542701912986973000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j33eri954f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717596000,"log":"[2026-02-17T14:40:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=31hysal1","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ph0d0l1jzb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717597000,"log":"[2026-02-17T14:52:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dadda3l9ra","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717598000,"log":"[2026-02-17T15:04:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fbe64xh6j0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717599000,"log":"[2026-02-17T14:00:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 183ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w8bmsv6o8e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717600000,"log":"[2026-02-17T13:17:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6627785484127187000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vvfn91yjg1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717601000,"log":"[2026-02-17T13:10:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8821104862800608000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ss1xt4c799","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717602000,"log":"[2026-02-17T14:22:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 779 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zt0gyl4i1p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717603000,"log":"[2026-02-17T13:52:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 166ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-paetw2dkr8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717604000,"log":"[2026-02-17T13:07:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-asq32wdeen","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717605000,"log":"[2026-02-17T13:27:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8764934492156382000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5oirsfj82m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717606000,"log":"[2026-02-17T14:11:02Z DEBUG sqlx::query] SELECT FROM streams returned 79 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nl8uegq6si","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717607000,"log":"[2026-02-17T15:19:01Z TRACE cache::redis] cache miss key=auth:token:2604","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t7mzvkbmpq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717608000,"log":"[2026-02-17T15:20:15Z TRACE cache::redis] cache set key=query:result:2357","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wc8f20whpx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717609000,"log":"[2026-02-17T14:09:42Z DEBUG sqlx::query] SELECT FROM streams returned 744 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cxq6klclp0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717610000,"log":"[2026-02-17T13:02:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3de6hhzigd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717611000,"log":"[2026-02-17T12:44:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-18q1p4i2hk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717612000,"log":"[2026-02-17T14:10:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 302 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9t5m2npiyc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717613000,"log":"[2026-02-17T13:40:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5380149072162635000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9t1850flw7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717614000,"log":"[2026-02-17T14:03:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 179ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h7c9cthbr6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717615000,"log":"[2026-02-17T15:10:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=5570","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ejgv3x0bu7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717616000,"log":"[2026-02-17T15:07:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=6650","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ded4a5df9v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717617000,"log":"[2026-02-17T14:17:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 634 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p2ladcdeg9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717618000,"log":"[2026-02-17T14:33:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=dy4gfo2h","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t4yx1aljm4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717619000,"log":"[2026-02-17T13:24:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/625888724385199600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gjkqpsyihv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717620000,"log":"[2026-02-17T13:21:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9342225769808130000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-75a7i33wz6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717621000,"log":"[2026-02-17T13:28:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8031802205252601000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d92dmkm638","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717622000,"log":"[2026-02-17T13:12:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2541458212157586000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w2vx31kxla","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717623000,"log":"[2026-02-17T13:10:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/530043602568677250.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kzhc3r0zbc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717624000,"log":"[2026-02-17T12:55:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c1ctnazd0t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717625000,"log":"[2026-02-17T14:35:47Z INFO tonic::transport::server] grpc ingester connected client_id=cg1cqbg8","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ze11hjogxv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717626000,"log":"[2026-02-17T14:19:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 277 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ce4x5jf56t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717627000,"log":"[2026-02-17T13:58:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 493ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gyysdtv5md","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717628000,"log":"[2026-02-17T15:04:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xgjau2idtg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717629000,"log":"[2026-02-17T14:20:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 884 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-84g3zov6pn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717630000,"log":"[2026-02-17T13:05:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7dcs6qf02m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717631000,"log":"[2026-02-17T13:27:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7475726042682891000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fv846yaqdi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717632000,"log":"[2026-02-17T12:48:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0itwzgf70o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717633000,"log":"[2026-02-17T14:37:52Z INFO tonic::transport::server] grpc ingester connected client_id=3mmjh77a","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-525tskkmoq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717634000,"log":"[2026-02-17T14:16:45Z DEBUG sqlx::query] DELETE FROM alerts returned 78 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a24ibw9665","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717635000,"log":"[2026-02-17T12:42:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eev8art9zg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717636000,"log":"[2026-02-17T14:01:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 160ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-epqso5vxey","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717637000,"log":"[2026-02-17T15:05:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2457","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lwr6a2ojoj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717638000,"log":"[2026-02-17T14:24:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 924 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-orjvbyxoe0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717639000,"log":"[2026-02-17T13:24:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/235106476858883970.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zha60rmz13","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717640000,"log":"[2026-02-17T13:02:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8j2pjvjnvy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717641000,"log":"[2026-02-17T13:11:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7648572408241636000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-brk91vtsmw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717642000,"log":"[2026-02-17T12:42:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-juweuudhhi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717643000,"log":"[2026-02-17T15:12:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7397","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qoe3v2nvgb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717644000,"log":"[2026-02-17T13:12:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/225961697395862240.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-284jy6fxzh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717645000,"log":"[2026-02-17T12:41:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9h4p13p78w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717646000,"log":"[2026-02-17T13:42:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 5ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i0rvsc848z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717647000,"log":"[2026-02-17T12:52:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xy0u2zvuwj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717648000,"log":"[2026-02-17T13:53:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 81ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8zsc6zkxmj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717649000,"log":"[2026-02-17T14:38:18Z INFO tonic::transport::server] grpc querier disconnected client_id=lgdgz6bg","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-589gvmai7b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717650000,"log":"[2026-02-17T14:45:33Z INFO tonic::transport::server] grpc querier disconnected client_id=nkgj6ckd","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3st3efqfcs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717651000,"log":"[2026-02-17T14:54:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vbcwkvrivf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717652000,"log":"[2026-02-17T15:03:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iioqi1kmkg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717653000,"log":"[2026-02-17T14:03:25Z INFO actix_web::middleware::logger] GET /api/_search 404 492ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-plkwzxt33i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717654000,"log":"[2026-02-17T14:48:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gkt88f7gm5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717655000,"log":"[2026-02-17T14:16:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 953 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hra6ab6la7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717656000,"log":"[2026-02-17T15:00:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1j2z6odrqz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717657000,"log":"[2026-02-17T12:48:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k61uwegmed","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717658000,"log":"[2026-02-17T14:52:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iqefsuj3um","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717659000,"log":"[2026-02-17T13:20:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9999395151296375000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t1nvfbf69p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717660000,"log":"[2026-02-17T12:48:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uinu48c83e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717661000,"log":"[2026-02-17T14:38:07Z INFO tonic::transport::server] grpc ingester connected client_id=nf6i4q1z","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mkyxcpgd8x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717662000,"log":"[2026-02-17T13:33:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6004466257351785000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5a4bzddjzj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717663000,"log":"[2026-02-17T14:04:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 424ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-joz815r21r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717664000,"log":"[2026-02-17T14:21:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 662 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wgnl6inixv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717665000,"log":"[2026-02-17T14:57:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t9hwbsoi7f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717666000,"log":"[2026-02-17T13:06:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ortv72vbsz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717667000,"log":"[2026-02-17T14:30:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 64 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-69v3mqqu85","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717668000,"log":"[2026-02-17T13:08:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1528062170181241300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7z4o5u2l0f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717669000,"log":"[2026-02-17T15:21:31Z TRACE cache::redis] cache miss key=auth:token:8437","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jd6w4ge4wl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717670000,"log":"[2026-02-17T14:11:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 803 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w0xij6urow","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717671000,"log":"[2026-02-17T12:54:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xdisaekz8r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717672000,"log":"[2026-02-17T13:43:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 194ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j9qdhkj2c6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717673000,"log":"[2026-02-17T13:46:25Z INFO actix_web::middleware::logger] GET /api/_search 400 257ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-37l8uhncnm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717674000,"log":"[2026-02-17T14:58:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9x8jhyezc2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717675000,"log":"[2026-02-17T13:38:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8725402964035646000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0ht5tvqjhp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717676000,"log":"[2026-02-17T14:46:32Z INFO tonic::transport::server] grpc ingester connected client_id=btqtdb1z","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-19cfw4afbf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717677000,"log":"[2026-02-17T13:52:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 221ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hlscww0uhz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717678000,"log":"[2026-02-17T14:04:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 238ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-joih53tg0q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717679000,"log":"[2026-02-17T13:52:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 404ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s2ycvgsqan","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717680000,"log":"[2026-02-17T15:03:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3zz7yske8m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717681000,"log":"[2026-02-17T14:58:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hhx0nl6xvh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717682000,"log":"[2026-02-17T14:41:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=0xb1pxql","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cgby5c6m5a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717683000,"log":"[2026-02-17T14:24:50Z DEBUG sqlx::query] DELETE FROM alerts returned 155 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ad5jd7f8b6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717684000,"log":"[2026-02-17T13:41:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3225012804899136500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5psibih4tc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717685000,"log":"[2026-02-17T12:56:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4e4cmvs3w3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717686000,"log":"[2026-02-17T15:13:41Z TRACE cache::redis] cache expire key=auth:token:6366","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-74043ot6gn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717687000,"log":"[2026-02-17T14:25:00Z DEBUG sqlx::query] DELETE FROM alerts returned 145 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rn010baffs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717688000,"log":"[2026-02-17T13:33:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3292339481717943000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-76mubezytz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717689000,"log":"[2026-02-17T13:57:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 76ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-th9tjdlkqy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717690000,"log":"[2026-02-17T15:05:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7031","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rbdzay0ty9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717691000,"log":"[2026-02-17T14:46:17Z INFO tonic::transport::server] grpc ingester connected client_id=sciruboh","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6xjaqyrez6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717692000,"log":"[2026-02-17T12:47:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eep280qkkb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717693000,"log":"[2026-02-17T12:52:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-keur7ipjy1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717694000,"log":"[2026-02-17T12:43:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-27tbjmba0m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717695000,"log":"[2026-02-17T12:57:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1z3m5xxsk8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717696000,"log":"[2026-02-17T13:11:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/740259032545129700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8f7i67nj5v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717697000,"log":"[2026-02-17T14:08:33Z DEBUG sqlx::query] INSERT FROM users returned 37 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tfwqad3iuj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717698000,"log":"[2026-02-17T15:11:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=1238","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yl7doguu7x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717699000,"log":"[2026-02-17T12:39:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5sddax5uok","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717700000,"log":"[2026-02-17T12:38:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dxzpigvwt6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717701000,"log":"[2026-02-17T14:23:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 275 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2t85taui2m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717702000,"log":"[2026-02-17T12:42:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v212j8udlv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717703000,"log":"[2026-02-17T14:39:22Z INFO tonic::transport::server] grpc ingester connected client_id=gfwubbdi","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wjzh2zm7vz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717704000,"log":"[2026-02-17T13:08:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mo4zqg1y94","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717705000,"log":"[2026-02-17T14:48:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8d3qtues7f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717706000,"log":"[2026-02-17T12:55:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jvhpn2cqa8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717707000,"log":"[2026-02-17T13:16:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9219085523402539000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-41adnhu158","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717708000,"log":"[2026-02-17T15:14:30Z TRACE cache::redis] cache hit key=query:result:9298","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-82ajyvqu3o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717709000,"log":"[2026-02-17T15:05:15Z DEBUG metrics::reporter] recorded metric active_connections value=1660","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-15f6qrf0zg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717710000,"log":"[2026-02-17T14:30:37Z DEBUG sqlx::query] SELECT FROM streams returned 479 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-77nc6qfwy2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717711000,"log":"[2026-02-17T15:05:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8900","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s3zq5paj71","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717712000,"log":"[2026-02-17T12:37:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7yc92nuez4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717713000,"log":"[2026-02-17T13:02:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-inx8q3lnok","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717714000,"log":"[2026-02-17T14:46:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=g1lsfc1j","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kfbqrlgxo8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717715000,"log":"[2026-02-17T12:53:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c0dqqf6y1b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717716000,"log":"[2026-02-17T12:47:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rfp6b9hsuk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717717000,"log":"[2026-02-17T14:33:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=pwlhza2l","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2fp24ncwgy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717718000,"log":"[2026-02-17T14:34:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=gqc4yb68","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zr8cslx0ux","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717719000,"log":"[2026-02-17T13:09:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7817431497416017000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-upef7b0oot","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717720000,"log":"[2026-02-17T14:04:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 238ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iye8gs4ojn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717721000,"log":"[2026-02-17T14:05:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 49ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qqvbshghk9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717722000,"log":"[2026-02-17T13:12:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7187935017673055000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-30j76d7fzq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717723000,"log":"[2026-02-17T13:38:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6533640046446572000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oonuzk3h9d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717724000,"log":"[2026-02-17T14:08:28Z DEBUG sqlx::query] INSERT FROM users returned 833 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d3tyndjbn8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717725000,"log":"[2026-02-17T14:26:15Z DEBUG sqlx::query] DELETE FROM alerts returned 460 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m9fwawzd8m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717726000,"log":"[2026-02-17T13:52:00Z INFO actix_web::middleware::logger] POST /api/_search 403 434ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-et2l47d14a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717727000,"log":"[2026-02-17T14:55:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5i0hibdmif","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717728000,"log":"[2026-02-17T14:32:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=dsu6wle3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pks6i5ou37","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717729000,"log":"[2026-02-17T13:39:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9864221184379804000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kdi2wg0lix","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717730000,"log":"[2026-02-17T14:52:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-edg5lx158f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717731000,"log":"[2026-02-17T12:45:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v5s3k1viej","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717732000,"log":"[2026-02-17T13:02:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8qhxo6egf2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717733000,"log":"[2026-02-17T15:01:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zavwwbi60h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717734000,"log":"[2026-02-17T14:03:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 327ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8oijm0b5l7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717735000,"log":"[2026-02-17T14:04:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 19ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qz00r67sne","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717736000,"log":"[2026-02-17T13:48:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 248ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5ky57yj7k5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717737000,"log":"[2026-02-17T14:44:37Z INFO tonic::transport::server] grpc ingester connected client_id=2x2h0p3r","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h4dm1suh5b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717738000,"log":"[2026-02-17T13:09:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7619573078778763000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dh4lp29h1y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717739000,"log":"[2026-02-17T13:33:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/516311033615803900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2spm8ffhpt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717740000,"log":"[2026-02-17T15:14:55Z TRACE cache::redis] cache miss key=query:result:6975","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-01m7p6ygd0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717741000,"log":"[2026-02-17T14:31:00Z DEBUG sqlx::query] DELETE FROM alerts returned 509 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uagodaoinc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717742000,"log":"[2026-02-17T14:25:35Z DEBUG sqlx::query] DELETE FROM alerts returned 945 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4j171j6kkt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717743000,"log":"[2026-02-17T13:21:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4516884869585829000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cs2ab3ht7w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717744000,"log":"[2026-02-17T14:35:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=i7w61wpv","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kqiptjhzt1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717745000,"log":"[2026-02-17T12:46:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g1d55j3n2i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717746000,"log":"[2026-02-17T13:21:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7018611585983983000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-inpcmuflh9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717747000,"log":"[2026-02-17T15:05:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6950","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1v4ezf0dt5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717748000,"log":"[2026-02-17T15:09:20Z DEBUG metrics::reporter] recorded metric active_connections value=6415","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7tz381jt3i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717749000,"log":"[2026-02-17T12:35:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zz8utf6xzr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717750000,"log":"[2026-02-17T14:23:30Z DEBUG sqlx::query] DELETE FROM alerts returned 724 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6r56pu5pjc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717751000,"log":"[2026-02-17T14:51:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2c7pbn2d48","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717752000,"log":"[2026-02-17T14:22:33Z DEBUG sqlx::query] INSERT FROM users returned 58 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5l2yf26vnd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717753000,"log":"[2026-02-17T12:50:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-skmxk220cb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717754000,"log":"[2026-02-17T13:59:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 412ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kd5o176k8n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717755000,"log":"[2026-02-17T12:51:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7n2t2ttkrw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717756000,"log":"[2026-02-17T12:46:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k4tcjnszub","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717757000,"log":"[2026-02-17T12:47:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d36a0l5nhz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717758000,"log":"[2026-02-17T13:39:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4320477615446476300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e0qphehzon","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717759000,"log":"[2026-02-17T13:41:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5913307996376253000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mcksitw9co","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717760000,"log":"[2026-02-17T13:45:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 335ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7w9cowao07","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717761000,"log":"[2026-02-17T14:49:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ennfvpp9aw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717762000,"log":"[2026-02-17T14:25:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 888 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zcaz5rpvq5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717763000,"log":"[2026-02-17T14:28:23Z DEBUG sqlx::query] INSERT FROM users returned 814 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-261fvm8vib","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717764000,"log":"[2026-02-17T13:31:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2448793369867585500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ea10ud920q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717765000,"log":"[2026-02-17T12:40:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mmzrecvs3q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717766000,"log":"[2026-02-17T12:45:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qxqs2v7ahy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717767000,"log":"[2026-02-17T14:03:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 482ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rlm711bfnr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717768000,"log":"[2026-02-17T13:55:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-peq2h3evmv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717769000,"log":"[2026-02-17T13:55:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 204ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mun6954v3n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717770000,"log":"[2026-02-17T12:35:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ap0dnewp7x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717771000,"log":"[2026-02-17T12:41:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-68o5m8mc0g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717772000,"log":"[2026-02-17T14:19:23Z DEBUG sqlx::query] INSERT FROM users returned 111 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4ggua9szm4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717773000,"log":"[2026-02-17T15:09:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5946","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hy1529tpvj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717774000,"log":"[2026-02-17T14:28:40Z DEBUG sqlx::query] DELETE FROM alerts returned 289 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wr3ncgcfhv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717775000,"log":"[2026-02-17T12:52:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gng54il2c7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717776000,"log":"[2026-02-17T14:00:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 93ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yawhyutwzg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717777000,"log":"[2026-02-17T12:35:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1nfe6nq0he","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717778000,"log":"[2026-02-17T15:01:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-usyjrzgvw3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717779000,"log":"[2026-02-17T13:19:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9750194814160593000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-11ihwfu1sk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717780000,"log":"[2026-02-17T14:21:22Z DEBUG sqlx::query] SELECT FROM streams returned 844 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0bxsn0r131","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717781000,"log":"[2026-02-17T15:18:24Z TRACE cache::redis] cache hit key=stream:schema:3679","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-09no1mi7nq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717782000,"log":"[2026-02-17T15:03:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2u3euu81gz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717783000,"log":"[2026-02-17T13:53:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 183ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ycz47a00fw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717784000,"log":"[2026-02-17T14:26:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 181 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ze6vckqukh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717785000,"log":"[2026-02-17T13:50:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 379ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z6oe38zsn1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717786000,"log":"[2026-02-17T14:21:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 613 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8osx7efxeb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717787000,"log":"[2026-02-17T13:49:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 427ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bza3sz1xhe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717788000,"log":"[2026-02-17T13:57:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 409ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rh9a8myidl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717789000,"log":"[2026-02-17T15:15:06Z TRACE cache::redis] cache hit key=auth:token:9999","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-of77iizzgp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717790000,"log":"[2026-02-17T13:09:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5851878001544735000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rom123rjbd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717791000,"log":"[2026-02-17T15:15:39Z TRACE cache::redis] cache set key=stream:schema:107","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dwsqz8e6sp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717792000,"log":"[2026-02-17T13:54:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 495ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-36lfhqwduu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717793000,"log":"[2026-02-17T14:55:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3s7b4ziepv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717794000,"log":"[2026-02-17T14:24:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 40 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qarizb9lgw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717795000,"log":"[2026-02-17T12:59:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t9ddo4mv7l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717796000,"log":"[2026-02-17T14:39:07Z INFO tonic::transport::server] grpc ingester connected client_id=e20xj0zw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aef1albr4o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717797000,"log":"[2026-02-17T13:38:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/908756317109136100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hs5308df8i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717798000,"log":"[2026-02-17T15:21:16Z TRACE cache::redis] cache delete key=auth:token:4816","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p1vwo8a96v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717799000,"log":"[2026-02-17T12:44:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ek9tm1d6fj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717800000,"log":"[2026-02-17T13:29:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5130582334923237000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0iy570txc5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717801000,"log":"[2026-02-17T13:13:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1697316312946815000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zg6ay856vi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717802000,"log":"[2026-02-17T13:01:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ph12x545a5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717803000,"log":"[2026-02-17T15:16:56Z TRACE cache::redis] cache get key=auth:token:305","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9curg3xpz0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717804000,"log":"[2026-02-17T14:24:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 529 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vikakfy55l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717805000,"log":"[2026-02-17T13:43:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 412ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a1pl0sqo14","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717806000,"log":"[2026-02-17T13:40:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7388710373829639000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tcz69s1lz7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717807000,"log":"[2026-02-17T12:51:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2wpbbug970","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717808000,"log":"[2026-02-17T13:07:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ywckj2d66","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717809000,"log":"[2026-02-17T13:22:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1541502968045411800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fu7bkyopq3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717810000,"log":"[2026-02-17T12:52:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1tc5zm3pbw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717811000,"log":"[2026-02-17T14:49:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g292zx96n4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717812000,"log":"[2026-02-17T13:09:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6350114377416019000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7y83wejizm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717813000,"log":"[2026-02-17T14:02:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 162ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2sisvy247g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717814000,"log":"[2026-02-17T15:01:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0rt45swcai","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717815000,"log":"[2026-02-17T13:20:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6368934985448040000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kerodh27bj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717816000,"log":"[2026-02-17T14:22:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 956 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5tfu2pueci","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717817000,"log":"[2026-02-17T13:44:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 439ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ztjpb29ns1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717818000,"log":"[2026-02-17T14:59:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ri4xw1s2ts","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717819000,"log":"[2026-02-17T13:14:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2068669382774623700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-egjp6jsxvg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717820000,"log":"[2026-02-17T14:56:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y835moljcl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717821000,"log":"[2026-02-17T13:16:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4579838393622024000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yqgqawy1gd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717822000,"log":"[2026-02-17T14:22:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 245 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-389oxopnoe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717823000,"log":"[2026-02-17T14:06:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 181ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-znfegfbbwi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717824000,"log":"[2026-02-17T14:32:37Z INFO tonic::transport::server] grpc ingester connected client_id=50nrfjmw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o7bleui16i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717825000,"log":"[2026-02-17T13:26:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3396541645165303000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tqdfm3hw1s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717826000,"log":"[2026-02-17T15:04:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k58bzw883t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717827000,"log":"[2026-02-17T13:52:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 294ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bvoqfzv0vb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717828000,"log":"[2026-02-17T12:51:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pgnd80kwz3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717829000,"log":"[2026-02-17T13:09:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9802708620088814000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gpe429gsmf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717830000,"log":"[2026-02-17T14:44:03Z INFO tonic::transport::server] grpc querier disconnected client_id=s4kryrd6","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g6emdkxulg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717831000,"log":"[2026-02-17T14:48:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=3m7yi9k6","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pbcu0pmmr5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717832000,"log":"[2026-02-17T13:02:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hxnvhhvksd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717833000,"log":"[2026-02-17T14:53:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-68d6ye4b3r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717834000,"log":"[2026-02-17T14:29:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 387 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1b3a06ejcy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717835000,"log":"[2026-02-17T12:53:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u97ciwp8z5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717836000,"log":"[2026-02-17T13:32:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8580509731771671000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0ktpe4xo4a","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717837000,"log":"[2026-02-17T13:35:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2712603671608635400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tq8smz4815","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717838000,"log":"[2026-02-17T15:08:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=9403","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eajt5kqqmn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717839000,"log":"[2026-02-17T14:59:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-py2de1cm5r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717840000,"log":"[2026-02-17T13:34:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9865287189532174000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z84887nkok","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717841000,"log":"[2026-02-17T14:49:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-435iroy4ag","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717842000,"log":"[2026-02-17T13:59:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9php5p1446","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717843000,"log":"[2026-02-17T13:01:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0tcj39ubg8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717844000,"log":"[2026-02-17T14:28:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 575 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x8q1lql4tc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717845000,"log":"[2026-02-17T14:32:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=s1ozrsvg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m7qgugpfz5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717846000,"log":"[2026-02-17T13:35:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1677739855543221000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ysox8hnch9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717847000,"log":"[2026-02-17T14:31:22Z DEBUG sqlx::query] SELECT FROM streams returned 502 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8n2zrybekq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717848000,"log":"[2026-02-17T15:19:15Z TRACE cache::redis] cache set key=query:result:5807","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nav4vkl07a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717849000,"log":"[2026-02-17T13:29:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4223238933366111000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x0xx3z5kyk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717850000,"log":"[2026-02-17T14:50:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cdtnox0k4y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717851000,"log":"[2026-02-17T14:27:30Z DEBUG sqlx::query] DELETE FROM alerts returned 40 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gzsgd0sj0m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717852000,"log":"[2026-02-17T14:58:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3iix6qonq9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717853000,"log":"[2026-02-17T12:47:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6fkob3f0r9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717854000,"log":"[2026-02-17T14:33:52Z INFO tonic::transport::server] grpc ingester connected client_id=0inxhu0v","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8xf5nopjzq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717855000,"log":"[2026-02-17T12:44:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fu6sflnn59","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717856000,"log":"[2026-02-17T14:40:44Z INFO tonic::transport::server] grpc router request_received client_id=nj4531dc","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ia0na1r64x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717857000,"log":"[2026-02-17T14:15:17Z DEBUG sqlx::query] SELECT FROM streams returned 965 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ihhypzqxf2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717858000,"log":"[2026-02-17T12:56:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3jgxozd00b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717859000,"log":"[2026-02-17T14:38:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=lidsammd","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kx9nuwdsyc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717860000,"log":"[2026-02-17T14:57:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rdm5wocl4t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717861000,"log":"[2026-02-17T15:06:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6064","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-69ebvrgd03","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717862000,"log":"[2026-02-17T13:43:45Z INFO actix_web::middleware::logger] GET /api/_search 403 173ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2lkfsbml7h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717863000,"log":"[2026-02-17T15:15:16Z TRACE cache::redis] cache delete key=auth:token:5533","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-10oz7fwo1o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717864000,"log":"[2026-02-17T13:12:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5408544385949039000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dj87z4nm3g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717865000,"log":"[2026-02-17T13:56:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 188ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ogm31b9lxz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717866000,"log":"[2026-02-17T15:19:14Z TRACE cache::redis] cache get key=stream:schema:8257","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d3cnt8zhoh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717867000,"log":"[2026-02-17T14:14:17Z DEBUG sqlx::query] SELECT FROM streams returned 156 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ihpb6x24sz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717868000,"log":"[2026-02-17T12:38:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6l4nvn3ab5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717869000,"log":"[2026-02-17T13:51:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 473ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jkmfg2bdf5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717870000,"log":"[2026-02-17T13:18:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6791474236689066000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kaive8kopt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717871000,"log":"[2026-02-17T14:46:22Z INFO tonic::transport::server] grpc ingester connected client_id=ylz5elly","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ms6jqtmmlu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717872000,"log":"[2026-02-17T13:43:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 35ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a3pu2bwbc8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717873000,"log":"[2026-02-17T13:03:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o49pfouwve","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717874000,"log":"[2026-02-17T14:15:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 247 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zgmy90bp41","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717875000,"log":"[2026-02-17T14:25:55Z DEBUG sqlx::query] DELETE FROM alerts returned 734 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iruqw43rws","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717876000,"log":"[2026-02-17T13:33:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3996823356999919000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-651c34a8cb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717877000,"log":"[2026-02-17T13:32:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2616030244886156300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nip1gfpxwq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717878000,"log":"[2026-02-17T15:05:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=9525","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gdn0e7375f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717879000,"log":"[2026-02-17T13:25:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5846546181210297000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vxbj7stxhv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717880000,"log":"[2026-02-17T14:23:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 829 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w7coe3ez26","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717881000,"log":"[2026-02-17T14:41:28Z INFO tonic::transport::server] grpc querier disconnected client_id=0rd36i5d","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tzqnq41me5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717882000,"log":"[2026-02-17T15:13:48Z TRACE cache::redis] cache hit key=org:config:8868","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9i0zzxdcq2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717883000,"log":"[2026-02-17T15:12:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7865","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7fjpue2ttd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717884000,"log":"[2026-02-17T15:14:07Z TRACE cache::redis] cache miss key=user:session:5173","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-53ng8rrhqf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717885000,"log":"[2026-02-17T13:05:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cswju5n1ac","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717886000,"log":"[2026-02-17T15:08:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1816","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-54ve9lav9r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717887000,"log":"[2026-02-17T14:50:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dechk8tyhu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717888000,"log":"[2026-02-17T15:02:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eb5p88zzwy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717889000,"log":"[2026-02-17T14:43:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=d2g1yvgk","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nxk9z8tdvy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717890000,"log":"[2026-02-17T12:47:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ix7jj1ftl0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717891000,"log":"[2026-02-17T14:39:24Z INFO tonic::transport::server] grpc router request_received client_id=p1uhl4p5","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dlor9sy2og","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717892000,"log":"[2026-02-17T13:23:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1459218104869728300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-16psqhi6in","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717893000,"log":"[2026-02-17T15:20:17Z TRACE cache::redis] cache expire key=user:session:6833","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-160em60x1w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717894000,"log":"[2026-02-17T14:50:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qpnvui2kdp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717895000,"log":"[2026-02-17T14:34:52Z INFO tonic::transport::server] grpc ingester connected client_id=zieksu1l","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-245aiacxwr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717896000,"log":"[2026-02-17T12:45:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7c9sj3dcda","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717897000,"log":"[2026-02-17T14:26:35Z DEBUG sqlx::query] DELETE FROM alerts returned 506 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-brm415gd3p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717898000,"log":"[2026-02-17T14:55:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hzyoxofj74","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717899000,"log":"[2026-02-17T14:42:29Z INFO tonic::transport::server] grpc router request_received client_id=j2cgsqez","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v0udervbxg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717900000,"log":"[2026-02-17T14:04:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 162ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g0mt5beh88","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717901000,"log":"[2026-02-17T15:10:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=8484","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bhxiqh6ek2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717902000,"log":"[2026-02-17T13:45:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 89ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-00yhr92ygc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717903000,"log":"[2026-02-17T15:17:28Z TRACE cache::redis] cache delete key=org:config:2333","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t01cfx3ynf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717904000,"log":"[2026-02-17T14:53:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ebxk1hh83","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717905000,"log":"[2026-02-17T14:13:22Z DEBUG sqlx::query] SELECT FROM streams returned 434 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l9wy3y3bhf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717906000,"log":"[2026-02-17T15:06:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=713","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yrqgsyk6xv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717907000,"log":"[2026-02-17T13:10:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5951871409231864000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o5i7h4x6um","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717908000,"log":"[2026-02-17T14:37:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rynusrgh","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5wehop0z8i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717909000,"log":"[2026-02-17T13:45:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 49ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f8pvd6p9z9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717910000,"log":"[2026-02-17T14:58:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tol5lbwvol","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717911000,"log":"[2026-02-17T13:19:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4174991128068166000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e2oux7nww6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717912000,"log":"[2026-02-17T14:40:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=z4x17dz2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1hjuy0f6nf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717913000,"log":"[2026-02-17T14:35:27Z INFO tonic::transport::server] grpc ingester connected client_id=ilivi0uq","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k6ieyqo72d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717914000,"log":"[2026-02-17T13:00:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tc44gx6mek","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717915000,"log":"[2026-02-17T14:56:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iv6ak47ns7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717916000,"log":"[2026-02-17T14:11:10Z DEBUG sqlx::query] DELETE FROM alerts returned 484 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6bwhypsry2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717917000,"log":"[2026-02-17T14:52:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-73z6vpfipm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717918000,"log":"[2026-02-17T13:34:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5045701052889154000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6m596le3fv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717919000,"log":"[2026-02-17T13:05:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fno3j65p2w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717920000,"log":"[2026-02-17T14:52:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c6c1qr60n2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717921000,"log":"[2026-02-17T13:33:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7281967920735623000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-czck0jvrg8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717922000,"log":"[2026-02-17T13:17:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/533696539129018240.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-scftkehqaj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717923000,"log":"[2026-02-17T14:48:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=zpzudwdv","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3gp1faoo91","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717924000,"log":"[2026-02-17T13:32:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/973917354336661900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pw0nhyki1t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717925000,"log":"[2026-02-17T14:58:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-71w8yevetl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717926000,"log":"[2026-02-17T15:08:00Z DEBUG metrics::reporter] recorded metric active_connections value=4246","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jgjd02lp8j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717927000,"log":"[2026-02-17T13:25:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3433054801020072400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vryzr41re2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717928000,"log":"[2026-02-17T13:07:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ng9sqpde1b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717929000,"log":"[2026-02-17T14:03:00Z INFO actix_web::middleware::logger] POST /api/_search 503 352ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gqzi6duzaf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717930000,"log":"[2026-02-17T14:10:20Z DEBUG sqlx::query] DELETE FROM alerts returned 457 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jhe1r6zfdb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717931000,"log":"[2026-02-17T14:59:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zga9akotly","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717932000,"log":"[2026-02-17T13:46:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 45ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5sqju08z6t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717933000,"log":"[2026-02-17T12:43:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gkgs1ec68m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717934000,"log":"[2026-02-17T12:58:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pzkwwihqsz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717935000,"log":"[2026-02-17T13:04:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-47f7arcjr3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717936000,"log":"[2026-02-17T13:44:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 391ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yq6lyijgel","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717937000,"log":"[2026-02-17T12:44:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u77kc4z5y4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717938000,"log":"[2026-02-17T13:01:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hligqrvunq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717939000,"log":"[2026-02-17T14:04:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 430ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jsihedzxya","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717940000,"log":"[2026-02-17T13:34:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8859053098987184000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x63damzcp4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717941000,"log":"[2026-02-17T12:42:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kpgkcpj5j4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717942000,"log":"[2026-02-17T12:41:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e4rh9ufzr2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717943000,"log":"[2026-02-17T13:36:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5743642538540361000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1vx8agtorf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717944000,"log":"[2026-02-17T13:42:45Z INFO actix_web::middleware::logger] GET /api/_search 503 249ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7jmubyust8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717945000,"log":"[2026-02-17T12:58:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fo6h4jj8a1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717946000,"log":"[2026-02-17T15:16:03Z TRACE cache::redis] cache set key=org:config:1867","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ntv7i60dpr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717947000,"log":"[2026-02-17T13:14:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3461354906111229400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qcyvu5nndg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717948000,"log":"[2026-02-17T12:54:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9jptxsadv6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717949000,"log":"[2026-02-17T12:57:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ayzh0k9uo4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717950000,"log":"[2026-02-17T14:38:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=p86227sh","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yiba7dpup6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717951000,"log":"[2026-02-17T13:42:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 390ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8x6ijvqx3u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717952000,"log":"[2026-02-17T14:42:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=not1k0ws","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9j2le92405","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717953000,"log":"[2026-02-17T12:42:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gedl0sedbp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717954000,"log":"[2026-02-17T14:03:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 395ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rddvngp229","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717955000,"log":"[2026-02-17T14:01:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 448ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rmte1brur9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717956000,"log":"[2026-02-17T13:49:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 155ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s4rdkhel8t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717957000,"log":"[2026-02-17T13:56:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 47ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0aovk1crwj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717958000,"log":"[2026-02-17T12:38:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kkjw75v04d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717959000,"log":"[2026-02-17T14:06:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 196ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-du4blcx7iz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717960000,"log":"[2026-02-17T14:45:38Z INFO tonic::transport::server] grpc querier disconnected client_id=x2tzagl1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l7eg7dii4m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717961000,"log":"[2026-02-17T13:23:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6227261040637195000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v01fi31l9l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717962000,"log":"[2026-02-17T14:36:54Z INFO tonic::transport::server] grpc router request_received client_id=y1khvyur","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p4za46x12p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717963000,"log":"[2026-02-17T13:34:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8317701856743809000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5gd6v21oow","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717964000,"log":"[2026-02-17T13:50:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 241ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zw9jbu6y3f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717965000,"log":"[2026-02-17T12:47:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rz3a7m5ien","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717966000,"log":"[2026-02-17T15:11:15Z DEBUG metrics::reporter] recorded metric active_connections value=323","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jdfq4za9p5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717967000,"log":"[2026-02-17T15:17:34Z TRACE cache::redis] cache delete key=stream:schema:314","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tf94n8h0hj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717968000,"log":"[2026-02-17T12:53:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cjwbqfyekq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717969000,"log":"[2026-02-17T13:09:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6696316845480037000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-24sner7udb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717970000,"log":"[2026-02-17T12:52:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-08fa2mr7s7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717971000,"log":"[2026-02-17T14:04:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 186ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6znnw0xfjt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717972000,"log":"[2026-02-17T15:17:57Z TRACE cache::redis] cache set key=user:session:4640","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6czhhmhapc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717973000,"log":"[2026-02-17T15:02:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pzllkvus6y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717974000,"log":"[2026-02-17T13:49:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 149ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vklg022bfp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717975000,"log":"[2026-02-17T12:36:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vtda15c9fd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717976000,"log":"[2026-02-17T13:55:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 446ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sy5nskts1o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717977000,"log":"[2026-02-17T13:34:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2381253889507278000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sbadnhpdpv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717978000,"log":"[2026-02-17T14:43:19Z INFO tonic::transport::server] grpc router request_received client_id=dad6c5qv","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z5i2l9tvcd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717979000,"log":"[2026-02-17T14:52:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-13vyg1c54o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717980000,"log":"[2026-02-17T14:15:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 791 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-shunsr8aag","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717981000,"log":"[2026-02-17T15:01:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wk1x9wj7lr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717982000,"log":"[2026-02-17T13:33:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3110086927433499600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rj25xtasna","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717983000,"log":"[2026-02-17T13:34:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1552898549282966000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oohgru0ei6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717984000,"log":"[2026-02-17T14:59:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6rj0b46on1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717985000,"log":"[2026-02-17T13:19:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1687205935052733400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z1bv54di3u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717986000,"log":"[2026-02-17T12:44:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6gxe6lq52j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717987000,"log":"[2026-02-17T14:23:45Z DEBUG sqlx::query] DELETE FROM alerts returned 297 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i12zsxtd1q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717988000,"log":"[2026-02-17T14:27:18Z DEBUG sqlx::query] INSERT FROM users returned 326 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-unsfnkssk1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717989000,"log":"[2026-02-17T14:01:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 304ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pnrj3cwc7y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717990000,"log":"[2026-02-17T13:50:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 316ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-phfioimvsd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717991000,"log":"[2026-02-17T13:40:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/915402620882570500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yktr5mjvui","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717992000,"log":"[2026-02-17T14:57:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a73m16vykk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717993000,"log":"[2026-02-17T13:16:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5893926594701437000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-leik0g2hxj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717994000,"log":"[2026-02-17T14:22:18Z DEBUG sqlx::query] INSERT FROM users returned 792 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jkqma9y0x7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717995000,"log":"[2026-02-17T13:31:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/794524879722766600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vsd39cjxvi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717996000,"log":"[2026-02-17T13:29:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7835387808399332000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c2qns7tdum","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331717997000,"log":"[2026-02-17T13:54:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 84ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jw63wq14o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331717998000,"log":"[2026-02-17T14:42:57Z INFO tonic::transport::server] grpc ingester connected client_id=02o1hdy5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u66bgpnqoy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331717999000,"log":"[2026-02-17T12:41:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vignva62nu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718000000,"log":"[2026-02-17T13:48:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 106ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eej37krlc7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718001000,"log":"[2026-02-17T14:08:38Z DEBUG sqlx::query] INSERT FROM users returned 426 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-li0la7htm8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718002000,"log":"[2026-02-17T14:05:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 92ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hw1s4t85pp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718003000,"log":"[2026-02-17T14:12:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 556 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bsf8u5ws15","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718004000,"log":"[2026-02-17T12:51:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b1n8km8odp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718005000,"log":"[2026-02-17T13:11:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8510689567980984000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ol7w9fyis7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718006000,"log":"[2026-02-17T15:10:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9204","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-258b1u3kma","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718007000,"log":"[2026-02-17T15:15:07Z TRACE cache::redis] cache miss key=user:session:4465","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-td316jsoqw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718008000,"log":"[2026-02-17T14:36:02Z INFO tonic::transport::server] grpc ingester connected client_id=ac1wxdmn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4nsm4g31iv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718009000,"log":"[2026-02-17T14:23:05Z DEBUG sqlx::query] DELETE FROM alerts returned 68 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vo2wnxbdwo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718010000,"log":"[2026-02-17T14:46:24Z INFO tonic::transport::server] grpc router request_received client_id=zhv31y1q","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qdt4euiu1j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718011000,"log":"[2026-02-17T13:58:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 317ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6kjthu2g1k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718012000,"log":"[2026-02-17T12:46:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6dd3hh6bfd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718013000,"log":"[2026-02-17T14:00:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 489ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1cux1qd31f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718014000,"log":"[2026-02-17T12:52:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-alse02ymsr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718015000,"log":"[2026-02-17T14:03:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 242ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vr5i60rl6z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718016000,"log":"[2026-02-17T13:07:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ixms8sbkjt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718017000,"log":"[2026-02-17T13:08:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xf460el31l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718018000,"log":"[2026-02-17T15:16:24Z TRACE cache::redis] cache hit key=stream:schema:216","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vs4d9qk10r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718019000,"log":"[2026-02-17T14:10:12Z DEBUG sqlx::query] SELECT FROM streams returned 486 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vojaxadnwq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718020000,"log":"[2026-02-17T14:25:47Z DEBUG sqlx::query] SELECT FROM streams returned 682 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3faopzvt3h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718021000,"log":"[2026-02-17T14:44:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=gpcwdayf","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-thrbcl5jrg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718022000,"log":"[2026-02-17T14:14:18Z DEBUG sqlx::query] INSERT FROM users returned 915 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-boe52mmah1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718023000,"log":"[2026-02-17T14:28:28Z DEBUG sqlx::query] INSERT FROM users returned 141 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t4zxfnauxp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718024000,"log":"[2026-02-17T13:12:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7996852768121463000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gngv4x2ix9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718025000,"log":"[2026-02-17T14:46:27Z INFO tonic::transport::server] grpc ingester connected client_id=ixc55wgs","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nxrwa2ui25","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718026000,"log":"[2026-02-17T13:07:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z5czp5x81l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718027000,"log":"[2026-02-17T13:29:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1526639137275758000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7zm49skdcd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718028000,"log":"[2026-02-17T13:48:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 244ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2eyyh7fa83","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718029000,"log":"[2026-02-17T15:17:35Z TRACE cache::redis] cache expire key=query:result:7951","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g8qazzjx3x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718030000,"log":"[2026-02-17T15:05:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=3956","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5k7jzygz7f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718031000,"log":"[2026-02-17T14:38:08Z INFO tonic::transport::server] grpc querier disconnected client_id=agqpyd41","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nirx0ldn9h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718032000,"log":"[2026-02-17T13:22:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4634430075352432000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7i9bw0n4au","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718033000,"log":"[2026-02-17T15:06:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=611","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zkc9zxasjz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718034000,"log":"[2026-02-17T13:22:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/39302190476164790.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sg85pp9so4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718035000,"log":"[2026-02-17T12:49:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yngz4gr312","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718036000,"log":"[2026-02-17T13:36:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4882968624055397000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ijkj176s5j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718037000,"log":"[2026-02-17T13:47:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 410ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-72eah18dzl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718038000,"log":"[2026-02-17T13:35:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4792318652069241000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y3rndxkls8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718039000,"log":"[2026-02-17T12:39:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k9th4vij1w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718040000,"log":"[2026-02-17T12:53:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-phn4x1k0je","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718041000,"log":"[2026-02-17T14:27:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 435 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kavbtj7awj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718042000,"log":"[2026-02-17T14:27:50Z DEBUG sqlx::query] DELETE FROM alerts returned 279 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yok5scjxcn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718043000,"log":"[2026-02-17T15:00:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a1bh2itrwr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718044000,"log":"[2026-02-17T12:43:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vgij7e8c3j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718045000,"log":"[2026-02-17T14:21:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 942 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2ukkrdi1pa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718046000,"log":"[2026-02-17T14:17:40Z DEBUG sqlx::query] DELETE FROM alerts returned 457 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-16o2gipbw8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718047000,"log":"[2026-02-17T14:44:22Z INFO tonic::transport::server] grpc ingester connected client_id=nbdi4unt","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-owhts7aepy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718048000,"log":"[2026-02-17T14:59:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h0mih1mbdk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718049000,"log":"[2026-02-17T13:09:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/912176013108068100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9y4xkqaa3b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718050000,"log":"[2026-02-17T13:36:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8240949245273846000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p921vskeaa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718051000,"log":"[2026-02-17T14:53:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-09nbyniy79","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718052000,"log":"[2026-02-17T14:53:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xp3l6ikpth","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718053000,"log":"[2026-02-17T12:40:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ph204gu3a5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718054000,"log":"[2026-02-17T14:51:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-12h7yp70au","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718055000,"log":"[2026-02-17T13:51:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 471ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8px3dvv9ng","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718056000,"log":"[2026-02-17T14:16:13Z DEBUG sqlx::query] INSERT FROM users returned 278 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gy3m05hxi2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718057000,"log":"[2026-02-17T13:51:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 47ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n00u9d5vh4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718058000,"log":"[2026-02-17T13:42:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 500ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fasmlgiy8l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718059000,"log":"[2026-02-17T12:52:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gb3mo4wxvw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718060000,"log":"[2026-02-17T13:53:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 415ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-af0yo02zd3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718061000,"log":"[2026-02-17T13:48:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 53ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kiifae5fnn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718062000,"log":"[2026-02-17T14:20:33Z DEBUG sqlx::query] INSERT FROM users returned 425 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ac20ar0oh3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718063000,"log":"[2026-02-17T13:47:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 452ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z911bh1nv3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718064000,"log":"[2026-02-17T15:08:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=848","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-parawvd6up","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718065000,"log":"[2026-02-17T13:54:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 267ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f0wckz5w49","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718066000,"log":"[2026-02-17T13:00:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-io6dowws32","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718067000,"log":"[2026-02-17T14:17:12Z DEBUG sqlx::query] SELECT FROM streams returned 638 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ko27zqm9pa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718068000,"log":"[2026-02-17T14:42:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=etrinjfm","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kerzy3k7zp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718069000,"log":"[2026-02-17T12:47:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-etko423yjv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718070000,"log":"[2026-02-17T13:30:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1223812768583565300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f2t4hcyprt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718071000,"log":"[2026-02-17T14:56:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1pbphhtybn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718072000,"log":"[2026-02-17T14:57:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qo42rubpfa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718073000,"log":"[2026-02-17T15:17:07Z TRACE cache::redis] cache miss key=user:session:6884","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-82kv1pji0v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718074000,"log":"[2026-02-17T14:20:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 924 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u9ahxg5zvh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718075000,"log":"[2026-02-17T14:35:57Z INFO tonic::transport::server] grpc ingester connected client_id=mutcqj2t","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hcqsw2n78m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718076000,"log":"[2026-02-17T13:14:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6824381466232765000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f8pmczgro1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718077000,"log":"[2026-02-17T12:56:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tv4ecofglt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718078000,"log":"[2026-02-17T12:54:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vnlr726vgu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718079000,"log":"[2026-02-17T12:42:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dd5lbvqm46","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718080000,"log":"[2026-02-17T13:51:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 289ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3vma9uvoqd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718081000,"log":"[2026-02-17T14:07:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 222 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qv6o6pt2jx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718082000,"log":"[2026-02-17T13:49:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 220ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2jjrpsfwow","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718083000,"log":"[2026-02-17T13:23:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5538615600022605000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fsha0zaab1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718084000,"log":"[2026-02-17T14:58:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-taygarz9p8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718085000,"log":"[2026-02-17T14:02:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 140ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e8pa7z6zjm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718086000,"log":"[2026-02-17T12:35:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4hcgjkdp1m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718087000,"log":"[2026-02-17T14:24:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 415 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-90fqnua5jz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718088000,"log":"[2026-02-17T14:37:48Z INFO tonic::transport::server] grpc querier disconnected client_id=j3u4eo0r","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7nn18feoti","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718089000,"log":"[2026-02-17T13:19:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6236845440865178000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x98eve52xj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718090000,"log":"[2026-02-17T15:09:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=3580","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5hz2octluv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718091000,"log":"[2026-02-17T13:28:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3749134572347312600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5fvm77i8ky","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718092000,"log":"[2026-02-17T13:29:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1012811880859587600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vjse9oo8ub","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718093000,"log":"[2026-02-17T14:36:37Z INFO tonic::transport::server] grpc ingester connected client_id=agi5pb1a","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p79gulqd7z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718094000,"log":"[2026-02-17T14:18:07Z DEBUG sqlx::query] SELECT FROM streams returned 511 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aqdl2um3pc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718095000,"log":"[2026-02-17T14:50:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4rhwow40nd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718096000,"log":"[2026-02-17T14:29:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 164 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q7jv2pw2ue","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718097000,"log":"[2026-02-17T14:21:27Z DEBUG sqlx::query] SELECT FROM streams returned 817 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dnkz2s5e65","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718098000,"log":"[2026-02-17T12:51:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-atgf9v55ku","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718099000,"log":"[2026-02-17T14:04:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 412ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ligfu574b6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718100000,"log":"[2026-02-17T14:29:45Z DEBUG sqlx::query] DELETE FROM alerts returned 106 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2toefehhef","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718101000,"log":"[2026-02-17T13:20:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/125949089527876980.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5xh5z20p80","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718102000,"log":"[2026-02-17T14:15:47Z DEBUG sqlx::query] SELECT FROM streams returned 795 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4k3bloa5hv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718103000,"log":"[2026-02-17T13:31:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6502603563130635000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m2dxwr9icb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718104000,"log":"[2026-02-17T13:59:40Z INFO actix_web::middleware::logger] POST /api/_search 404 303ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hfc4x52zna","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718105000,"log":"[2026-02-17T14:40:17Z INFO tonic::transport::server] grpc ingester connected client_id=ejyicd3k","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m0wgkf6leg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718106000,"log":"[2026-02-17T12:44:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2hw3ajtonm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718107000,"log":"[2026-02-17T14:24:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 359 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ilmyshjv6c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718108000,"log":"[2026-02-17T13:48:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 173ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ec18twm5jf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718109000,"log":"[2026-02-17T12:45:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5axslluxni","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718110000,"log":"[2026-02-17T13:21:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/370261102307603000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-guc9i44ylx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718111000,"log":"[2026-02-17T13:59:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 320ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1h7ooyzs5m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718112000,"log":"[2026-02-17T13:49:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hp32xxmijm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718113000,"log":"[2026-02-17T14:33:24Z INFO tonic::transport::server] grpc router request_received client_id=9ke3gv22","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-46bf00zwo0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718114000,"log":"[2026-02-17T12:50:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ayceb5fkei","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718115000,"log":"[2026-02-17T14:00:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 339ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h7cdwq2t5x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718116000,"log":"[2026-02-17T12:36:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mdgl8xymdb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718117000,"log":"[2026-02-17T14:57:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-urvc1zl1zr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718118000,"log":"[2026-02-17T13:15:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7352745498891395000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-flhzc1w0as","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718119000,"log":"[2026-02-17T14:00:20Z INFO actix_web::middleware::logger] POST /api/_search 201 250ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fpksn4dsew","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718120000,"log":"[2026-02-17T15:05:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=589","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fbz9odf4a2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718121000,"log":"[2026-02-17T12:47:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tmflffzml4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718122000,"log":"[2026-02-17T13:21:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9036482685066348000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-77r13tqdlm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718123000,"log":"[2026-02-17T15:05:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=9761","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-je5g4ovrbq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718124000,"log":"[2026-02-17T12:44:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-us9dbvmwen","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718125000,"log":"[2026-02-17T14:16:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 970 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2xi153v5z9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718126000,"log":"[2026-02-17T14:19:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 632 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e16vqxr5em","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718127000,"log":"[2026-02-17T14:48:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7i6bx5d20q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718128000,"log":"[2026-02-17T14:11:07Z DEBUG sqlx::query] SELECT FROM streams returned 649 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fj51bsocun","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718129000,"log":"[2026-02-17T15:02:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-16cmbvvy4i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718130000,"log":"[2026-02-17T12:59:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j2pdloj3co","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718131000,"log":"[2026-02-17T13:17:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7506399769790865000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5468051wmv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718132000,"log":"[2026-02-17T13:56:20Z INFO actix_web::middleware::logger] POST /api/_search 401 207ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rhfpb9cgns","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718133000,"log":"[2026-02-17T14:15:22Z DEBUG sqlx::query] SELECT FROM streams returned 429 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xww1s6u9tj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718134000,"log":"[2026-02-17T13:19:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3232283202477715000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tjdb2am7u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718135000,"log":"[2026-02-17T13:55:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 393ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c9r6z8vjsv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718136000,"log":"[2026-02-17T12:37:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j7sbvi2gki","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718137000,"log":"[2026-02-17T15:09:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=9158","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-431fng8l69","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718138000,"log":"[2026-02-17T14:21:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 893 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r7f00a8uwn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718139000,"log":"[2026-02-17T13:10:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5841806590630088000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kovqek63qv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718140000,"log":"[2026-02-17T13:34:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7050728820014076000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-17ft4r11dk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718141000,"log":"[2026-02-17T13:49:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 326ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s7mu7au0t7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718142000,"log":"[2026-02-17T13:32:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5348113531079069000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y7ns9sqd0c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718143000,"log":"[2026-02-17T13:11:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7251975567702062000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cee23r327z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718144000,"log":"[2026-02-17T12:48:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ljmleaelpg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718145000,"log":"[2026-02-17T13:57:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 129ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8cycghlqdx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718146000,"log":"[2026-02-17T12:57:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fnfe0q0hkz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718147000,"log":"[2026-02-17T13:07:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xzns79zftt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718148000,"log":"[2026-02-17T13:57:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 95ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cgbduzcd8o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718149000,"log":"[2026-02-17T13:46:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 158ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tjcvdmb7kp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718150000,"log":"[2026-02-17T13:24:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5267426722682094000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qni17xjopc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718151000,"log":"[2026-02-17T12:44:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0f9elu6z9z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718152000,"log":"[2026-02-17T14:37:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=j3dzm1gs","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d1fralb9cd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718153000,"log":"[2026-02-17T12:56:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i89ketelas","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718154000,"log":"[2026-02-17T13:44:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 428ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ilnim1mc6b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718155000,"log":"[2026-02-17T13:15:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4094969277724165000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qnse87s1ga","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718156000,"log":"[2026-02-17T13:58:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 341ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aypezfkbyv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718157000,"log":"[2026-02-17T13:27:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3884909021993534500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xiiwp36j91","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718158000,"log":"[2026-02-17T12:57:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p3ssmy9dr4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718159000,"log":"[2026-02-17T14:38:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2w77lr67","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fwe9ekielu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718160000,"log":"[2026-02-17T13:43:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 313ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ag8l40aswi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718161000,"log":"[2026-02-17T14:44:17Z INFO tonic::transport::server] grpc ingester connected client_id=psuboj0k","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kasdwqv750","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718162000,"log":"[2026-02-17T13:01:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tsfjwxrr0i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718163000,"log":"[2026-02-17T13:22:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2877831270025901000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-88e2s9v891","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718164000,"log":"[2026-02-17T13:16:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5154712871026747000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a3k0883b02","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718165000,"log":"[2026-02-17T14:39:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=iwry2n56","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o2u5osvxyp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718166000,"log":"[2026-02-17T15:05:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=6364","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-19m0pizo3z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718167000,"log":"[2026-02-17T14:48:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5o4obbaub1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718168000,"log":"[2026-02-17T13:53:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 210ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lphnzflad4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718169000,"log":"[2026-02-17T14:43:04Z INFO tonic::transport::server] grpc router request_received client_id=6t98jv52","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ykj2kwnw7l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718170000,"log":"[2026-02-17T13:12:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9809621396659050000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4ugrkswkfc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718171000,"log":"[2026-02-17T12:48:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wcpccnaz8o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718172000,"log":"[2026-02-17T13:55:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 348ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wt1qkuw12b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718173000,"log":"[2026-02-17T15:03:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qctl3r2xso","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718174000,"log":"[2026-02-17T14:25:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 820 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-boizqawp9f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718175000,"log":"[2026-02-17T14:12:47Z DEBUG sqlx::query] SELECT FROM streams returned 798 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yplqjfcky4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718176000,"log":"[2026-02-17T14:28:33Z DEBUG sqlx::query] INSERT FROM users returned 840 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z48r3fl4qv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718177000,"log":"[2026-02-17T14:11:27Z DEBUG sqlx::query] SELECT FROM streams returned 186 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-764iffn1t4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718178000,"log":"[2026-02-17T14:05:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 212ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f2w51o2qbx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718179000,"log":"[2026-02-17T13:40:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1279574543129873400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-05f7059ij1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718180000,"log":"[2026-02-17T15:02:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-czwb0f9ks4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718181000,"log":"[2026-02-17T13:16:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5232635874915497000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-05j9zabnf6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718182000,"log":"[2026-02-17T13:42:00Z INFO actix_web::middleware::logger] POST /api/_search 503 373ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6uxt9n3uu1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718183000,"log":"[2026-02-17T13:37:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/506818391979626500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mzkoffu9rp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718184000,"log":"[2026-02-17T14:49:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qn3hi4s2vv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718185000,"log":"[2026-02-17T14:24:32Z DEBUG sqlx::query] SELECT FROM streams returned 595 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ottab2j3c6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718186000,"log":"[2026-02-17T12:49:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ratxn1hgzi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718187000,"log":"[2026-02-17T15:09:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1963","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tj46wcxxfo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718188000,"log":"[2026-02-17T14:00:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 352ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3kujnxum4c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718189000,"log":"[2026-02-17T13:48:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 315ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k6d62bwg0q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718190000,"log":"[2026-02-17T13:13:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4204165061985603000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bpnjztk2py","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718191000,"log":"[2026-02-17T13:17:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/247984040127302560.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pw8k52xiy9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718192000,"log":"[2026-02-17T13:11:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4032752695863377400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9rsumx84cm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718193000,"log":"[2026-02-17T14:17:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 115 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9hmu38gkmi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718194000,"log":"[2026-02-17T13:34:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3537049231172811000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4kaax9jfs2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718195000,"log":"[2026-02-17T14:29:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 958 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vu2x1nlqg8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718196000,"log":"[2026-02-17T14:49:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ma88c7jec","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718197000,"log":"[2026-02-17T13:16:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2812298715107803600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5flqhmcm2m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718198000,"log":"[2026-02-17T13:07:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5i98wq4n7q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718199000,"log":"[2026-02-17T14:56:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xvtvgo9gxy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718200000,"log":"[2026-02-17T13:32:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8569665970440283000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6g7kof27iv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718201000,"log":"[2026-02-17T14:34:07Z INFO tonic::transport::server] grpc ingester connected client_id=46jk7h7b","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tvo9rcsi3m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718202000,"log":"[2026-02-17T14:37:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ch11m3ea","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qrtg01b8og","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718203000,"log":"[2026-02-17T14:22:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 317 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-piu3my5bil","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718204000,"log":"[2026-02-17T12:43:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bzwq8ewzi7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718205000,"log":"[2026-02-17T13:36:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4814663949853191000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k6rjonxgrs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718206000,"log":"[2026-02-17T15:11:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8567","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ah2b8slcfy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718207000,"log":"[2026-02-17T12:39:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pehb83ehd2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718208000,"log":"[2026-02-17T15:04:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e82tzl2on5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718209000,"log":"[2026-02-17T12:57:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1c558h7u2z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718210000,"log":"[2026-02-17T14:10:52Z DEBUG sqlx::query] SELECT FROM streams returned 787 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nckm3v407r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718211000,"log":"[2026-02-17T14:54:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zrw5s2vdqh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718212000,"log":"[2026-02-17T15:06:48Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=5834","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mebehno9py","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718213000,"log":"[2026-02-17T12:39:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ur74egc1yg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718214000,"log":"[2026-02-17T13:28:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2094425459909308200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j0n6drs6pg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718215000,"log":"[2026-02-17T13:27:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2706578571264420000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n76i2631fi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718216000,"log":"[2026-02-17T15:08:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3519","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6ugek56j3c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718217000,"log":"[2026-02-17T12:44:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uf5ybcfv09","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718218000,"log":"[2026-02-17T14:23:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 837 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-70r5jwfbib","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718219000,"log":"[2026-02-17T14:36:14Z INFO tonic::transport::server] grpc router request_received client_id=2cw2f5wu","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lviqxw9r11","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718220000,"log":"[2026-02-17T15:20:11Z TRACE cache::redis] cache expire key=auth:token:718","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7hnv9eztel","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718221000,"log":"[2026-02-17T13:06:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ef0pbunoa3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718222000,"log":"[2026-02-17T14:05:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 429ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3k4180jltj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718223000,"log":"[2026-02-17T13:05:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-41ozoxt04y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718224000,"log":"[2026-02-17T15:11:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=7774","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7nq7xqzur5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718225000,"log":"[2026-02-17T13:54:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 291ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rfra68og83","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718226000,"log":"[2026-02-17T14:33:54Z INFO tonic::transport::server] grpc router request_received client_id=o3hdypx6","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9s96o3glv7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718227000,"log":"[2026-02-17T15:18:15Z TRACE cache::redis] cache set key=query:result:1318","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pcq0vh79ia","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718228000,"log":"[2026-02-17T14:15:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 107 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fq96mvmqa3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718229000,"log":"[2026-02-17T14:17:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 673 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zkhrus48uz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718230000,"log":"[2026-02-17T15:19:30Z TRACE cache::redis] cache hit key=query:result:6317","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g4sb3btr3b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718231000,"log":"[2026-02-17T14:59:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ubw8urnvga","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718232000,"log":"[2026-02-17T13:05:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0c970t2qbh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718233000,"log":"[2026-02-17T13:19:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3381838398944925000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jlw5ancqxx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718234000,"log":"[2026-02-17T12:58:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pjpqurhfk1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718235000,"log":"[2026-02-17T15:13:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=892","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4suul9408x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718236000,"log":"[2026-02-17T13:41:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7235545110813714000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-72fdiej3ct","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718237000,"log":"[2026-02-17T12:45:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0dzi14o8fw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718238000,"log":"[2026-02-17T14:05:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 167ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3z4ntsb0ip","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718239000,"log":"[2026-02-17T12:42:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5ac1en1viu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718240000,"log":"[2026-02-17T13:59:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 328ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qybm5jkm9q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718241000,"log":"[2026-02-17T12:47:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2dnhxbihjl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718242000,"log":"[2026-02-17T12:46:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7vqtkgszhh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718243000,"log":"[2026-02-17T13:26:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7969450967825145000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-39tfr1zsk1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718244000,"log":"[2026-02-17T14:18:17Z DEBUG sqlx::query] SELECT FROM streams returned 244 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iazk4dfbs0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718245000,"log":"[2026-02-17T13:40:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4390768436907937000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yk3rl1aa4t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718246000,"log":"[2026-02-17T12:46:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w2gdoivegy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718247000,"log":"[2026-02-17T12:35:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nj2kjfap92","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718248000,"log":"[2026-02-17T14:05:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 221ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1x25he0na1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718249000,"log":"[2026-02-17T14:47:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=8ptluzz9","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jbr52xrxna","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718250000,"log":"[2026-02-17T15:13:55Z TRACE cache::redis] cache miss key=query:result:5471","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lybpomsh90","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718251000,"log":"[2026-02-17T14:18:03Z DEBUG sqlx::query] INSERT FROM users returned 675 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ebh2ig44pa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718252000,"log":"[2026-02-17T13:08:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7663904754386044000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-28efhdfle2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718253000,"log":"[2026-02-17T13:50:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 326ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ghb6hol6hb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718254000,"log":"[2026-02-17T15:19:56Z TRACE cache::redis] cache get key=auth:token:6783","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x3ojsgrhls","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718255000,"log":"[2026-02-17T14:30:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 365 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n2f75bift0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718256000,"log":"[2026-02-17T13:42:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 149ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-461r3nz67a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718257000,"log":"[2026-02-17T13:02:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zu3d1x5l45","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718258000,"log":"[2026-02-17T14:54:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9yn5gvmw4e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718259000,"log":"[2026-02-17T13:46:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 332ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s7edqxupsi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718260000,"log":"[2026-02-17T12:49:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dpvgfpy2e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718261000,"log":"[2026-02-17T13:49:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 103ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fdqb9jts52","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718262000,"log":"[2026-02-17T14:02:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 229ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nsoet2rf61","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718263000,"log":"[2026-02-17T14:11:55Z DEBUG sqlx::query] DELETE FROM alerts returned 523 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m81caq0nlb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718264000,"log":"[2026-02-17T12:55:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mxryeqqk97","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718265000,"log":"[2026-02-17T13:54:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 97ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fe4l9h95qu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718266000,"log":"[2026-02-17T14:00:45Z INFO actix_web::middleware::logger] GET /api/_search 503 309ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-25otqt7mcb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718267000,"log":"[2026-02-17T15:17:52Z TRACE cache::redis] cache delete key=user:session:8547","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iossq4osth","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718268000,"log":"[2026-02-17T14:30:12Z DEBUG sqlx::query] SELECT FROM streams returned 728 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sxuz4xempo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718269000,"log":"[2026-02-17T13:31:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/151572892751670180.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-70fsge86c1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718270000,"log":"[2026-02-17T13:36:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2595258661372712400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g9jpx2r7h4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718271000,"log":"[2026-02-17T14:21:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 960 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7u0z1pliay","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718272000,"log":"[2026-02-17T13:38:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4105612778002052600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yqscw125jj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718273000,"log":"[2026-02-17T15:07:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9471","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cjiwrjpvus","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718274000,"log":"[2026-02-17T13:49:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 6ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hzu52c9bz7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718275000,"log":"[2026-02-17T14:14:35Z DEBUG sqlx::query] DELETE FROM alerts returned 713 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-69y8f6tzy4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718276000,"log":"[2026-02-17T13:56:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 283ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6gm7a96idt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718277000,"log":"[2026-02-17T15:04:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2ob7zyat91","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718278000,"log":"[2026-02-17T14:38:59Z INFO tonic::transport::server] grpc router request_received client_id=meq26pw1","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-va6nonc8cf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718279000,"log":"[2026-02-17T14:41:22Z INFO tonic::transport::server] grpc ingester connected client_id=dyyw8xdj","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s03stgf5ul","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718280000,"log":"[2026-02-17T15:04:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j51h8tyxn7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718281000,"log":"[2026-02-17T14:15:32Z DEBUG sqlx::query] SELECT FROM streams returned 861 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4i1u8x97cm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718282000,"log":"[2026-02-17T15:04:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6wmowrbygz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718283000,"log":"[2026-02-17T14:00:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 252ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gs8r57vq9j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718284000,"log":"[2026-02-17T13:38:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1075736346738029800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3py0he17ih","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718285000,"log":"[2026-02-17T14:56:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4ximsg8t4n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718286000,"log":"[2026-02-17T14:31:20Z DEBUG sqlx::query] DELETE FROM alerts returned 737 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f7aysjn24i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718287000,"log":"[2026-02-17T13:01:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sc5xhjkz0l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718288000,"log":"[2026-02-17T14:54:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uuthvjry7z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718289000,"log":"[2026-02-17T12:43:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gcped1ntx9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718290000,"log":"[2026-02-17T14:28:48Z DEBUG sqlx::query] INSERT FROM users returned 209 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h0uhe0yxht","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718291000,"log":"[2026-02-17T13:18:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6216257910476632000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-woznn03ohy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718292000,"log":"[2026-02-17T14:09:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 940 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3akqlnvh29","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718293000,"log":"[2026-02-17T13:10:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8098331285455535000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iij3zxti77","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718294000,"log":"[2026-02-17T13:25:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6836366762957762000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pvgjre3gbg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718295000,"log":"[2026-02-17T14:38:03Z INFO tonic::transport::server] grpc querier disconnected client_id=oxoixjoo","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vbjcmgcy76","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718296000,"log":"[2026-02-17T13:30:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7209539946941726000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1apwy2uwbc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718297000,"log":"[2026-02-17T13:43:20Z INFO actix_web::middleware::logger] POST /api/_search 500 178ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-412lkjnn9u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718298000,"log":"[2026-02-17T15:02:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q8oa2t289q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718299000,"log":"[2026-02-17T13:35:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/928614160628891100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-10b20sj1ch","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718300000,"log":"[2026-02-17T12:41:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-307zpvwlnx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718301000,"log":"[2026-02-17T15:16:54Z TRACE cache::redis] cache hit key=stream:schema:623","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-efit8z5q80","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718302000,"log":"[2026-02-17T14:24:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 386 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e9cf3lw2eo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718303000,"log":"[2026-02-17T14:45:39Z INFO tonic::transport::server] grpc router request_received client_id=nltpdtnu","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ofunmk0kr1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718304000,"log":"[2026-02-17T14:50:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6zr3r9jru6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718305000,"log":"[2026-02-17T13:01:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rmgqekqlg8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718306000,"log":"[2026-02-17T13:05:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5mcwnh7inj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718307000,"log":"[2026-02-17T14:49:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f4dsq3kmib","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718308000,"log":"[2026-02-17T13:22:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9161053396583774000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7fppjuotyj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718309000,"log":"[2026-02-17T13:54:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 120ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-odvpu12pry","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718310000,"log":"[2026-02-17T13:41:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3861710879747586600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xxz1c0nmf7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718311000,"log":"[2026-02-17T14:47:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=k7bog66s","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gxgmiepekg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718312000,"log":"[2026-02-17T14:09:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 271 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z7a07jxzc3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718313000,"log":"[2026-02-17T13:13:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2819982794833999400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oll6hardd6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718314000,"log":"[2026-02-17T14:46:14Z INFO tonic::transport::server] grpc router request_received client_id=pla4zh65","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9499khizpr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718315000,"log":"[2026-02-17T13:03:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q0i5p5edms","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718316000,"log":"[2026-02-17T14:31:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 581 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b5e7hq74g9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718317000,"log":"[2026-02-17T12:42:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w5vh8dod2u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718318000,"log":"[2026-02-17T12:55:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4iy31mhsxl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718319000,"log":"[2026-02-17T12:42:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mn2c34cfz0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718320000,"log":"[2026-02-17T13:32:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/593109968507295000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-obbiadlpe8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718321000,"log":"[2026-02-17T14:47:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=q7kv4rtt","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9q3duc1l5x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718322000,"log":"[2026-02-17T12:57:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1odv0ic12b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718323000,"log":"[2026-02-17T13:46:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 196ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4r4jvakvtn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718324000,"log":"[2026-02-17T13:03:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ktacnuvd35","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718325000,"log":"[2026-02-17T13:50:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 60ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zjk2mjoqij","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718326000,"log":"[2026-02-17T12:37:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9s6pja77uq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718327000,"log":"[2026-02-17T15:10:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1467","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-19bseb3q2r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718328000,"log":"[2026-02-17T13:23:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5675758621754110000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i1ivtarq5u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718329000,"log":"[2026-02-17T15:13:57Z TRACE cache::redis] cache set key=user:session:5055","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4n2wv8ao4v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718330000,"log":"[2026-02-17T15:03:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8nwbxnqr37","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718331000,"log":"[2026-02-17T13:25:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7512571893451831000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-llgmy06eub","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718332000,"log":"[2026-02-17T13:30:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3115110861981760000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zpm51qmf7d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718333000,"log":"[2026-02-17T12:37:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-okcroc2dlp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718334000,"log":"[2026-02-17T14:37:39Z INFO tonic::transport::server] grpc router request_received client_id=d9d7vbod","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-evj28cfyyx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718335000,"log":"[2026-02-17T15:14:05Z TRACE cache::redis] cache expire key=query:result:846","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6rhgl2di6o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718336000,"log":"[2026-02-17T13:35:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7301219269796771000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2x4ly3bp12","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718337000,"log":"[2026-02-17T14:38:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rbkl2qot","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-znsn3dzzp6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718338000,"log":"[2026-02-17T13:10:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7126855968316328000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lx654jjh9e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718339000,"log":"[2026-02-17T14:36:24Z INFO tonic::transport::server] grpc router request_received client_id=5j28ml0a","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wps138s6dj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718340000,"log":"[2026-02-17T14:20:12Z DEBUG sqlx::query] SELECT FROM streams returned 968 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-657qmaeha1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718341000,"log":"[2026-02-17T12:36:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4byataxuvx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718342000,"log":"[2026-02-17T14:59:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rsop2v0ypu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718343000,"log":"[2026-02-17T15:09:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=6910","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zxy7bedvve","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718344000,"log":"[2026-02-17T15:06:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=4474","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-02kuxdouog","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718345000,"log":"[2026-02-17T13:18:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3874822270191371300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cq0ywt9lvp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718346000,"log":"[2026-02-17T12:40:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g86p11q2mo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718347000,"log":"[2026-02-17T14:48:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=l5371oht","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-94qiu9mofk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718348000,"log":"[2026-02-17T12:56:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-11rmpo64zp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718349000,"log":"[2026-02-17T15:01:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ukzvsn433a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718350000,"log":"[2026-02-17T14:18:22Z DEBUG sqlx::query] SELECT FROM streams returned 833 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2swbcc0syz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718351000,"log":"[2026-02-17T14:22:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 627 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gdm7udy5ru","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718352000,"log":"[2026-02-17T13:51:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 369ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e9hxmi799n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718353000,"log":"[2026-02-17T12:53:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5yvdmty8if","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718354000,"log":"[2026-02-17T12:44:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4ak1mb7815","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718355000,"log":"[2026-02-17T13:20:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2065564571630083000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-awskqh4wrr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718356000,"log":"[2026-02-17T14:02:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 311ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pmpyn2we1i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718357000,"log":"[2026-02-17T14:39:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=dgmyms2u","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3hkrh8bvqp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718358000,"log":"[2026-02-17T14:10:17Z DEBUG sqlx::query] SELECT FROM streams returned 685 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wi86nlyn0r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718359000,"log":"[2026-02-17T14:40:29Z INFO tonic::transport::server] grpc router request_received client_id=pchlkvyj","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bstb5re2vd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718360000,"log":"[2026-02-17T14:43:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=kn78bsal","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bghipynhdy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718361000,"log":"[2026-02-17T13:22:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5471829338815784000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2fmll9dqae","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718362000,"log":"[2026-02-17T14:45:03Z INFO tonic::transport::server] grpc querier disconnected client_id=emupx0l2","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7fjikdqbce","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718363000,"log":"[2026-02-17T13:50:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 377ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z5nbzll3qd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718364000,"log":"[2026-02-17T14:57:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ltq8c1thzo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718365000,"log":"[2026-02-17T15:01:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dpifl8xqxe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718366000,"log":"[2026-02-17T14:37:58Z INFO tonic::transport::server] grpc querier disconnected client_id=zefkgsm5","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-16rga8b4o4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718367000,"log":"[2026-02-17T13:08:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iv6pr8nika","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718368000,"log":"[2026-02-17T14:45:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=tbnq79pg","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i6q2erajm5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718369000,"log":"[2026-02-17T12:49:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nu5nitoy3d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718370000,"log":"[2026-02-17T15:20:24Z TRACE cache::redis] cache hit key=stream:schema:7625","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qbuzw6ggv6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718371000,"log":"[2026-02-17T12:56:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-33fspfqof6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718372000,"log":"[2026-02-17T13:42:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 289ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7zmusbr2of","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718373000,"log":"[2026-02-17T14:39:37Z INFO tonic::transport::server] grpc ingester connected client_id=qnhl0vvv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hoalz1xkah","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718374000,"log":"[2026-02-17T14:06:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 418ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ddjmcfk8nc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718375000,"log":"[2026-02-17T15:20:50Z TRACE cache::redis] cache get key=query:result:5349","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ha43vgivg9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718376000,"log":"[2026-02-17T13:08:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rgotph3mn5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718377000,"log":"[2026-02-17T15:07:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=700","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vpdl1ry124","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718378000,"log":"[2026-02-17T13:56:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 85ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-70edcpkjrb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718379000,"log":"[2026-02-17T13:13:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2852228061819146000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v50fr1q5rp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718380000,"log":"[2026-02-17T13:06:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jcrdbpiu6x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718381000,"log":"[2026-02-17T13:25:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4578662756501111000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t6euz4uzyv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718382000,"log":"[2026-02-17T14:08:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 958 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ftik1wvwqh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718383000,"log":"[2026-02-17T14:41:42Z INFO tonic::transport::server] grpc ingester connected client_id=ihbc9r73","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6i65f8tox0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718384000,"log":"[2026-02-17T13:29:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5935360646252588000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1xtak9juwc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718385000,"log":"[2026-02-17T13:54:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 232ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fuadwust7e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718386000,"log":"[2026-02-17T14:11:40Z DEBUG sqlx::query] DELETE FROM alerts returned 890 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kbscp3bjlz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718387000,"log":"[2026-02-17T13:23:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6086584211062603000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gat0rjqm5p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718388000,"log":"[2026-02-17T12:37:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5esapj1oby","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718389000,"log":"[2026-02-17T13:27:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6354815262350548000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-obh242nz11","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718390000,"log":"[2026-02-17T12:46:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2hjd5t672a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718391000,"log":"[2026-02-17T15:15:33Z TRACE cache::redis] cache set key=org:config:3507","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fqx3o6cdjf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718392000,"log":"[2026-02-17T14:51:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k6fy75be2p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718393000,"log":"[2026-02-17T13:11:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4639419055499745000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p636kobnj4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718394000,"log":"[2026-02-17T14:36:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=7bo3bw7n","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-occoui472f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718395000,"log":"[2026-02-17T13:14:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2059314396573495000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qhw3eexh1k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718396000,"log":"[2026-02-17T12:58:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c2y0ip128v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718397000,"log":"[2026-02-17T14:21:37Z DEBUG sqlx::query] SELECT FROM streams returned 862 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iprk8a9bp6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718398000,"log":"[2026-02-17T14:42:17Z INFO tonic::transport::server] grpc ingester connected client_id=8961hogl","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-antpyo8mac","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718399000,"log":"[2026-02-17T13:15:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2804148386863137000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4n07xeda1b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718400000,"log":"[2026-02-17T12:53:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-184wqlhfgc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718401000,"log":"[2026-02-17T14:42:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=by4e7uhm","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fog9l3l0mo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718402000,"log":"[2026-02-17T14:42:04Z INFO tonic::transport::server] grpc router request_received client_id=w2lcj8w9","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l139uyxboh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718403000,"log":"[2026-02-17T14:49:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qnrpsh6qyj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718404000,"log":"[2026-02-17T12:52:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5332t41lu9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718405000,"log":"[2026-02-17T15:01:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mf3e02cs0f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718406000,"log":"[2026-02-17T14:25:17Z DEBUG sqlx::query] SELECT FROM streams returned 783 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-womjgro8gx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718407000,"log":"[2026-02-17T13:34:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5228673595660680000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-430pjm4dtt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718408000,"log":"[2026-02-17T14:46:18Z INFO tonic::transport::server] grpc querier disconnected client_id=0lye93xo","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fah18j27vz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718409000,"log":"[2026-02-17T15:17:30Z TRACE cache::redis] cache hit key=query:result:6333","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3p73xgrh3m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718410000,"log":"[2026-02-17T14:16:42Z DEBUG sqlx::query] SELECT FROM streams returned 214 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pj8a4mm4rb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718411000,"log":"[2026-02-17T13:04:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ft1v5v2nye","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718412000,"log":"[2026-02-17T14:41:47Z INFO tonic::transport::server] grpc ingester connected client_id=vzjqa7oo","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ltyntucm58","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718413000,"log":"[2026-02-17T13:38:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2985775644748978000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k7wwreeedd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718414000,"log":"[2026-02-17T13:46:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 356ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-efyuu6yimm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718415000,"log":"[2026-02-17T13:16:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7566295485440662000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ocku3vodi9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718416000,"log":"[2026-02-17T13:41:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 453ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aok3wleq81","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718417000,"log":"[2026-02-17T15:02:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jm351f9njj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718418000,"log":"[2026-02-17T13:54:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 494ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-azy1ut3oxl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718419000,"log":"[2026-02-17T13:28:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8008904091633651000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6zawslgfe3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718420000,"log":"[2026-02-17T15:12:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7374","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0wdg3sl70s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718421000,"log":"[2026-02-17T14:28:10Z DEBUG sqlx::query] DELETE FROM alerts returned 651 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yqm7lj8q7k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718422000,"log":"[2026-02-17T12:54:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pww4vulh7j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718423000,"log":"[2026-02-17T13:33:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5250237028237237000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c6hk1ysnxj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718424000,"log":"[2026-02-17T13:53:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 382ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1ijtqvo1kh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718425000,"log":"[2026-02-17T14:02:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 320ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ftaq60s9oz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718426000,"log":"[2026-02-17T13:44:40Z INFO actix_web::middleware::logger] POST /api/_search 404 461ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qpona262di","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718427000,"log":"[2026-02-17T14:30:07Z DEBUG sqlx::query] SELECT FROM streams returned 577 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rt8pl60h55","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718428000,"log":"[2026-02-17T13:27:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2925177798049909000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hv4tsww2vn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718429000,"log":"[2026-02-17T13:50:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 374ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iyko91vbvr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718430000,"log":"[2026-02-17T13:09:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6150515954059583000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-le2a4m8x9u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718431000,"log":"[2026-02-17T13:10:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8383244193766821000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9pvt2gmvh4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718432000,"log":"[2026-02-17T14:23:02Z DEBUG sqlx::query] SELECT FROM streams returned 19 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9praad5soy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718433000,"log":"[2026-02-17T12:45:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u942rtpjyk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718434000,"log":"[2026-02-17T14:27:45Z DEBUG sqlx::query] DELETE FROM alerts returned 769 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tr41bn9ddt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718435000,"log":"[2026-02-17T13:21:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3802459858991975400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2aijaqagvv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718436000,"log":"[2026-02-17T13:04:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mklluxhqjv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718437000,"log":"[2026-02-17T15:12:50Z DEBUG metrics::reporter] recorded metric active_connections value=6084","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yczuvp7v46","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718438000,"log":"[2026-02-17T13:30:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5130334618283776000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b0tkk64hge","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718439000,"log":"[2026-02-17T13:49:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 226ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8col1q88ix","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718440000,"log":"[2026-02-17T13:48:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 211ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-blbwhb8r57","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718441000,"log":"[2026-02-17T14:44:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=jlw2nj8p","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8tc6xyv83g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718442000,"log":"[2026-02-17T13:31:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5355001728317768000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-onef410767","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718443000,"log":"[2026-02-17T13:54:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 357ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6kk6y92j1f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718444000,"log":"[2026-02-17T15:05:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=1814","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-977wjvdfa9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718445000,"log":"[2026-02-17T14:41:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=hor36dty","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1ukeyvb2c7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718446000,"log":"[2026-02-17T13:07:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fhksbfdy79","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718447000,"log":"[2026-02-17T13:53:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 349ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c5jhd6qa2x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718448000,"log":"[2026-02-17T14:34:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=xvpwrlim","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7nd9bma78u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718449000,"log":"[2026-02-17T14:16:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 843 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7yis30x5gb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718450000,"log":"[2026-02-17T14:37:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=z6costhi","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hw2zhktg2w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718451000,"log":"[2026-02-17T13:25:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5880363003959439000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h4p7v13zmi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718452000,"log":"[2026-02-17T15:19:28Z TRACE cache::redis] cache delete key=org:config:3378","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zugizkz851","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718453000,"log":"[2026-02-17T13:15:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7100584320317145000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8fsqk6ga31","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718454000,"log":"[2026-02-17T12:58:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dszha4sytk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718455000,"log":"[2026-02-17T13:01:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ybv5s54de","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718456000,"log":"[2026-02-17T13:56:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 132ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gpc7yq65tu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718457000,"log":"[2026-02-17T13:48:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 100ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-33giizvlc3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718458000,"log":"[2026-02-17T13:19:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7515310383619378000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n96nx0zno7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718459000,"log":"[2026-02-17T13:37:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5720544185212406000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-baycw5mjl5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718460000,"log":"[2026-02-17T13:20:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1435217088816336100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h1d85pll0q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718461000,"log":"[2026-02-17T14:09:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 604 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3motosl6yy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718462000,"log":"[2026-02-17T15:05:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8134","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jue360fee5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718463000,"log":"[2026-02-17T15:15:55Z TRACE cache::redis] cache miss key=query:result:4615","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x96enuy4gz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718464000,"log":"[2026-02-17T15:03:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tbd8m32dx2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718465000,"log":"[2026-02-17T14:12:30Z DEBUG sqlx::query] DELETE FROM alerts returned 266 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r86d4dm4xd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718466000,"log":"[2026-02-17T13:31:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7527277296546817000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wi042b2tjq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718467000,"log":"[2026-02-17T13:47:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 399ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-974lpn9pot","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718468000,"log":"[2026-02-17T14:30:52Z DEBUG sqlx::query] SELECT FROM streams returned 273 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5r7j7vi9bx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718469000,"log":"[2026-02-17T14:15:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 117 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e22uh2gxkx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718470000,"log":"[2026-02-17T14:24:52Z DEBUG sqlx::query] SELECT FROM streams returned 737 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-typ0wpyzhc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718471000,"log":"[2026-02-17T14:03:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 380ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-11eia0f1hj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718472000,"log":"[2026-02-17T14:25:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 568 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tkyl19url9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718473000,"log":"[2026-02-17T13:49:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 251ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2ovjoso1kz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718474000,"log":"[2026-02-17T13:52:20Z INFO actix_web::middleware::logger] POST /api/_search 500 184ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g7fjcp31q5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718475000,"log":"[2026-02-17T13:15:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7605860331683940000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wqw0l9wjod","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718476000,"log":"[2026-02-17T15:20:54Z TRACE cache::redis] cache hit key=stream:schema:3233","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yxy5n7ru92","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718477000,"log":"[2026-02-17T13:05:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7rbj3mdsh7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718478000,"log":"[2026-02-17T15:12:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7722","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-166kvvpblo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718479000,"log":"[2026-02-17T13:00:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7uqda80o1w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718480000,"log":"[2026-02-17T15:08:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4057","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3sqxcizb9i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718481000,"log":"[2026-02-17T15:21:46Z TRACE cache::redis] cache delete key=auth:token:7759","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-prqm1yau9a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718482000,"log":"[2026-02-17T13:31:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5367870867321638000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kz5338j3hd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718483000,"log":"[2026-02-17T14:05:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 281ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t6jfk2gkqh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718484000,"log":"[2026-02-17T13:49:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 499ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gv5u055yf3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718485000,"log":"[2026-02-17T14:26:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 936 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vkqd2endyt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718486000,"log":"[2026-02-17T13:00:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-765m6hqyz9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718487000,"log":"[2026-02-17T14:07:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 823 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p9mgyo1pp8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718488000,"log":"[2026-02-17T15:17:56Z TRACE cache::redis] cache get key=auth:token:4383","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-imqfbj2w9d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718489000,"log":"[2026-02-17T14:13:08Z DEBUG sqlx::query] INSERT FROM users returned 942 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-04kl5h0vsa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718490000,"log":"[2026-02-17T14:49:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s66xp1l2qh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718491000,"log":"[2026-02-17T12:55:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-23yyojcbn8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718492000,"log":"[2026-02-17T15:04:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pvvt7foswi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718493000,"log":"[2026-02-17T14:05:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 96ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uhz9wysu79","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718494000,"log":"[2026-02-17T14:41:57Z INFO tonic::transport::server] grpc ingester connected client_id=rc2dpswd","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hxfnr6oc6l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718495000,"log":"[2026-02-17T14:45:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=j6necohz","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e7wem9g7ex","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718496000,"log":"[2026-02-17T14:07:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 608 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-csfibcvybp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718497000,"log":"[2026-02-17T15:18:35Z TRACE cache::redis] cache expire key=query:result:8402","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hmsyo9xvm0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718498000,"log":"[2026-02-17T14:55:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w84eo9uftc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718499000,"log":"[2026-02-17T15:08:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=9835","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-raid6cui8q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718500000,"log":"[2026-02-17T14:26:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 189 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ya8eqi4vgq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718501000,"log":"[2026-02-17T14:26:57Z DEBUG sqlx::query] SELECT FROM streams returned 603 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9onsztubm9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718502000,"log":"[2026-02-17T14:01:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 109ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6cqzhrirkp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718503000,"log":"[2026-02-17T13:27:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3303454617113221600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5pirg0whwg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718504000,"log":"[2026-02-17T14:59:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bay6xq02d4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718505000,"log":"[2026-02-17T13:33:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7132103716887928000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ud1p2cfqlb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718506000,"log":"[2026-02-17T14:06:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 410ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e6mxj8841m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718507000,"log":"[2026-02-17T14:09:53Z DEBUG sqlx::query] INSERT FROM users returned 351 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5vqqnw7joz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718508000,"log":"[2026-02-17T14:30:53Z DEBUG sqlx::query] INSERT FROM users returned 987 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vcizfyb3k0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718509000,"log":"[2026-02-17T14:00:36Z INFO actix_web::middleware::logger] POST /api/_bulk 503 216ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o7pja3gw3g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718510000,"log":"[2026-02-17T14:15:12Z DEBUG sqlx::query] SELECT FROM streams returned 477 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3vxm1ijlzz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718511000,"log":"[2026-02-17T13:28:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/105393954971966240.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xkxqj1x4v3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718512000,"log":"[2026-02-17T13:53:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 352ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dp9pu6o2o6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718513000,"log":"[2026-02-17T13:24:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2915305356135804400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vbo3t83s36","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718514000,"log":"[2026-02-17T14:55:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-28a73616kc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718515000,"log":"[2026-02-17T14:26:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 423 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-spsiltvk66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718516000,"log":"[2026-02-17T14:59:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3x0viak4b7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718517000,"log":"[2026-02-17T14:18:53Z DEBUG sqlx::query] INSERT FROM users returned 70 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xln0q9v8fw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718518000,"log":"[2026-02-17T12:35:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rhvzt5sr11","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718519000,"log":"[2026-02-17T12:56:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lmchtco61m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718520000,"log":"[2026-02-17T13:57:10Z INFO actix_web::middleware::logger] PUT /api/_search 200 430ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c6h2pu3ce5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718521000,"log":"[2026-02-17T13:49:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-17i7azjdnv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718522000,"log":"[2026-02-17T12:58:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bi06f448a7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718523000,"log":"[2026-02-17T12:54:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7c733c75sj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718524000,"log":"[2026-02-17T14:18:08Z DEBUG sqlx::query] INSERT FROM users returned 507 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s3mq11l26c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718525000,"log":"[2026-02-17T13:43:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 201 8ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ldn7zyf132","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718526000,"log":"[2026-02-17T14:00:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 330ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2mvluggv23","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718527000,"log":"[2026-02-17T14:53:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jp3hv6m5lx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718528000,"log":"[2026-02-17T13:40:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4035766861231238000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1b0r7hin0q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718529000,"log":"[2026-02-17T13:36:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7196701181140692000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6m7ng8dpao","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718530000,"log":"[2026-02-17T12:38:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dhi8eiu9zh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718531000,"log":"[2026-02-17T12:44:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f5gwba6klp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718532000,"log":"[2026-02-17T15:03:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rfmz15rc27","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718533000,"log":"[2026-02-17T15:04:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ddax4ujpkd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718534000,"log":"[2026-02-17T13:50:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 496ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jt7tde2g6f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718535000,"log":"[2026-02-17T15:08:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2638","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-plnyfl76ek","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718536000,"log":"[2026-02-17T14:54:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gd9gce7pz3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718537000,"log":"[2026-02-17T14:25:10Z DEBUG sqlx::query] DELETE FROM alerts returned 573 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ewh6vhqf31","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718538000,"log":"[2026-02-17T13:32:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6614984318789467000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hebke8655u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718539000,"log":"[2026-02-17T14:55:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2qpltc1crx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718540000,"log":"[2026-02-17T15:13:47Z TRACE cache::redis] cache expire key=user:session:5550","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lw88pds5qm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718541000,"log":"[2026-02-17T14:31:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 600 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ow3c3bqab5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718542000,"log":"[2026-02-17T14:48:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=ozq2upmy","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a0phx3j0d9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718543000,"log":"[2026-02-17T13:50:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 318ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-izlx5quabn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718544000,"log":"[2026-02-17T14:32:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2epc03v9","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oxf2ndu0kz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718545000,"log":"[2026-02-17T14:59:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n6ju6lzpuj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718546000,"log":"[2026-02-17T12:56:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yw7tutwv8q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718547000,"log":"[2026-02-17T13:05:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wmxb71d84a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718548000,"log":"[2026-02-17T13:46:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 74ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7cwp6d2fkr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718549000,"log":"[2026-02-17T12:47:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i8za8bb6pp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718550000,"log":"[2026-02-17T13:45:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 196ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8r4zhrwpw5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718551000,"log":"[2026-02-17T13:08:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2985468045704860000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8k45cjgahd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718552000,"log":"[2026-02-17T14:10:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 447 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1pwnb4gbxy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718553000,"log":"[2026-02-17T14:40:03Z INFO tonic::transport::server] grpc querier disconnected client_id=zslo412t","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-408fzttabm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718554000,"log":"[2026-02-17T13:42:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 445ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k3dj1d9x4i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718555000,"log":"[2026-02-17T15:21:44Z TRACE cache::redis] cache get key=stream:schema:5141","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7zw9svdgt3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718556000,"log":"[2026-02-17T12:36:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2dejjg8anv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718557000,"log":"[2026-02-17T12:55:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oz3cojtrqo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718558000,"log":"[2026-02-17T14:39:08Z INFO tonic::transport::server] grpc querier disconnected client_id=8sjsixi1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ockpooncui","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718559000,"log":"[2026-02-17T12:58:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-611pzork2a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718560000,"log":"[2026-02-17T14:15:08Z DEBUG sqlx::query] INSERT FROM users returned 774 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u0p3kbbr91","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718561000,"log":"[2026-02-17T14:55:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w0pxb1b9gn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718562000,"log":"[2026-02-17T14:37:53Z INFO tonic::transport::server] grpc querier disconnected client_id=hcze08bu","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qajuz7zxmx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718563000,"log":"[2026-02-17T14:25:50Z DEBUG sqlx::query] DELETE FROM alerts returned 770 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8hqgcxx8by","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718564000,"log":"[2026-02-17T14:37:27Z INFO tonic::transport::server] grpc ingester connected client_id=50n0nevk","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n4g4320eus","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718565000,"log":"[2026-02-17T14:03:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 223ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-426jukw49s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718566000,"log":"[2026-02-17T14:28:50Z DEBUG sqlx::query] DELETE FROM alerts returned 207 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-swar8fegyd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718567000,"log":"[2026-02-17T13:42:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 412ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h8e59mcsbd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718568000,"log":"[2026-02-17T14:54:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o0wtgefm2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718569000,"log":"[2026-02-17T13:21:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1846594024989532000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lalu46f3h7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718570000,"log":"[2026-02-17T12:54:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cnt3bx79ny","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718571000,"log":"[2026-02-17T15:10:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=7057","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-my1cracv2r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718572000,"log":"[2026-02-17T14:29:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 256 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rhisq84h78","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718573000,"log":"[2026-02-17T13:46:00Z INFO actix_web::middleware::logger] POST /api/_search 403 174ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bqptzon7jw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718574000,"log":"[2026-02-17T13:03:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z0vi8qrpkz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718575000,"log":"[2026-02-17T12:59:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gajvu6yc52","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718576000,"log":"[2026-02-17T13:09:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7468221626281810000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ed027wfd6w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718577000,"log":"[2026-02-17T12:36:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uv1y2c141d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718578000,"log":"[2026-02-17T13:00:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2djad76zua","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718579000,"log":"[2026-02-17T14:50:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-07q78czgaz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718580000,"log":"[2026-02-17T13:11:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4010668172186709000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v1uratpygj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718581000,"log":"[2026-02-17T12:53:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0m2awh07g4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718582000,"log":"[2026-02-17T13:45:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 281ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0twi5ljgdg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718583000,"log":"[2026-02-17T13:07:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tdne15wh0z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718584000,"log":"[2026-02-17T13:55:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 344ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v3aw1src9a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718585000,"log":"[2026-02-17T14:39:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=927b54ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jkfqma9q0q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718586000,"log":"[2026-02-17T12:58:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-enmz6neryf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718587000,"log":"[2026-02-17T14:19:08Z DEBUG sqlx::query] INSERT FROM users returned 126 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-htnn9mt2fy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718588000,"log":"[2026-02-17T12:43:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iyc1ybiyda","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718589000,"log":"[2026-02-17T14:17:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 625 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hx2jesrc9o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718590000,"log":"[2026-02-17T14:07:13Z DEBUG sqlx::query] INSERT FROM users returned 430 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ykqq4lqzvl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718591000,"log":"[2026-02-17T13:40:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6894709861591097000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ta73oili4c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718592000,"log":"[2026-02-17T13:45:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 434ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4do7rpjjnw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718593000,"log":"[2026-02-17T13:12:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6445649346557463000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g524mntvy0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718594000,"log":"[2026-02-17T12:59:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5phub5mhjk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718595000,"log":"[2026-02-17T15:15:44Z TRACE cache::redis] cache get key=stream:schema:4350","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5ps1eeqo66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718596000,"log":"[2026-02-17T13:08:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h5b8y4302d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718597000,"log":"[2026-02-17T14:08:12Z DEBUG sqlx::query] SELECT FROM streams returned 374 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sax6i684mc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718598000,"log":"[2026-02-17T13:45:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 249ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7i2qldb8nv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718599000,"log":"[2026-02-17T14:13:43Z DEBUG sqlx::query] INSERT FROM users returned 195 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c6zhiohh56","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718600000,"log":"[2026-02-17T12:54:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cnd6pg7qje","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718601000,"log":"[2026-02-17T14:16:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 794 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ybhllhqp04","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718602000,"log":"[2026-02-17T13:02:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ndl4d5fp5w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718603000,"log":"[2026-02-17T13:32:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7831059021484355000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s9xzndgogv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718604000,"log":"[2026-02-17T15:07:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7939","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-44fx3h6iz5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718605000,"log":"[2026-02-17T14:48:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4e3r1h3hdy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718606000,"log":"[2026-02-17T13:08:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jtzw54k9am","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718607000,"log":"[2026-02-17T14:51:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f0ghsjjqbg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718608000,"log":"[2026-02-17T14:15:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 472 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-js60uio1mm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718609000,"log":"[2026-02-17T14:31:54Z INFO tonic::transport::server] grpc router request_received client_id=93xnlajp","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ir275umauw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718610000,"log":"[2026-02-17T14:56:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ghuadaycux","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718611000,"log":"[2026-02-17T14:43:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=axoqntre","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3efu2n47pu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718612000,"log":"[2026-02-17T13:58:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 403 320ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hys9htkkn8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718613000,"log":"[2026-02-17T13:08:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u1ldfee9xj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718614000,"log":"[2026-02-17T13:27:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2823748767748917000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dirx36btg6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718615000,"log":"[2026-02-17T14:14:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 597 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-moi9q5u8fb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718616000,"log":"[2026-02-17T13:18:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2285177407157135000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fqxa4s51z0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718617000,"log":"[2026-02-17T13:08:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c47csk062v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718618000,"log":"[2026-02-17T15:20:34Z TRACE cache::redis] cache delete key=stream:schema:9665","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5a70r0lswo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718619000,"log":"[2026-02-17T12:54:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tozkvik0a5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718620000,"log":"[2026-02-17T15:16:19Z TRACE cache::redis] cache miss key=stream:schema:8988","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1mmzaeqy2l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718621000,"log":"[2026-02-17T15:08:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9573","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-09symox8xk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718622000,"log":"[2026-02-17T13:19:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5584776172040733000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6zf14t7k1x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718623000,"log":"[2026-02-17T14:19:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 242 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-axtijcfne5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718624000,"log":"[2026-02-17T12:54:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yz2p0pf445","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718625000,"log":"[2026-02-17T13:27:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9601209070729746000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kc3jn1y0dx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718626000,"log":"[2026-02-17T12:43:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oj1xw2vcqh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718627000,"log":"[2026-02-17T13:39:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4212369489586636000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9fh2uhv5rg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718628000,"log":"[2026-02-17T14:04:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 92ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-62fm3qhv9t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718629000,"log":"[2026-02-17T12:56:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q8zi5qgfhl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718630000,"log":"[2026-02-17T13:42:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 195ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-86p7954qat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718631000,"log":"[2026-02-17T15:14:45Z TRACE cache::redis] cache set key=query:result:6631","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wb2bpj8j3e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718632000,"log":"[2026-02-17T13:27:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7054992718426811000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-87mr60tqcv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718633000,"log":"[2026-02-17T13:21:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3948503570805735000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bj4gxcsura","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718634000,"log":"[2026-02-17T13:40:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1019891613198529200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f98dbyqmug","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718635000,"log":"[2026-02-17T13:08:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8987639842561610000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kyi4ru7mvl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718636000,"log":"[2026-02-17T14:52:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eixt4nrgu6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718637000,"log":"[2026-02-17T13:58:20Z INFO actix_web::middleware::logger] POST /api/_search 500 410ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-35f62zfbrg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718638000,"log":"[2026-02-17T13:01:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-asjw9rj2yh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718639000,"log":"[2026-02-17T14:22:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 349 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u2s1bifm1q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718640000,"log":"[2026-02-17T13:48:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d8iuwkbcxc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718641000,"log":"[2026-02-17T14:05:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 316ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dph8si2ie5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718642000,"log":"[2026-02-17T12:56:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gd0xc18y8a","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718643000,"log":"[2026-02-17T14:42:34Z INFO tonic::transport::server] grpc router request_received client_id=e38zzu0z","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rg5zfji4vq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718644000,"log":"[2026-02-17T14:25:57Z DEBUG sqlx::query] SELECT FROM streams returned 597 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-59n0f33pur","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718645000,"log":"[2026-02-17T13:37:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8196534961439914000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-98yrlcrrsr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718646000,"log":"[2026-02-17T13:59:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 58ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-672jmgz1s1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718647000,"log":"[2026-02-17T13:50:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 218ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a0hjcwl8sf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718648000,"log":"[2026-02-17T12:37:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-51xmajgevs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718649000,"log":"[2026-02-17T14:07:00Z DEBUG sqlx::query] DELETE FROM alerts returned 560 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tinbzw2qvc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718650000,"log":"[2026-02-17T13:55:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 459ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xqk68384yu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718651000,"log":"[2026-02-17T15:07:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2483","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rkr6efrbuc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718652000,"log":"[2026-02-17T14:33:33Z INFO tonic::transport::server] grpc querier disconnected client_id=0ydo7n0t","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d0grf1clx8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718653000,"log":"[2026-02-17T13:05:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b9gczo0deb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718654000,"log":"[2026-02-17T12:46:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6n0sj83o6i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718655000,"log":"[2026-02-17T13:44:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 468ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4n07qmxo9g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718656000,"log":"[2026-02-17T14:36:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=fziy3arr","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tpepwe833r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718657000,"log":"[2026-02-17T13:49:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 311ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jabuftpso4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718658000,"log":"[2026-02-17T14:02:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 2ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xlx11frttu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718659000,"log":"[2026-02-17T14:18:33Z DEBUG sqlx::query] INSERT FROM users returned 767 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c7uy0wsfze","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718660000,"log":"[2026-02-17T14:29:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 774 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-teun9dcpeb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718661000,"log":"[2026-02-17T13:33:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7990127835367924000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-31ucsool0e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718662000,"log":"[2026-02-17T14:17:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 367 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g3klaqlsl7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718663000,"log":"[2026-02-17T13:24:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1090211326051886500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2gtqvvbq3z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718664000,"log":"[2026-02-17T13:50:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 41ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rq4svylxb6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718665000,"log":"[2026-02-17T14:11:42Z DEBUG sqlx::query] SELECT FROM streams returned 973 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7mww5ve1n0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718666000,"log":"[2026-02-17T14:32:42Z INFO tonic::transport::server] grpc ingester connected client_id=5i4rsjo0","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dm3o6leqe2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718667000,"log":"[2026-02-17T14:55:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c5j159axn7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718668000,"log":"[2026-02-17T13:55:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 307ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nuwvue430l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718669000,"log":"[2026-02-17T12:58:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vglbxdgsg8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718670000,"log":"[2026-02-17T13:01:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8sxso90xng","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718671000,"log":"[2026-02-17T13:54:40Z INFO actix_web::middleware::logger] POST /api/_search 400 340ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x5c63a3sls","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718672000,"log":"[2026-02-17T12:54:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oalmcg53h5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718673000,"log":"[2026-02-17T15:12:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=8210","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7krs2xvxpq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718674000,"log":"[2026-02-17T13:44:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 244ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kf4d9gjqii","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718675000,"log":"[2026-02-17T13:07:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s7c6tm9759","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718676000,"log":"[2026-02-17T15:11:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1818","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mwkvcl3t7j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718677000,"log":"[2026-02-17T13:14:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/711886110184043500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-usy25biggu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718678000,"log":"[2026-02-17T13:52:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 308ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jp4cb4yifc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718679000,"log":"[2026-02-17T15:20:27Z TRACE cache::redis] cache set key=user:session:1399","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nh4vf1miql","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718680000,"log":"[2026-02-17T14:00:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 474ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-78oj08ehat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718681000,"log":"[2026-02-17T15:08:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6837","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oq0ba7pg7u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718682000,"log":"[2026-02-17T13:17:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9484209455056990000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e26dck1r7j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718683000,"log":"[2026-02-17T14:01:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 253ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-861vqk4a34","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718684000,"log":"[2026-02-17T13:05:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kfv886hfa1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718685000,"log":"[2026-02-17T14:07:48Z DEBUG sqlx::query] INSERT FROM users returned 425 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oqt549sjh7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718686000,"log":"[2026-02-17T12:35:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p7mn0gs7ds","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718687000,"log":"[2026-02-17T15:01:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-88j8s8nfvk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718688000,"log":"[2026-02-17T14:45:12Z INFO tonic::transport::server] grpc ingester connected client_id=lwzh7o5j","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8s91b3t4jr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718689000,"log":"[2026-02-17T14:29:57Z DEBUG sqlx::query] SELECT FROM streams returned 695 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ief2dbdr2z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718690000,"log":"[2026-02-17T13:03:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j5auignca4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718691000,"log":"[2026-02-17T13:59:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 503 68ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-su9n0nj3l8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718692000,"log":"[2026-02-17T14:15:42Z DEBUG sqlx::query] SELECT FROM streams returned 131 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xdxw50b4pu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718693000,"log":"[2026-02-17T15:05:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4184","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fju90lhawm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718694000,"log":"[2026-02-17T14:10:02Z DEBUG sqlx::query] SELECT FROM streams returned 580 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l1ku8xpnk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718695000,"log":"[2026-02-17T14:32:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=z8l4u1c6","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yg6mrk2be5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718696000,"log":"[2026-02-17T14:09:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 31 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hklmcb96cu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718697000,"log":"[2026-02-17T12:55:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wua5n8dx00","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718698000,"log":"[2026-02-17T13:22:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1203627944708050000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b87mr1phyp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718699000,"log":"[2026-02-17T12:56:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rasptptz4k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718700000,"log":"[2026-02-17T15:06:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=3925","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-46lsakrczc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718701000,"log":"[2026-02-17T14:00:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 27ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vy1o1vlpuc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718702000,"log":"[2026-02-17T14:27:07Z DEBUG sqlx::query] SELECT FROM streams returned 516 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1078kpbyw2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718703000,"log":"[2026-02-17T14:51:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7gjjwrh710","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718704000,"log":"[2026-02-17T12:59:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j0h43jv7es","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718705000,"log":"[2026-02-17T14:12:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 208 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8grs5w63yj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718706000,"log":"[2026-02-17T13:39:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/689700259434207200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eva3h18yx2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718707000,"log":"[2026-02-17T14:48:12Z INFO tonic::transport::server] grpc ingester connected client_id=oo7g2f0u","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3kyifvemgd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718708000,"log":"[2026-02-17T13:29:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7071244896639495000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w9epob4tqj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718709000,"log":"[2026-02-17T13:10:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8071133350870236000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2szh57ox6w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718710000,"log":"[2026-02-17T14:27:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 499 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pp5jcobs7s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718711000,"log":"[2026-02-17T13:45:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 3ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vhq4xltk78","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718712000,"log":"[2026-02-17T14:38:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=e2v3yebm","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kpd5y6xg3z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718713000,"log":"[2026-02-17T13:19:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6682595332023944000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x64m5166ei","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718714000,"log":"[2026-02-17T12:50:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-asgqnh3if2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718715000,"log":"[2026-02-17T14:30:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 940 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b7lq2l2kap","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718716000,"log":"[2026-02-17T15:00:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6tmz0zbjvf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718717000,"log":"[2026-02-17T13:06:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cribupt9zt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718718000,"log":"[2026-02-17T12:51:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wd8l4e3053","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718719000,"log":"[2026-02-17T14:04:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 472ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-slvp989u8f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718720000,"log":"[2026-02-17T14:11:52Z DEBUG sqlx::query] SELECT FROM streams returned 630 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l85grsaj7k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718721000,"log":"[2026-02-17T13:04:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-59e5zr6ezi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718722000,"log":"[2026-02-17T13:35:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3885292244840863000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-78gipdltb4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718723000,"log":"[2026-02-17T15:18:50Z TRACE cache::redis] cache get key=query:result:9057","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-im2vraip5c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718724000,"log":"[2026-02-17T12:51:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s5qub99044","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718725000,"log":"[2026-02-17T14:56:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1x6ko11z29","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718726000,"log":"[2026-02-17T13:18:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5971247419933206000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qdq6z41em3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718727000,"log":"[2026-02-17T13:37:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8421592688373410000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f2isrgqbsd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718728000,"log":"[2026-02-17T13:43:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 15ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r69ukms3xp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718729000,"log":"[2026-02-17T14:17:17Z DEBUG sqlx::query] SELECT FROM streams returned 672 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qwf1zv412m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718730000,"log":"[2026-02-17T12:51:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lhet880jxm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718731000,"log":"[2026-02-17T13:52:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 241ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xcylpicat2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718732000,"log":"[2026-02-17T12:45:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x1112e7pem","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718733000,"log":"[2026-02-17T13:01:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tsjbtpngck","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718734000,"log":"[2026-02-17T14:05:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 380ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lw9d1araw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718735000,"log":"[2026-02-17T15:15:32Z TRACE cache::redis] cache get key=user:session:9664","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rkgy9odlme","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718736000,"log":"[2026-02-17T12:47:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ahkm4rcqfb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718737000,"log":"[2026-02-17T14:10:03Z DEBUG sqlx::query] INSERT FROM users returned 400 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j22fsemlfn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718738000,"log":"[2026-02-17T14:40:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=1mhmbwx9","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m8kyg6l266","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718739000,"log":"[2026-02-17T14:46:34Z INFO tonic::transport::server] grpc router request_received client_id=04pa4jzt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-244mddwd0w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718740000,"log":"[2026-02-17T14:53:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j206d80lps","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718741000,"log":"[2026-02-17T15:13:05Z DEBUG metrics::reporter] recorded metric active_connections value=4921","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uwtjrqeqgj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718742000,"log":"[2026-02-17T14:54:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ofm1aojomv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718743000,"log":"[2026-02-17T15:05:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1886","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2uk99o5ium","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718744000,"log":"[2026-02-17T15:01:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kpm87w4idd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718745000,"log":"[2026-02-17T14:29:23Z DEBUG sqlx::query] INSERT FROM users returned 977 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g7tx37xn5o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718746000,"log":"[2026-02-17T15:08:44Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7014","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-myyufyzt7n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718747000,"log":"[2026-02-17T14:56:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pjjz8lriqc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718748000,"log":"[2026-02-17T14:55:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b6cmus5yze","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718749000,"log":"[2026-02-17T14:36:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pms3jofe","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6z3jh9z55p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718750000,"log":"[2026-02-17T13:41:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/56332963719372130.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r9cck2ff4k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718751000,"log":"[2026-02-17T15:09:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9554","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fcqwvg1zcu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718752000,"log":"[2026-02-17T14:12:27Z DEBUG sqlx::query] SELECT FROM streams returned 483 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vvayrd65i2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718753000,"log":"[2026-02-17T14:21:35Z DEBUG sqlx::query] DELETE FROM alerts returned 342 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-14vksz3fhz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718754000,"log":"[2026-02-17T14:38:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=z3mndm61","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tvc59790ct","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718755000,"log":"[2026-02-17T14:00:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 268ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o1tdwcuzxq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718756000,"log":"[2026-02-17T12:49:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qso93cttbd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718757000,"log":"[2026-02-17T14:01:05Z INFO actix_web::middleware::logger] GET /api/_search 201 316ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x4szde1ocx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718758000,"log":"[2026-02-17T14:15:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 988 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hzl4qscnfu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718759000,"log":"[2026-02-17T13:58:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-769oozcbn3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718760000,"log":"[2026-02-17T13:56:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 421ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wk3uygr5dj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718761000,"log":"[2026-02-17T12:50:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ni7g0oxkts","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718762000,"log":"[2026-02-17T14:14:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 340 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ossnun8j15","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718763000,"log":"[2026-02-17T14:06:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 297ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y5dsg92xv8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718764000,"log":"[2026-02-17T14:57:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5ojzcjqum3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718765000,"log":"[2026-02-17T13:58:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 442ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pwz7vjvbc6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718766000,"log":"[2026-02-17T13:29:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4858629935446621000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ful4d7cct5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718767000,"log":"[2026-02-17T13:59:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 497ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c6yhj39m4e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718768000,"log":"[2026-02-17T13:15:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5078887348850915000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-evlc55ukol","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718769000,"log":"[2026-02-17T14:29:40Z DEBUG sqlx::query] DELETE FROM alerts returned 19 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tq3ojka1wf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718770000,"log":"[2026-02-17T13:52:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 467ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yd0zrjopsy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718771000,"log":"[2026-02-17T14:56:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i8709n25fy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718772000,"log":"[2026-02-17T12:59:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ch06t2hgcd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718773000,"log":"[2026-02-17T14:01:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 334ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1nmnkfhksv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718774000,"log":"[2026-02-17T12:46:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-95enjg22v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718775000,"log":"[2026-02-17T13:20:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4960750500975129000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-agzcrqvbjh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718776000,"log":"[2026-02-17T12:42:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gcn0vfa2de","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718777000,"log":"[2026-02-17T14:27:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 112 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0i5v8f45xj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718778000,"log":"[2026-02-17T12:57:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-chiy8bw6tn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718779000,"log":"[2026-02-17T12:44:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u8aoptwuw6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718780000,"log":"[2026-02-17T13:17:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5974795167579481000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7bbrw21zu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718781000,"log":"[2026-02-17T13:55:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 368ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-29gdclq1m4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718782000,"log":"[2026-02-17T15:16:31Z TRACE cache::redis] cache miss key=auth:token:1071","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d6tqa0vf2c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718783000,"log":"[2026-02-17T15:18:36Z TRACE cache::redis] cache hit key=auth:token:5022","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iifin1a9dz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718784000,"log":"[2026-02-17T15:03:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rkp87ei71o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718785000,"log":"[2026-02-17T12:36:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ssdorv7ao9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718786000,"log":"[2026-02-17T14:39:58Z INFO tonic::transport::server] grpc querier disconnected client_id=punm1m27","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l5olfy4go8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718787000,"log":"[2026-02-17T14:28:43Z DEBUG sqlx::query] INSERT FROM users returned 717 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kssjerbhzh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718788000,"log":"[2026-02-17T14:17:02Z DEBUG sqlx::query] SELECT FROM streams returned 322 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5fwlszjfo4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718789000,"log":"[2026-02-17T13:41:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 122ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pf5l4pg48v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718790000,"log":"[2026-02-17T13:29:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/938482082453115900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hs60teptmm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718791000,"log":"[2026-02-17T13:54:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 94ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-epnn5m9ol5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718792000,"log":"[2026-02-17T13:56:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 386ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w52obw3u09","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718793000,"log":"[2026-02-17T13:52:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 443ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tar3shfrxe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718794000,"log":"[2026-02-17T12:40:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-te7ixbozcy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718795000,"log":"[2026-02-17T13:56:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 200ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dgua8zvlmv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718796000,"log":"[2026-02-17T12:45:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-32bvbwymu0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718797000,"log":"[2026-02-17T14:12:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 909 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eejh0li8yd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718798000,"log":"[2026-02-17T14:28:13Z DEBUG sqlx::query] INSERT FROM users returned 820 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e3fdgkhh3v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718799000,"log":"[2026-02-17T13:19:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/10233488588970996.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kvy9tkllzc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718800000,"log":"[2026-02-17T15:15:01Z TRACE cache::redis] cache miss key=auth:token:6574","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a4wtzbji1e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718801000,"log":"[2026-02-17T12:58:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aajr2qh26r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718802000,"log":"[2026-02-17T15:16:07Z TRACE cache::redis] cache miss key=user:session:428","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-deqcit3zgu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718803000,"log":"[2026-02-17T13:55:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 296ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jp516rubdl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718804000,"log":"[2026-02-17T12:45:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-554ed8yheb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718805000,"log":"[2026-02-17T13:21:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9611464700197830000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lgnfgvwr3x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718806000,"log":"[2026-02-17T14:15:33Z DEBUG sqlx::query] INSERT FROM users returned 594 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jpryh4r4qo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718807000,"log":"[2026-02-17T14:18:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 322 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rk6lvdjo8l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718808000,"log":"[2026-02-17T13:30:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4092709845107196400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yqnj6r14xa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718809000,"log":"[2026-02-17T13:56:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 275ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oumu6wymyu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718810000,"log":"[2026-02-17T15:02:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-155mfsqo7x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718811000,"log":"[2026-02-17T14:55:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l1d4mynfw7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718812000,"log":"[2026-02-17T13:44:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 147ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xh956pl3oi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718813000,"log":"[2026-02-17T15:15:53Z TRACE cache::redis] cache expire key=org:config:2313","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v58wvanbud","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718814000,"log":"[2026-02-17T15:03:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nzlks8sflr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718815000,"log":"[2026-02-17T13:17:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7700359517524551000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bjphuxeid9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718816000,"log":"[2026-02-17T14:21:10Z DEBUG sqlx::query] DELETE FROM alerts returned 912 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tt3lukyafe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718817000,"log":"[2026-02-17T12:54:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n8ivu6pm6j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718818000,"log":"[2026-02-17T14:59:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eg66yqp0ua","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718819000,"log":"[2026-02-17T12:38:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h86ps25f0m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718820000,"log":"[2026-02-17T13:09:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/186907778830773800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3xaq8uk6oc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718821000,"log":"[2026-02-17T12:51:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y69a0bt3bh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718822000,"log":"[2026-02-17T13:14:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6632899060410704000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wdikzevcuy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718823000,"log":"[2026-02-17T13:08:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/122043559331972420.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6rtwl2d8qv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718824000,"log":"[2026-02-17T14:14:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 20 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vgwecq1kod","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718825000,"log":"[2026-02-17T15:02:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-swenuudqo3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718826000,"log":"[2026-02-17T14:02:05Z INFO actix_web::middleware::logger] GET /api/_search 500 336ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8sxmp60or6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718827000,"log":"[2026-02-17T12:49:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-88cgnneqnj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718828000,"log":"[2026-02-17T13:02:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gzetjjv8g3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718829000,"log":"[2026-02-17T13:22:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3008262049604952600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n95zfuxsfu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718830000,"log":"[2026-02-17T12:39:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3zt3ibdytg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718831000,"log":"[2026-02-17T13:03:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1wrk862lbj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718832000,"log":"[2026-02-17T15:07:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6148","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p9q55m9a6n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718833000,"log":"[2026-02-17T14:46:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=o0bhgghu","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wocev1a93h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718834000,"log":"[2026-02-17T14:05:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 313ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g5rx73poaj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718835000,"log":"[2026-02-17T13:13:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2175895865185795000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5zgnax8zvk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718836000,"log":"[2026-02-17T15:08:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=8675","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r80ra3kyt6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718837000,"log":"[2026-02-17T13:46:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 398ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-05vbs1mdbg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718838000,"log":"[2026-02-17T12:48:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qujtky31ev","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718839000,"log":"[2026-02-17T15:15:57Z TRACE cache::redis] cache set key=user:session:7953","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nyvf83s15t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718840000,"log":"[2026-02-17T13:39:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6542596943350601000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pzqv4b4yev","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718841000,"log":"[2026-02-17T13:10:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6440381803328234000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rdfudfby6o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718842000,"log":"[2026-02-17T14:41:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2w1wd5kr","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bhlqh37s0b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718843000,"log":"[2026-02-17T14:25:33Z DEBUG sqlx::query] INSERT FROM users returned 16 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7qpol84l3m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718844000,"log":"[2026-02-17T13:00:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7xdjll18t6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718845000,"log":"[2026-02-17T13:20:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3081661232285981700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-alfjr8mwmc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718846000,"log":"[2026-02-17T13:50:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 340ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o0krz5rf2b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718847000,"log":"[2026-02-17T13:28:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9432372833542834000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iheqkr3kui","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718848000,"log":"[2026-02-17T13:38:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8812083844291488000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lp5ynfvsyk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718849000,"log":"[2026-02-17T14:01:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 312ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wr6yzrlwa8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718850000,"log":"[2026-02-17T14:34:18Z INFO tonic::transport::server] grpc querier disconnected client_id=kt3ughcc","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jl853oyfsf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718851000,"log":"[2026-02-17T13:42:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 494ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pa76zc09ta","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718852000,"log":"[2026-02-17T13:14:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4004073142781632500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-54dcc8uq0m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718853000,"log":"[2026-02-17T15:15:17Z TRACE cache::redis] cache expire key=user:session:1438","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-voh27qwslp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718854000,"log":"[2026-02-17T13:39:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2979832492857714700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e25kgw6rjv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718855000,"log":"[2026-02-17T14:19:15Z DEBUG sqlx::query] DELETE FROM alerts returned 909 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-70zaojuf6j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718856000,"log":"[2026-02-17T15:01:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aaod60sefd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718857000,"log":"[2026-02-17T13:30:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2620954520192997400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jcfopmvsi0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718858000,"log":"[2026-02-17T13:36:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3513464142262817000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fbwjr9c80k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718859000,"log":"[2026-02-17T14:07:50Z DEBUG sqlx::query] DELETE FROM alerts returned 815 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jngorioya6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718860000,"log":"[2026-02-17T13:56:05Z INFO actix_web::middleware::logger] GET /api/_search 500 92ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j1kbndle0f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718861000,"log":"[2026-02-17T14:50:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wch86fd3lf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718862000,"log":"[2026-02-17T14:40:52Z INFO tonic::transport::server] grpc ingester connected client_id=akiuysrn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hegf5jnqoc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718863000,"log":"[2026-02-17T13:24:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8713513141269393000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g02ogcyibp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718864000,"log":"[2026-02-17T13:08:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8z76nnh06n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718865000,"log":"[2026-02-17T14:18:57Z DEBUG sqlx::query] SELECT FROM streams returned 484 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-su28tgwa9b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718866000,"log":"[2026-02-17T14:58:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1i7nu9de6s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718867000,"log":"[2026-02-17T14:43:53Z INFO tonic::transport::server] grpc querier disconnected client_id=zwz8tgsx","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y34fg527az","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718868000,"log":"[2026-02-17T13:20:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9553166452263868000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vuh47kp14z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718869000,"log":"[2026-02-17T12:44:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fjg1wgiu6u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718870000,"log":"[2026-02-17T13:14:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8234334246458617000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yctw2fnr0s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718871000,"log":"[2026-02-17T13:17:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8507850000087144000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gktfmf477z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718872000,"log":"[2026-02-17T14:29:32Z DEBUG sqlx::query] SELECT FROM streams returned 169 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e9p0equyiq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718873000,"log":"[2026-02-17T14:35:07Z INFO tonic::transport::server] grpc ingester connected client_id=0wu1i2zt","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-idv7cmqv61","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718874000,"log":"[2026-02-17T13:51:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 235ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d7fee9d8lv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718875000,"log":"[2026-02-17T13:49:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 213ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5o6mgqmu6b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718876000,"log":"[2026-02-17T12:50:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gmbwuf4l57","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718877000,"log":"[2026-02-17T13:23:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8934334901377330000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6u4ptcgj3f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718878000,"log":"[2026-02-17T14:25:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 73 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fjcf0j90iq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718879000,"log":"[2026-02-17T13:21:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1809657500375254500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dd3fqynuzm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718880000,"log":"[2026-02-17T12:59:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hsnjtuc0s6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718881000,"log":"[2026-02-17T15:15:26Z TRACE cache::redis] cache get key=auth:token:3610","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-psd4tlzdxb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718882000,"log":"[2026-02-17T13:11:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6107150507195320000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v0964wrlvj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718883000,"log":"[2026-02-17T14:20:22Z DEBUG sqlx::query] SELECT FROM streams returned 195 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t00bjn8djt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718884000,"log":"[2026-02-17T13:15:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5114909147103153000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ig6feyui46","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718885000,"log":"[2026-02-17T15:07:50Z DEBUG metrics::reporter] recorded metric active_connections value=4732","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-srnb1a92zd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718886000,"log":"[2026-02-17T13:45:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 142ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n6khrr04jq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718887000,"log":"[2026-02-17T13:48:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qfv323wy3z","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718888000,"log":"[2026-02-17T15:05:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9525","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5lug17v9hp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718889000,"log":"[2026-02-17T14:58:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nwmbqzwbxf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718890000,"log":"[2026-02-17T12:50:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7xuy2svp91","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718891000,"log":"[2026-02-17T13:59:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 441ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sp6rr8r9js","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718892000,"log":"[2026-02-17T14:24:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 41 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a8mc525yl7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718893000,"log":"[2026-02-17T14:33:47Z INFO tonic::transport::server] grpc ingester connected client_id=b87ec4rn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-koae3uzeb6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718894000,"log":"[2026-02-17T13:21:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1690214331458996700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v4s09lj4ez","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718895000,"log":"[2026-02-17T13:58:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 395ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9hnztdv2sd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718896000,"log":"[2026-02-17T14:26:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 808 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hickjnzjkz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718897000,"log":"[2026-02-17T13:00:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dy9bpb47gq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718898000,"log":"[2026-02-17T14:05:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 21ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5n2tj15ez0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718899000,"log":"[2026-02-17T15:07:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8521","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ntbbp1o1da","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718900000,"log":"[2026-02-17T13:53:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 227ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y8neck5ke6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718901000,"log":"[2026-02-17T14:35:04Z INFO tonic::transport::server] grpc router request_received client_id=9aaab8gq","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4mf358ic80","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718902000,"log":"[2026-02-17T12:55:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0fi58cmy5r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718903000,"log":"[2026-02-17T15:00:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nke2dcwv3l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718904000,"log":"[2026-02-17T12:36:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t9qt1av6cl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718905000,"log":"[2026-02-17T14:41:58Z INFO tonic::transport::server] grpc querier disconnected client_id=4dxte2wk","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vsyliv1qt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718906000,"log":"[2026-02-17T12:48:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tc74j6w342","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718907000,"log":"[2026-02-17T14:11:23Z DEBUG sqlx::query] INSERT FROM users returned 935 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mz34g5t3fc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718908000,"log":"[2026-02-17T15:16:49Z TRACE cache::redis] cache miss key=stream:schema:8796","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7wjwdyrnni","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718909000,"log":"[2026-02-17T13:37:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/770857994907721000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y20hrukjil","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718910000,"log":"[2026-02-17T13:31:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1285035457134416000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zdjkwaicxh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718911000,"log":"[2026-02-17T12:59:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-69v3t5feq9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718912000,"log":"[2026-02-17T14:54:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u02ajh2fi2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718913000,"log":"[2026-02-17T13:03:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q81ndxs0r2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718914000,"log":"[2026-02-17T15:15:40Z TRACE cache::redis] cache delete key=query:result:6173","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ot239nj2z5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718915000,"log":"[2026-02-17T14:31:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=jtebgtb3","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dj0cx3wh1h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718916000,"log":"[2026-02-17T14:08:23Z DEBUG sqlx::query] INSERT FROM users returned 108 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oes81xatpa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718917000,"log":"[2026-02-17T12:41:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-aef6o385j9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718918000,"log":"[2026-02-17T13:32:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5118403504207021000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-grpdaar4j1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718919000,"log":"[2026-02-17T15:20:01Z TRACE cache::redis] cache miss key=auth:token:3978","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wszf2yl945","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718920000,"log":"[2026-02-17T13:38:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5355313428801500000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d2s6ga0nkn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718921000,"log":"[2026-02-17T14:32:03Z INFO tonic::transport::server] grpc querier disconnected client_id=ra93w038","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rf0nzn0ow1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718922000,"log":"[2026-02-17T14:09:33Z DEBUG sqlx::query] INSERT FROM users returned 651 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a1w3noruky","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718923000,"log":"[2026-02-17T12:58:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5pikvodzgn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718924000,"log":"[2026-02-17T15:10:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5135","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bi14bdz7wy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718925000,"log":"[2026-02-17T12:40:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-la9hpue5qr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718926000,"log":"[2026-02-17T14:50:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9eaabym05b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718927000,"log":"[2026-02-17T14:26:20Z DEBUG sqlx::query] DELETE FROM alerts returned 295 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0fs35ay0ag","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718928000,"log":"[2026-02-17T14:09:25Z DEBUG sqlx::query] DELETE FROM alerts returned 821 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-38hlb7mvha","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718929000,"log":"[2026-02-17T13:44:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 402ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wrb0hypn51","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718930000,"log":"[2026-02-17T15:07:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5516","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-664hwu8hua","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718931000,"log":"[2026-02-17T15:11:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=212","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vqiciys9sv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718932000,"log":"[2026-02-17T12:57:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rztr833je","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718933000,"log":"[2026-02-17T14:31:59Z INFO tonic::transport::server] grpc router request_received client_id=pge2lvqd","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p1r9a2x42c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718934000,"log":"[2026-02-17T14:51:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sml5akyffy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718935000,"log":"[2026-02-17T13:40:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4826941563242830000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dqpqm9krn4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718936000,"log":"[2026-02-17T13:07:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d1il94g9px","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718937000,"log":"[2026-02-17T13:11:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1370051663693830000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-amhgq43thh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718938000,"log":"[2026-02-17T13:19:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7509081159294601000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1k2x9gz0fa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718939000,"log":"[2026-02-17T13:11:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5631022392841231000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qo2ydmh6cd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718940000,"log":"[2026-02-17T13:40:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2213221080097631500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yvua0po9iw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718941000,"log":"[2026-02-17T13:52:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 481ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m8zbbzujnx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718942000,"log":"[2026-02-17T14:53:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-auy2orxq6q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718943000,"log":"[2026-02-17T14:06:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 334ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tlqe8rk5i8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718944000,"log":"[2026-02-17T13:51:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 349ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8i32ahuvyb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718945000,"log":"[2026-02-17T13:41:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9163860433106679000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-syfy7qeuki","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718946000,"log":"[2026-02-17T14:08:18Z DEBUG sqlx::query] INSERT FROM users returned 194 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tamlyypldu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718947000,"log":"[2026-02-17T14:37:59Z INFO tonic::transport::server] grpc router request_received client_id=o50m4w9i","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5o5epypj5m","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718948000,"log":"[2026-02-17T14:50:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wxsu0eqaid","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718949000,"log":"[2026-02-17T14:29:37Z DEBUG sqlx::query] SELECT FROM streams returned 390 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bc3mk056f6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718950000,"log":"[2026-02-17T13:58:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 471ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5bdh3y9buw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718951000,"log":"[2026-02-17T13:33:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7715735001762831000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z9kvzoimpo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718952000,"log":"[2026-02-17T15:10:35Z DEBUG metrics::reporter] recorded metric active_connections value=6689","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lj3cax8vdq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718953000,"log":"[2026-02-17T13:28:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1623086726646971600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nv06ao7t7s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718954000,"log":"[2026-02-17T15:20:04Z TRACE cache::redis] cache delete key=stream:schema:1484","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e3od1jlass","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718955000,"log":"[2026-02-17T14:15:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 982 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wj8k18bgyz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718956000,"log":"[2026-02-17T13:24:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2856298987156984300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1i5fip1csz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718957000,"log":"[2026-02-17T13:19:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9586698567819741000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xv91f4s53x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718958000,"log":"[2026-02-17T14:30:23Z DEBUG sqlx::query] INSERT FROM users returned 443 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4y2jofjjxw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718959000,"log":"[2026-02-17T12:59:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sq57br803o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718960000,"log":"[2026-02-17T12:39:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g0cuyy6ft6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718961000,"log":"[2026-02-17T13:41:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2642181187601966000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-18lxa8cu90","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718962000,"log":"[2026-02-17T13:25:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/653073787384557200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z4s1vk9y1v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718963000,"log":"[2026-02-17T13:26:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2573392861406318600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2iry44qoou","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718964000,"log":"[2026-02-17T14:54:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tuhgbjhvcl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718965000,"log":"[2026-02-17T14:46:13Z INFO tonic::transport::server] grpc querier disconnected client_id=ech5tpwb","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b8expb05ev","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718966000,"log":"[2026-02-17T12:53:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7n6o3e33d4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718967000,"log":"[2026-02-17T13:47:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b7gw24ol7u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718968000,"log":"[2026-02-17T13:41:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1839853564376274700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ku8ry19mor","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718969000,"log":"[2026-02-17T14:57:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8258noky5y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718970000,"log":"[2026-02-17T13:01:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qxcoam8qr3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718971000,"log":"[2026-02-17T14:34:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=0xc69lxt","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4lvp6dr7a1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718972000,"log":"[2026-02-17T15:13:00Z DEBUG metrics::reporter] recorded metric active_connections value=8939","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tdew1ezfxn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718973000,"log":"[2026-02-17T14:14:15Z DEBUG sqlx::query] DELETE FROM alerts returned 511 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7hopigoojg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718974000,"log":"[2026-02-17T14:33:29Z INFO tonic::transport::server] grpc router request_received client_id=9muydqjn","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jq4me0pzq5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718975000,"log":"[2026-02-17T15:12:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=3507","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-foavk099ww","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718976000,"log":"[2026-02-17T14:32:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=cxcz1sge","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j1ndludhz7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718977000,"log":"[2026-02-17T14:48:18Z INFO tonic::transport::server] grpc querier disconnected client_id=zdf1kx0t","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8iz2bcbrfp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718978000,"log":"[2026-02-17T13:31:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6496082919717456000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jnceaptrie","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718979000,"log":"[2026-02-17T12:57:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1t2cwh4l78","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718980000,"log":"[2026-02-17T13:51:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 347ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k3xb306w85","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718981000,"log":"[2026-02-17T14:30:38Z DEBUG sqlx::query] INSERT FROM users returned 516 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m2wmlxat8g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718982000,"log":"[2026-02-17T13:51:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 363ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oi8bijrnzk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718983000,"log":"[2026-02-17T13:47:00Z INFO actix_web::middleware::logger] POST /api/_search 204 237ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-guxlzc0r0w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718984000,"log":"[2026-02-17T15:17:06Z TRACE cache::redis] cache hit key=auth:token:6340","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yueon9899i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718985000,"log":"[2026-02-17T12:59:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6qv89s5ckh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718986000,"log":"[2026-02-17T14:28:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 417 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-umnbi6k5hi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718987000,"log":"[2026-02-17T15:05:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=6462","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n3m7clymxf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718988000,"log":"[2026-02-17T15:20:28Z TRACE cache::redis] cache delete key=org:config:5885","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xq2strbldd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718989000,"log":"[2026-02-17T15:09:01Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=521","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a9j1kcjqwh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718990000,"log":"[2026-02-17T12:58:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zvnjxotzgn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718991000,"log":"[2026-02-17T14:58:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4a33w1xab1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718992000,"log":"[2026-02-17T12:54:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3twmpyp7bu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718993000,"log":"[2026-02-17T13:52:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 415ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3vf1arg7y9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718994000,"log":"[2026-02-17T14:45:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=53rvbr7k","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kmv9af2mhi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718995000,"log":"[2026-02-17T14:41:48Z INFO tonic::transport::server] grpc querier disconnected client_id=wehc7b5s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zgqlqvh4ld","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718996000,"log":"[2026-02-17T13:22:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/83529063941256700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-57u2kttlyc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331718997000,"log":"[2026-02-17T14:28:38Z DEBUG sqlx::query] INSERT FROM users returned 603 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e64935fiba","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331718998000,"log":"[2026-02-17T14:20:45Z DEBUG sqlx::query] DELETE FROM alerts returned 346 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uu62vpdf0v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331718999000,"log":"[2026-02-17T13:45:25Z INFO actix_web::middleware::logger] GET /api/_search 404 491ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w7taf7ufzd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719000000,"log":"[2026-02-17T14:11:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 833 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3bb940iec8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719001000,"log":"[2026-02-17T13:05:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cz6sf4wxlb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719002000,"log":"[2026-02-17T12:47:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-psnawqdyxj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719003000,"log":"[2026-02-17T15:13:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4976","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0dm3tlxn2s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719004000,"log":"[2026-02-17T13:34:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7838390175751293000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4aott4vof0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719005000,"log":"[2026-02-17T14:49:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-75s1l5n3gt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719006000,"log":"[2026-02-17T13:35:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5142575696315705000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qigsnx5ayk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719007000,"log":"[2026-02-17T13:02:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fgp8zqiv31","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719008000,"log":"[2026-02-17T13:52:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 147ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-52wxwsk6p4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719009000,"log":"[2026-02-17T14:22:25Z DEBUG sqlx::query] DELETE FROM alerts returned 13 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8lrlz1803e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719010000,"log":"[2026-02-17T15:01:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gste6v2rsp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719011000,"log":"[2026-02-17T12:41:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5w7xcff8cz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719012000,"log":"[2026-02-17T13:54:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 385ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-af5q0vgkey","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719013000,"log":"[2026-02-17T14:03:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 16ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7sxpnmfqag","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719014000,"log":"[2026-02-17T12:37:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gdqd4eudlq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719015000,"log":"[2026-02-17T15:08:50Z DEBUG metrics::reporter] recorded metric active_connections value=8642","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pofqoukrm7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719016000,"log":"[2026-02-17T13:45:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 472ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bilkxu79du","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719017000,"log":"[2026-02-17T13:43:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 438ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d1dvhm4fr3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719018000,"log":"[2026-02-17T13:44:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 420ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a55wimwsth","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719019000,"log":"[2026-02-17T13:16:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7171409922370239000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jd846r42hh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719020000,"log":"[2026-02-17T13:35:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6446419812585509000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qxel5gack2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719021000,"log":"[2026-02-17T13:02:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bah0lpc9av","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719022000,"log":"[2026-02-17T12:57:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lap7fyoyd1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719023000,"log":"[2026-02-17T14:17:15Z DEBUG sqlx::query] DELETE FROM alerts returned 822 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y6lqsgosp7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719024000,"log":"[2026-02-17T13:20:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5524822639344036000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1u2htr5asy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719025000,"log":"[2026-02-17T13:29:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9428567011206558000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jh3qexx933","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719026000,"log":"[2026-02-17T14:22:30Z DEBUG sqlx::query] DELETE FROM alerts returned 809 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ej5y0tqw96","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719027000,"log":"[2026-02-17T12:53:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5fn6jt2n5u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719028000,"log":"[2026-02-17T13:45:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 391ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6qyuz8sba1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719029000,"log":"[2026-02-17T13:28:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4756042790181600000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rtqg2w1exr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719030000,"log":"[2026-02-17T14:06:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 13ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f0fl7mqzhh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719031000,"log":"[2026-02-17T14:10:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 429 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z685o0y2sx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719032000,"log":"[2026-02-17T14:49:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m9luoxa76j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719033000,"log":"[2026-02-17T13:13:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1330123495303077000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-40jjuqegod","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719034000,"log":"[2026-02-17T12:45:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gjx7vv67pz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719035000,"log":"[2026-02-17T12:39:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tdlrt97hjj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719036000,"log":"[2026-02-17T14:34:22Z INFO tonic::transport::server] grpc ingester connected client_id=eiv2xhm8","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1r4e1xepum","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719037000,"log":"[2026-02-17T14:10:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 601 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h02fcpz7gq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719038000,"log":"[2026-02-17T13:43:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 355ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1wmgr0zw2p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719039000,"log":"[2026-02-17T13:42:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 422ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g825qc655d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719040000,"log":"[2026-02-17T13:43:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 333ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ueamerhwan","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719041000,"log":"[2026-02-17T13:00:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0elca1j2rc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719042000,"log":"[2026-02-17T13:52:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o60gf00v37","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719043000,"log":"[2026-02-17T14:37:02Z INFO tonic::transport::server] grpc ingester connected client_id=6bw80hlb","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tt0p67fjo4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719044000,"log":"[2026-02-17T14:38:47Z INFO tonic::transport::server] grpc ingester connected client_id=zg7bkcbc","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-njyy88m2d5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719045000,"log":"[2026-02-17T12:59:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-16onlo9l5c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719046000,"log":"[2026-02-17T14:58:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o9xtz0plk6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719047000,"log":"[2026-02-17T14:02:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 240ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fjmel4ofsj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719048000,"log":"[2026-02-17T13:13:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5946997882671437000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sj9jmly4qs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719049000,"log":"[2026-02-17T15:11:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=5970","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3aefddcdna","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719050000,"log":"[2026-02-17T14:25:38Z DEBUG sqlx::query] INSERT FROM users returned 7 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0n38kfwq6l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719051000,"log":"[2026-02-17T12:56:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ybjtownznp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719052000,"log":"[2026-02-17T14:39:52Z INFO tonic::transport::server] grpc ingester connected client_id=r4atc2d2","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wujzwu8fin","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719053000,"log":"[2026-02-17T13:45:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hb20q0bbht","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719054000,"log":"[2026-02-17T13:34:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1934470084265194000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7s463u99lt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719055000,"log":"[2026-02-17T13:02:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j8uy1quej6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719056000,"log":"[2026-02-17T14:20:15Z DEBUG sqlx::query] DELETE FROM alerts returned 397 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jyi9urue0h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719057000,"log":"[2026-02-17T13:38:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3866347243690422300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gbr53copma","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719058000,"log":"[2026-02-17T15:03:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zfcjkv9byy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719059000,"log":"[2026-02-17T13:45:45Z INFO actix_web::middleware::logger] GET /api/_search 503 72ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0nvdaptbna","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719060000,"log":"[2026-02-17T13:52:45Z INFO actix_web::middleware::logger] GET /api/_search 403 112ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cxu9a04w02","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719061000,"log":"[2026-02-17T14:47:43Z INFO tonic::transport::server] grpc querier disconnected client_id=ny7mp5ca","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n3xnuxq991","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719062000,"log":"[2026-02-17T15:00:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a490zcxnbz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719063000,"log":"[2026-02-17T14:18:18Z DEBUG sqlx::query] INSERT FROM users returned 499 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o2y9qbm3by","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719064000,"log":"[2026-02-17T14:26:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 709 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ra84tsl18e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719065000,"log":"[2026-02-17T12:43:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0cq1vkkr0t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719066000,"log":"[2026-02-17T13:43:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 235ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3gathqjs1q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719067000,"log":"[2026-02-17T13:58:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 25ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pa2neb94k4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719068000,"log":"[2026-02-17T12:39:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ol92umjr5e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719069000,"log":"[2026-02-17T13:07:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hlm6v7fjf9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719070000,"log":"[2026-02-17T14:34:29Z INFO tonic::transport::server] grpc router request_received client_id=ed6omkzw","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sk96ig6sg5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719071000,"log":"[2026-02-17T13:08:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-09pt220hqn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719072000,"log":"[2026-02-17T14:09:52Z DEBUG sqlx::query] SELECT FROM streams returned 205 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nm0pzjlke5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719073000,"log":"[2026-02-17T12:42:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q93gnmz7mi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719074000,"log":"[2026-02-17T12:45:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vo0s4abety","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719075000,"log":"[2026-02-17T13:25:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/514740422489079500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gw3ckboh5g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719076000,"log":"[2026-02-17T14:03:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 289ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-keftogxjb9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719077000,"log":"[2026-02-17T12:48:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-v60eujwa14","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719078000,"log":"[2026-02-17T12:36:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ek7x8nldnb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719079000,"log":"[2026-02-17T13:57:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 97ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mfwq8jywhm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719080000,"log":"[2026-02-17T14:50:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4zdu7kpmcx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719081000,"log":"[2026-02-17T14:26:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 748 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k3xxkayqvn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719082000,"log":"[2026-02-17T12:42:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rn58eh96pz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719083000,"log":"[2026-02-17T13:32:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6576087725994668000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qraq3u2cyy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719084000,"log":"[2026-02-17T14:27:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 506 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2dd45ts31v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719085000,"log":"[2026-02-17T15:16:08Z TRACE cache::redis] cache get key=org:config:7072","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n9ugys9c68","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719086000,"log":"[2026-02-17T14:18:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 215 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6g23v7j7zo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719087000,"log":"[2026-02-17T13:39:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4332184069840423400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w2ix5stpgn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719088000,"log":"[2026-02-17T12:57:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sabomqqxoh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719089000,"log":"[2026-02-17T14:55:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1cmgp23ree","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719090000,"log":"[2026-02-17T13:15:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2444202125298276000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q6y7gr3dih","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719091000,"log":"[2026-02-17T13:37:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7598046533308660000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q7oxnjzk8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719092000,"log":"[2026-02-17T14:32:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=q6mgc8r0","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zmx2ffi28o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719093000,"log":"[2026-02-17T14:54:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c2w0wbzpst","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719094000,"log":"[2026-02-17T14:21:28Z DEBUG sqlx::query] INSERT FROM users returned 534 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-di5ddtj5gx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719095000,"log":"[2026-02-17T13:06:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2akj1esgkg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719096000,"log":"[2026-02-17T14:13:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 737 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8ulf5xr9vk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719097000,"log":"[2026-02-17T14:24:28Z DEBUG sqlx::query] INSERT FROM users returned 707 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pje0tjufmk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719098000,"log":"[2026-02-17T14:05:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 289ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ncv90e33q6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719099000,"log":"[2026-02-17T14:15:38Z DEBUG sqlx::query] INSERT FROM users returned 432 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-032uczy11q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719100000,"log":"[2026-02-17T14:55:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eacv579chw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719101000,"log":"[2026-02-17T13:08:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8svuk9qh9t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719102000,"log":"[2026-02-17T13:05:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v69st2vfsg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719103000,"log":"[2026-02-17T14:39:59Z INFO tonic::transport::server] grpc router request_received client_id=cf4uonm7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wje9htz7i0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719104000,"log":"[2026-02-17T14:25:08Z DEBUG sqlx::query] INSERT FROM users returned 670 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y95076z5u8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719105000,"log":"[2026-02-17T13:53:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 286ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-px1ck25i96","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719106000,"log":"[2026-02-17T14:42:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=wdv6q2ii","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1r6t9m562h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719107000,"log":"[2026-02-17T14:48:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rfbkszub","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nrmlct9cf8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719108000,"log":"[2026-02-17T14:44:13Z INFO tonic::transport::server] grpc querier disconnected client_id=oyngko0i","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fwlud3g6hk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719109000,"log":"[2026-02-17T14:41:53Z INFO tonic::transport::server] grpc querier disconnected client_id=z7njjhkk","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yrdkefg2rd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719110000,"log":"[2026-02-17T12:50:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qvmdybfhge","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719111000,"log":"[2026-02-17T14:20:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 670 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5gteshawpl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719112000,"log":"[2026-02-17T15:09:58Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3399","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mnrcl4z9gl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719113000,"log":"[2026-02-17T12:56:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rgxdconmcl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719114000,"log":"[2026-02-17T13:26:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3640628966949488600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7jpcy34tq0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719115000,"log":"[2026-02-17T14:46:39Z INFO tonic::transport::server] grpc router request_received client_id=la1tt6kf","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hfe1l1nikp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719116000,"log":"[2026-02-17T14:31:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 924 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rd0zgdr5um","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719117000,"log":"[2026-02-17T15:17:17Z TRACE cache::redis] cache expire key=user:session:6809","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m2xbp1gcyg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719118000,"log":"[2026-02-17T14:06:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 238ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1pujn05cbr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719119000,"log":"[2026-02-17T15:06:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=3649","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bt9vjcrnin","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719120000,"log":"[2026-02-17T12:57:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7i0tlq4x3h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719121000,"log":"[2026-02-17T14:29:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 278 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5e800cn31q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719122000,"log":"[2026-02-17T14:19:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 608 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i0az5iinga","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719123000,"log":"[2026-02-17T14:07:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 9 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-73k3cam3r6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719124000,"log":"[2026-02-17T13:51:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 482ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-btrvcfu8mr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719125000,"log":"[2026-02-17T13:49:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 476ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x0kw62gnpj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719126000,"log":"[2026-02-17T13:14:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7434715277809492000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nn4f5mntdd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719127000,"log":"[2026-02-17T13:06:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-madmypyrui","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719128000,"log":"[2026-02-17T13:59:25Z INFO actix_web::middleware::logger] GET /api/_search 200 185ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cw2vikdx5r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719129000,"log":"[2026-02-17T14:23:25Z DEBUG sqlx::query] DELETE FROM alerts returned 787 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b9hqk5egmx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719130000,"log":"[2026-02-17T13:03:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xmmy8tdfes","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719131000,"log":"[2026-02-17T12:57:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ax2xp0dpqk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719132000,"log":"[2026-02-17T15:05:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2858","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5q5qhpvhbp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719133000,"log":"[2026-02-17T14:36:18Z INFO tonic::transport::server] grpc querier disconnected client_id=r2qoeo3w","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iu5k5phfq2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719134000,"log":"[2026-02-17T13:48:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 416ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vhat48lujn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719135000,"log":"[2026-02-17T14:11:58Z DEBUG sqlx::query] INSERT FROM users returned 402 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mn4y55r85z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719136000,"log":"[2026-02-17T12:41:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-om90i13bn3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719137000,"log":"[2026-02-17T12:39:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8d7gi0rso7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719138000,"log":"[2026-02-17T13:28:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5861335098058442000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k9te19jefw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719139000,"log":"[2026-02-17T13:14:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/205969009003863500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-exlapy2bwe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719140000,"log":"[2026-02-17T12:41:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i4buj1qsss","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719141000,"log":"[2026-02-17T13:26:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1236675669728926000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ndr8h96zz5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719142000,"log":"[2026-02-17T14:55:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-khki67om3i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719143000,"log":"[2026-02-17T14:28:32Z DEBUG sqlx::query] SELECT FROM streams returned 929 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yv1ky9mtee","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719144000,"log":"[2026-02-17T14:59:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zld9yhxpjs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719145000,"log":"[2026-02-17T13:35:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9371921874030260000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bpnxg3psus","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719146000,"log":"[2026-02-17T14:08:42Z DEBUG sqlx::query] SELECT FROM streams returned 463 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wqfi3yj69j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719147000,"log":"[2026-02-17T13:22:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7613376669060170000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kcam88xj6v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719148000,"log":"[2026-02-17T15:14:49Z TRACE cache::redis] cache miss key=stream:schema:2754","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-48dq682oc6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719149000,"log":"[2026-02-17T14:31:27Z DEBUG sqlx::query] SELECT FROM streams returned 544 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6p4s8mwbnh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719150000,"log":"[2026-02-17T13:22:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7442379840289393000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9axepyl4id","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719151000,"log":"[2026-02-17T13:52:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 204ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0oeeu1mlhq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719152000,"log":"[2026-02-17T14:12:28Z DEBUG sqlx::query] INSERT FROM users returned 93 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4eljenzah8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719153000,"log":"[2026-02-17T13:08:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6gk4x0hal7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719154000,"log":"[2026-02-17T14:40:32Z INFO tonic::transport::server] grpc ingester connected client_id=c5vcsmyt","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-55woki2bp4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719155000,"log":"[2026-02-17T15:00:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6tmcmoz05d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719156000,"log":"[2026-02-17T13:46:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 393ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bgxlud28ti","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719157000,"log":"[2026-02-17T14:32:23Z INFO tonic::transport::server] grpc querier disconnected client_id=occ9g49d","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pegdayjd7k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719158000,"log":"[2026-02-17T14:41:43Z INFO tonic::transport::server] grpc querier disconnected client_id=59nx5784","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fc4ixr9pzd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719159000,"log":"[2026-02-17T14:26:08Z DEBUG sqlx::query] INSERT FROM users returned 956 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9qxib88c6f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719160000,"log":"[2026-02-17T13:42:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 343ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z2bpcwjlpg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719161000,"log":"[2026-02-17T12:37:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fo5jesa103","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719162000,"log":"[2026-02-17T13:05:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-97bvov608r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719163000,"log":"[2026-02-17T13:18:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1233929601598802200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a2ccp4j5o2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719164000,"log":"[2026-02-17T13:27:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2702432083524848000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oyy8dtasdk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719165000,"log":"[2026-02-17T13:30:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7532898716112040000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-py2j1n4is1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719166000,"log":"[2026-02-17T13:21:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8877049877693946000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0uk88rztye","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719167000,"log":"[2026-02-17T14:59:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1dplz0r4bi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719168000,"log":"[2026-02-17T14:47:02Z INFO tonic::transport::server] grpc ingester connected client_id=wh35b3m6","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6oqhlliuef","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719169000,"log":"[2026-02-17T15:04:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fflf8ifmsr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719170000,"log":"[2026-02-17T14:25:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 617 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fvklep7tf6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719171000,"log":"[2026-02-17T13:55:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 352ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f24b6icbqj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719172000,"log":"[2026-02-17T14:30:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 442 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gem7vcj7z1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719173000,"log":"[2026-02-17T12:40:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gz9kpqpv8n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719174000,"log":"[2026-02-17T14:20:52Z DEBUG sqlx::query] SELECT FROM streams returned 794 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nr7yz86r6a","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719175000,"log":"[2026-02-17T14:46:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=sllzzmhl","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tg03qgrdk1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719176000,"log":"[2026-02-17T15:09:00Z DEBUG metrics::reporter] recorded metric active_connections value=6564","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cbk60sl2u9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719177000,"log":"[2026-02-17T13:06:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kgw32s8fco","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719178000,"log":"[2026-02-17T12:36:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d7pmpuplqw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719179000,"log":"[2026-02-17T14:11:48Z DEBUG sqlx::query] INSERT FROM users returned 667 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8hgw1wv284","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719180000,"log":"[2026-02-17T14:25:45Z DEBUG sqlx::query] DELETE FROM alerts returned 872 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-065bvi9o2s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719181000,"log":"[2026-02-17T13:09:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6801129282054686000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9vc08oyfwn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719182000,"log":"[2026-02-17T13:51:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 430ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oea48ne3ss","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719183000,"log":"[2026-02-17T14:36:32Z INFO tonic::transport::server] grpc ingester connected client_id=jwt84ji3","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kcx8y7fe8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719184000,"log":"[2026-02-17T15:18:48Z TRACE cache::redis] cache hit key=org:config:1482","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d7ww67xb2v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719185000,"log":"[2026-02-17T14:16:30Z DEBUG sqlx::query] DELETE FROM alerts returned 827 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sr2cxch01z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719186000,"log":"[2026-02-17T12:51:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dwxnq6dmv6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719187000,"log":"[2026-02-17T14:27:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 552 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3u4khlomc3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719188000,"log":"[2026-02-17T13:32:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/281144000032043100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z8jmp2fosn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719189000,"log":"[2026-02-17T15:10:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=8210","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d77gjprknx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719190000,"log":"[2026-02-17T13:47:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 424ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-il91oycurv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719191000,"log":"[2026-02-17T13:21:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/510861941202493950.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b4guuat7jb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719192000,"log":"[2026-02-17T13:12:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8318847249020534000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gq73jasqfz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719193000,"log":"[2026-02-17T13:55:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 245ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-be72jmyzlh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719194000,"log":"[2026-02-17T13:06:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7bdbnr3tzb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719195000,"log":"[2026-02-17T15:20:40Z TRACE cache::redis] cache delete key=query:result:6260","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u640m5jeng","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719196000,"log":"[2026-02-17T13:48:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 201 284ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vm1eysk0z2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719197000,"log":"[2026-02-17T14:41:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=hwoptgpz","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bzyidqlerq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719198000,"log":"[2026-02-17T15:15:29Z TRACE cache::redis] cache expire key=stream:schema:2765","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qlymacuog6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719199000,"log":"[2026-02-17T15:02:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nrinq3a44r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719200000,"log":"[2026-02-17T14:43:34Z INFO tonic::transport::server] grpc router request_received client_id=h7f5u07r","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6ubhezyrwu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719201000,"log":"[2026-02-17T14:12:17Z DEBUG sqlx::query] SELECT FROM streams returned 281 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fexu40w48k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719202000,"log":"[2026-02-17T13:03:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ciylso92pz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719203000,"log":"[2026-02-17T12:51:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rabjtgmdjy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719204000,"log":"[2026-02-17T14:54:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-efe9elkncy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719205000,"log":"[2026-02-17T14:22:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 143 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l8cn0pnxke","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719206000,"log":"[2026-02-17T12:51:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3ciu3ral27","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719207000,"log":"[2026-02-17T13:59:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 52ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-angwcia9ux","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719208000,"log":"[2026-02-17T13:43:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 469ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7ktcqfjngg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719209000,"log":"[2026-02-17T12:43:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dwz1tsu2vu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719210000,"log":"[2026-02-17T12:39:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nqf5cocad2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719211000,"log":"[2026-02-17T14:30:35Z DEBUG sqlx::query] DELETE FROM alerts returned 550 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wz5ep1d3ud","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719212000,"log":"[2026-02-17T15:01:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4l6uusjib0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719213000,"log":"[2026-02-17T12:54:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7sxuubwi80","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719214000,"log":"[2026-02-17T13:35:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6120244228041494000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7tjq0ityvt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719215000,"log":"[2026-02-17T15:00:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aklrbreqdj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719216000,"log":"[2026-02-17T14:49:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-75b0pluz3z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719217000,"log":"[2026-02-17T14:02:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 400ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mtoyx8osr3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719218000,"log":"[2026-02-17T13:19:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4953219503414454000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sclahlfdue","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719219000,"log":"[2026-02-17T14:21:33Z DEBUG sqlx::query] INSERT FROM users returned 2 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ihaflvja4s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719220000,"log":"[2026-02-17T14:07:45Z DEBUG sqlx::query] DELETE FROM alerts returned 789 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oc1wzzni18","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719221000,"log":"[2026-02-17T14:58:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u9zcoxqmlh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719222000,"log":"[2026-02-17T14:59:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j0z4qynhs1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719223000,"log":"[2026-02-17T13:25:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/590065147064604500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0oyfuj85vr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719224000,"log":"[2026-02-17T14:09:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 983 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8b2ssr0c9c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719225000,"log":"[2026-02-17T12:51:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b6mzapiksy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719226000,"log":"[2026-02-17T14:59:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3aqoaz8fwk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719227000,"log":"[2026-02-17T13:18:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8506233338495788000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cffqz85oly","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719228000,"log":"[2026-02-17T14:00:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 400 167ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j8qprgzw45","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719229000,"log":"[2026-02-17T14:37:34Z INFO tonic::transport::server] grpc router request_received client_id=egasj1qh","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7n6qqtv16g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719230000,"log":"[2026-02-17T12:39:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lnvgv7rowk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719231000,"log":"[2026-02-17T14:49:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m2yyz4i4zq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719232000,"log":"[2026-02-17T12:53:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vn25ohsbyg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719233000,"log":"[2026-02-17T14:46:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=6nz1m0jk","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xxjojohx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719234000,"log":"[2026-02-17T14:00:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 448ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g3u20d55ts","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719235000,"log":"[2026-02-17T14:58:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wb19nrew0w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719236000,"log":"[2026-02-17T13:58:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 394ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yb5ueo480i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719237000,"log":"[2026-02-17T13:59:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 500 45ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q3pldi76cn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719238000,"log":"[2026-02-17T15:20:14Z TRACE cache::redis] cache get key=stream:schema:2720","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jd3e98x2gu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719239000,"log":"[2026-02-17T14:46:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=uw5bm1s9","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xib73upoql","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719240000,"log":"[2026-02-17T12:47:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-utw0x4zp2r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719241000,"log":"[2026-02-17T15:06:29Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5087","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1o5icvi8ll","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719242000,"log":"[2026-02-17T13:32:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3085488887250822000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-do8ccwe8mt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719243000,"log":"[2026-02-17T12:54:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zwdcdn1j58","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719244000,"log":"[2026-02-17T13:18:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9844329067584213000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f347b6dhgp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719245000,"log":"[2026-02-17T15:17:09Z TRACE cache::redis] cache set key=stream:schema:787","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xbmxscm32u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719246000,"log":"[2026-02-17T14:41:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=avf7afby","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b9odmbzobd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719247000,"log":"[2026-02-17T13:00:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pg9zt7ydk5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719248000,"log":"[2026-02-17T14:23:43Z DEBUG sqlx::query] INSERT FROM users returned 486 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-il8h9jwuch","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719249000,"log":"[2026-02-17T13:38:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1656385071971191600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0plfx26pyl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719250000,"log":"[2026-02-17T13:27:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3484427377371757000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fph1ojodgc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719251000,"log":"[2026-02-17T14:44:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=99f2zjyq","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rrdy9008vd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719252000,"log":"[2026-02-17T13:32:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7064774572085673000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k43pfqtnbb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719253000,"log":"[2026-02-17T13:15:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8130501639164853000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a0qgw7q9i7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719254000,"log":"[2026-02-17T13:25:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6376916947371445000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tpxim4k5g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719255000,"log":"[2026-02-17T14:45:24Z INFO tonic::transport::server] grpc router request_received client_id=f06jpd2k","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nrlemjre29","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719256000,"log":"[2026-02-17T12:51:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2wkinl9xm0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719257000,"log":"[2026-02-17T14:10:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 956 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x884iqhw29","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719258000,"log":"[2026-02-17T14:05:05Z INFO actix_web::middleware::logger] GET /api/_search 500 384ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a6vwx67ip3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719259000,"log":"[2026-02-17T13:37:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1276747405596045800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-quo8bn5klf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719260000,"log":"[2026-02-17T14:55:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wbxtn73ngt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719261000,"log":"[2026-02-17T14:18:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 189 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3etjbw3cd9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719262000,"log":"[2026-02-17T14:59:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jrusll6k6g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719263000,"log":"[2026-02-17T15:13:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=3022","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w1yixdhrsk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719264000,"log":"[2026-02-17T14:22:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 51 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mhwrem7w7v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719265000,"log":"[2026-02-17T12:36:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dplugbf7wx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719266000,"log":"[2026-02-17T14:07:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 597 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h0yir1bt0x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719267000,"log":"[2026-02-17T14:25:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 72 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s64bd9d3q3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719268000,"log":"[2026-02-17T14:53:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-29e13cbzae","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719269000,"log":"[2026-02-17T12:54:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wuqq4gy7wg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719270000,"log":"[2026-02-17T13:42:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 269ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-llddj9nm5d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719271000,"log":"[2026-02-17T13:32:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8781455063702461000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mpb5qxou5f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719272000,"log":"[2026-02-17T13:14:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/438652277424644860.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xnku2ykjm5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719273000,"log":"[2026-02-17T14:15:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 680 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ad754li8p0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719274000,"log":"[2026-02-17T14:23:00Z DEBUG sqlx::query] DELETE FROM alerts returned 533 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cecjfczqd6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719275000,"log":"[2026-02-17T14:57:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2repsx6jmp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719276000,"log":"[2026-02-17T13:29:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5136998875384824000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mne1j0g5tg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719277000,"log":"[2026-02-17T12:36:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3hgkrvdlpv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719278000,"log":"[2026-02-17T13:00:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-49w6icv0u3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719279000,"log":"[2026-02-17T15:16:33Z TRACE cache::redis] cache set key=org:config:7314","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ld0c1sma7w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719280000,"log":"[2026-02-17T13:00:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cs2gglk0th","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719281000,"log":"[2026-02-17T14:08:43Z DEBUG sqlx::query] INSERT FROM users returned 554 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7nwgct62mo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719282000,"log":"[2026-02-17T15:03:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-by7lrwtbjq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719283000,"log":"[2026-02-17T12:47:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-77kwjva4ua","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719284000,"log":"[2026-02-17T14:03:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 503 425ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e9ih6y9c1v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719285000,"log":"[2026-02-17T14:18:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 751 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1otltdmvrs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719286000,"log":"[2026-02-17T12:43:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wnl7wezkm1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719287000,"log":"[2026-02-17T12:57:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m2p1xdllyp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719288000,"log":"[2026-02-17T12:54:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8nbdrlbb59","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719289000,"log":"[2026-02-17T15:14:58Z TRACE cache::redis] cache delete key=org:config:8110","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-huhexvxpln","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719290000,"log":"[2026-02-17T14:47:38Z INFO tonic::transport::server] grpc querier disconnected client_id=9kogaf9j","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-90c6x6ffpa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719291000,"log":"[2026-02-17T13:04:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3m8mcu0c8g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719292000,"log":"[2026-02-17T15:18:49Z TRACE cache::redis] cache miss key=stream:schema:1900","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1dq7kie54n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719293000,"log":"[2026-02-17T15:06:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=1172","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q2szuym4zs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719294000,"log":"[2026-02-17T13:06:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5w234a8pr0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719295000,"log":"[2026-02-17T13:36:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7285974430012368000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-81mrsr621h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719296000,"log":"[2026-02-17T14:54:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l1eah0cadr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719297000,"log":"[2026-02-17T12:43:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xroyraz6y9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719298000,"log":"[2026-02-17T14:45:32Z INFO tonic::transport::server] grpc ingester connected client_id=s76erogn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mxogzm0gx1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719299000,"log":"[2026-02-17T13:41:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2420753965895812000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1o0mqnmgy0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719300000,"log":"[2026-02-17T12:59:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eq3uw2fbbw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719301000,"log":"[2026-02-17T15:17:23Z TRACE cache::redis] cache expire key=org:config:6542","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f1kugepejc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719302000,"log":"[2026-02-17T13:50:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 190ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-za4cw4y1w3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719303000,"log":"[2026-02-17T14:44:23Z INFO tonic::transport::server] grpc querier disconnected client_id=yz8tgila","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3k1cti1ywi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719304000,"log":"[2026-02-17T13:20:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7746432369271332000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hei9ragrlt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719305000,"log":"[2026-02-17T13:17:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7181891954980975000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5y02d1uqlg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719306000,"log":"[2026-02-17T14:40:23Z INFO tonic::transport::server] grpc querier disconnected client_id=r0684ztm","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ji0t6znq7w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719307000,"log":"[2026-02-17T14:45:09Z INFO tonic::transport::server] grpc router request_received client_id=9lxxjusn","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l6dq6eb3tp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719308000,"log":"[2026-02-17T12:41:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y2gvd2bp9d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719309000,"log":"[2026-02-17T15:18:13Z TRACE cache::redis] cache miss key=org:config:9038","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ibf2qevh5m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719310000,"log":"[2026-02-17T14:56:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jhyq65qucw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719311000,"log":"[2026-02-17T13:08:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3230296827529881600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1kxdalc4yb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719312000,"log":"[2026-02-17T14:31:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 609 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bl53slhpr6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719313000,"log":"[2026-02-17T15:04:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1g8wwcs045","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719314000,"log":"[2026-02-17T13:49:25Z INFO actix_web::middleware::logger] GET /api/_search 400 268ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ynvjc9izmo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719315000,"log":"[2026-02-17T13:07:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-btvcnvw1k9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719316000,"log":"[2026-02-17T13:37:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8951860008672518000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pt1vpnibic","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719317000,"log":"[2026-02-17T14:26:32Z DEBUG sqlx::query] SELECT FROM streams returned 102 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zrqcudakfv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719318000,"log":"[2026-02-17T14:53:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-awk0kss1gc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719319000,"log":"[2026-02-17T14:00:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 148ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qosaieukzz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719320000,"log":"[2026-02-17T15:16:16Z TRACE cache::redis] cache delete key=auth:token:7223","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pllbji30sg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719321000,"log":"[2026-02-17T14:53:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jcysflzblc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719322000,"log":"[2026-02-17T14:25:23Z DEBUG sqlx::query] INSERT FROM users returned 476 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dzuxldb16c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719323000,"log":"[2026-02-17T14:00:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 29ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-36574dfgpr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719324000,"log":"[2026-02-17T14:49:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lraes0zuex","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719325000,"log":"[2026-02-17T14:37:28Z INFO tonic::transport::server] grpc querier disconnected client_id=7y8pk2f5","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-aigbyp28cz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719326000,"log":"[2026-02-17T12:44:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h8kdhz28kt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719327000,"log":"[2026-02-17T12:39:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lvs4fota5n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719328000,"log":"[2026-02-17T14:33:38Z INFO tonic::transport::server] grpc querier disconnected client_id=tb6dl5ja","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cnjou4l4aj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719329000,"log":"[2026-02-17T14:30:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 499 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xeoxfpdeff","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719330000,"log":"[2026-02-17T13:50:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 133ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sbfhxdtlwh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719331000,"log":"[2026-02-17T15:10:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=6710","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cr2dakrex7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719332000,"log":"[2026-02-17T15:00:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c8aw3x8ogb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719333000,"log":"[2026-02-17T13:59:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 277ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fl41i4fxsu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719334000,"log":"[2026-02-17T13:36:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/169707315601366200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2wml81o0v7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719335000,"log":"[2026-02-17T13:09:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5038541729446457000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5wzqgvxt5r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719336000,"log":"[2026-02-17T14:34:54Z INFO tonic::transport::server] grpc router request_received client_id=bjxiw3ii","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t9kzv676yn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719337000,"log":"[2026-02-17T14:02:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 2ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-awlj0omr8v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719338000,"log":"[2026-02-17T14:47:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=14j0rj47","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-84w63c5zu4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719339000,"log":"[2026-02-17T12:50:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6xd7enh4hf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719340000,"log":"[2026-02-17T13:20:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7922098817125305000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pdxgcbcekc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719341000,"log":"[2026-02-17T13:20:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3509280275787168000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u7gxylnj27","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719342000,"log":"[2026-02-17T13:36:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2643606283819972600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oqvlzhbu2e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719343000,"log":"[2026-02-17T12:51:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ck2dg9wwa0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719344000,"log":"[2026-02-17T15:05:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=3885","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-svwl7grfd3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719345000,"log":"[2026-02-17T12:35:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cld6lbh341","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719346000,"log":"[2026-02-17T15:01:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kjovu7infr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719347000,"log":"[2026-02-17T14:20:27Z DEBUG sqlx::query] SELECT FROM streams returned 994 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u0h2160nuj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719348000,"log":"[2026-02-17T13:53:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 440ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c7rifxwcer","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719349000,"log":"[2026-02-17T14:02:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 175ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9ys21815qg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719350000,"log":"[2026-02-17T14:21:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 170 rows in 88ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-36n6yeaq38","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719351000,"log":"[2026-02-17T15:19:22Z TRACE cache::redis] cache delete key=user:session:9709","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qmsl5l9gq7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719352000,"log":"[2026-02-17T13:54:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 321ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hj9930u9p0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719353000,"log":"[2026-02-17T15:00:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bzn5l9cnrg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719354000,"log":"[2026-02-17T13:33:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2477586269975997400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yfe036wuxo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719355000,"log":"[2026-02-17T13:53:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 338ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j2wsobj3ja","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719356000,"log":"[2026-02-17T14:12:35Z DEBUG sqlx::query] DELETE FROM alerts returned 956 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-454yjcvhra","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719357000,"log":"[2026-02-17T14:08:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 544 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-00bgtkyego","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719358000,"log":"[2026-02-17T13:47:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 124ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qwv37inkoi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719359000,"log":"[2026-02-17T14:05:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 278ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s71lbccyfo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719360000,"log":"[2026-02-17T13:35:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9848061444545995000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uqkwnazgvj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719361000,"log":"[2026-02-17T12:48:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bc7jspgxh6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719362000,"log":"[2026-02-17T14:35:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=28vag5f1","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-40snwt20ln","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719363000,"log":"[2026-02-17T14:57:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5yw3g115xn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719364000,"log":"[2026-02-17T13:53:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 189ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kzm4s7g4i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719365000,"log":"[2026-02-17T13:55:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 370ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3uytydagxf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719366000,"log":"[2026-02-17T13:29:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/876649033975505000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hdjemdv42k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719367000,"log":"[2026-02-17T13:28:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/513715066338442700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nrhd2gl72d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719368000,"log":"[2026-02-17T13:16:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4262373342928378400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yhxx9kdpsj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719369000,"log":"[2026-02-17T13:53:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 153ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-91f3vf069e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719370000,"log":"[2026-02-17T15:14:03Z TRACE cache::redis] cache set key=org:config:6338","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g1ksddivlz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719371000,"log":"[2026-02-17T13:19:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6669684915869758000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2200iwl5ba","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719372000,"log":"[2026-02-17T13:27:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7994858236559954000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0lcnalo3nb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719373000,"log":"[2026-02-17T14:13:05Z DEBUG sqlx::query] DELETE FROM alerts returned 330 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mvic2jj75g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719374000,"log":"[2026-02-17T13:28:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2161215183532683300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zb0q845rm0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719375000,"log":"[2026-02-17T14:19:53Z DEBUG sqlx::query] INSERT FROM users returned 889 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6c4py1j8kc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719376000,"log":"[2026-02-17T14:07:37Z DEBUG sqlx::query] SELECT FROM streams returned 754 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vt0ox3so0c","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719377000,"log":"[2026-02-17T13:55:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 125ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wywdofx1ii","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719378000,"log":"[2026-02-17T13:25:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6264876607623804000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j8qbh68210","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719379000,"log":"[2026-02-17T13:20:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4324526093241414000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nq5l94h9l5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719380000,"log":"[2026-02-17T13:09:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6476539687087688000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ihjlzo927d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719381000,"log":"[2026-02-17T14:32:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=xexnq5fu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i4dfwn7egb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719382000,"log":"[2026-02-17T13:30:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7283884820153814000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l3dwcphsf1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719383000,"log":"[2026-02-17T12:38:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3hedn1bwf8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719384000,"log":"[2026-02-17T12:53:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-70gfpb4qhs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719385000,"log":"[2026-02-17T14:38:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=og4h9mq3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0yajabqnp1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719386000,"log":"[2026-02-17T15:09:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=6130","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2f36k1ft0p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719387000,"log":"[2026-02-17T15:17:46Z TRACE cache::redis] cache delete key=auth:token:39","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oe5pntmhjh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719388000,"log":"[2026-02-17T15:10:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3184","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-95s63bb8j6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719389000,"log":"[2026-02-17T13:30:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4388220903009179000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-38h2a5zuz2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719390000,"log":"[2026-02-17T14:45:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=010uqovq","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vl2qyhzmas","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719391000,"log":"[2026-02-17T13:01:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5etgx21h0b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719392000,"log":"[2026-02-17T13:10:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/250985029501222820.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c55i6tl1xo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719393000,"log":"[2026-02-17T14:34:44Z INFO tonic::transport::server] grpc router request_received client_id=1kkjb2yt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mj0vjno9vr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719394000,"log":"[2026-02-17T13:01:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hchmmzc49j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719395000,"log":"[2026-02-17T13:16:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3682232910049543000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wevxo4ahcm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719396000,"log":"[2026-02-17T13:00:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0m6shmve1o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719397000,"log":"[2026-02-17T12:35:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aqhp9alet0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719398000,"log":"[2026-02-17T13:31:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2683192309148112400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xr3se6ee0d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719399000,"log":"[2026-02-17T13:45:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 52ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dvf0k8t5y7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719400000,"log":"[2026-02-17T13:30:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1920942684796638200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jorehb241z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719401000,"log":"[2026-02-17T14:37:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=zn3tuwme","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-utla0ct1bx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719402000,"log":"[2026-02-17T14:00:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 400ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r3o2wwam4s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719403000,"log":"[2026-02-17T15:00:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ug44c22j9o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719404000,"log":"[2026-02-17T12:44:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f9aq6720cl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719405000,"log":"[2026-02-17T15:12:20Z DEBUG metrics::reporter] recorded metric active_connections value=8308","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j7q46pzx9y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719406000,"log":"[2026-02-17T13:40:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9617427043182774000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-25h0svh8v2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719407000,"log":"[2026-02-17T14:31:38Z DEBUG sqlx::query] INSERT FROM users returned 521 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r4c8xsynav","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719408000,"log":"[2026-02-17T13:56:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 168ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y5kmfi287i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719409000,"log":"[2026-02-17T14:46:38Z INFO tonic::transport::server] grpc querier disconnected client_id=pig3pxu9","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dnlwgyjatb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719410000,"log":"[2026-02-17T15:10:25Z DEBUG metrics::reporter] recorded metric active_connections value=3260","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-67rb1k926h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719411000,"log":"[2026-02-17T14:08:20Z DEBUG sqlx::query] DELETE FROM alerts returned 667 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xhwsw82odp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719412000,"log":"[2026-02-17T12:56:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z3q4l3idsf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719413000,"log":"[2026-02-17T13:29:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6638171308507286000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-341os3ln3h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719414000,"log":"[2026-02-17T15:01:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kkulwtwp53","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719415000,"log":"[2026-02-17T13:52:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hiyfa2vp97","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719416000,"log":"[2026-02-17T13:38:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6723618268880853000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jac795lxle","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719417000,"log":"[2026-02-17T12:43:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-49iynaqam9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719418000,"log":"[2026-02-17T15:14:27Z TRACE cache::redis] cache set key=user:session:9108","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-he2ytol68m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719419000,"log":"[2026-02-17T14:55:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-92i4q47f6f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719420000,"log":"[2026-02-17T14:43:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=fe8j1zjj","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yhahog5r8h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719421000,"log":"[2026-02-17T15:20:31Z TRACE cache::redis] cache miss key=auth:token:2218","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r3bfvtq7jp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719422000,"log":"[2026-02-17T13:55:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 285ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ab3zrbwx67","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719423000,"log":"[2026-02-17T13:59:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 97ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-abcuv0iclx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719424000,"log":"[2026-02-17T13:10:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6525404451280445000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r6h0xeho6s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719425000,"log":"[2026-02-17T12:47:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-my806gljyt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719426000,"log":"[2026-02-17T12:42:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-igj20nynvj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719427000,"log":"[2026-02-17T13:37:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6595391670939542000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n8vfxykce7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719428000,"log":"[2026-02-17T12:52:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ywrbgvjjoe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719429000,"log":"[2026-02-17T15:18:52Z TRACE cache::redis] cache delete key=user:session:9530","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ng9kw3i6bw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719430000,"log":"[2026-02-17T12:59:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-570iriecyy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719431000,"log":"[2026-02-17T14:12:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 458 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4zzc5m9o6k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719432000,"log":"[2026-02-17T12:48:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-br5vj58x9x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719433000,"log":"[2026-02-17T13:24:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2357164092728232400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-svekomz1h8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719434000,"log":"[2026-02-17T13:50:25Z INFO actix_web::middleware::logger] GET /api/_search 200 255ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nzaxqhbdd1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719435000,"log":"[2026-02-17T14:23:28Z DEBUG sqlx::query] INSERT FROM users returned 333 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cvfny22kqk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719436000,"log":"[2026-02-17T13:36:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5921617294172117000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7ogpq72m5l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719437000,"log":"[2026-02-17T15:02:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-prf6kfzmac","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719438000,"log":"[2026-02-17T12:37:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f9tiko6rse","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719439000,"log":"[2026-02-17T14:33:51Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=1pnbupcy","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4g6naoxob2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719440000,"log":"[2026-02-17T13:07:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hvewx5b18g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719441000,"log":"[2026-02-17T14:49:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ci71y7hr74","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719442000,"log":"[2026-02-17T15:16:22Z TRACE cache::redis] cache delete key=user:session:272","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n84voca1n2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719443000,"log":"[2026-02-17T12:45:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r2dq8fj6bd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719444000,"log":"[2026-02-17T15:21:00Z TRACE cache::redis] cache hit key=query:result:4033","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qb9eiav8g8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719445000,"log":"[2026-02-17T14:31:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 188 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y4jjpb5c2o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719446000,"log":"[2026-02-17T13:05:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9ab93m3bmn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719447000,"log":"[2026-02-17T14:20:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 429 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8jo3lfhasc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719448000,"log":"[2026-02-17T12:53:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ates54xx2j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719449000,"log":"[2026-02-17T14:57:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vk724fordu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719450000,"log":"[2026-02-17T14:16:02Z DEBUG sqlx::query] SELECT FROM streams returned 237 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uwu8tf5n7z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719451000,"log":"[2026-02-17T14:37:49Z INFO tonic::transport::server] grpc router request_received client_id=wde6yhzy","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2kht5j8fag","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719452000,"log":"[2026-02-17T14:09:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 402 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3o2gyqncku","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719453000,"log":"[2026-02-17T14:57:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r11lxf6oo2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719454000,"log":"[2026-02-17T12:42:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1p5se9qjtx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719455000,"log":"[2026-02-17T13:15:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7321890556363693000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ovimdihy14","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719456000,"log":"[2026-02-17T13:46:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 145ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pfkg55888g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719457000,"log":"[2026-02-17T13:29:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3091910476154038300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v7cuw09v7b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719458000,"log":"[2026-02-17T13:13:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2787682073372022000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g5u3jdehp3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719459000,"log":"[2026-02-17T14:05:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 210ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oen4zi4xic","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719460000,"log":"[2026-02-17T13:05:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y0uzlelgv4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719461000,"log":"[2026-02-17T13:22:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/438954451457966850.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6tgirpy8ca","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719462000,"log":"[2026-02-17T14:06:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 487ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9a2lfu52eq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719463000,"log":"[2026-02-17T15:17:55Z TRACE cache::redis] cache miss key=query:result:4413","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iun1xqvb64","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719464000,"log":"[2026-02-17T13:44:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 274ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uqt8kuumqh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719465000,"log":"[2026-02-17T13:07:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-os95zm6yd3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719466000,"log":"[2026-02-17T13:23:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4310694237411512000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g7n3oztv9z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719467000,"log":"[2026-02-17T15:01:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-40njuxm7jf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719468000,"log":"[2026-02-17T14:30:25Z DEBUG sqlx::query] DELETE FROM alerts returned 18 rows in 58ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7f0lms5nmn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719469000,"log":"[2026-02-17T12:35:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w7a13digqu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719470000,"log":"[2026-02-17T13:56:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 10ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-50yls26rtd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719471000,"log":"[2026-02-17T13:42:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 264ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nty0top8sh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719472000,"log":"[2026-02-17T13:21:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3562907484415722500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a40c02c1os","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719473000,"log":"[2026-02-17T12:36:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kkh9ukfwz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719474000,"log":"[2026-02-17T12:48:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uqa8vuhs9p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719475000,"log":"[2026-02-17T13:30:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2418488863697099300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yv1slhar5e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719476000,"log":"[2026-02-17T14:06:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 269ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xal2i6sbaa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719477000,"log":"[2026-02-17T15:07:22Z DEBUG metrics::reporter] recorded metric http_requests_total value=4536","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b7eqycjiji","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719478000,"log":"[2026-02-17T12:35:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-acv363f714","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719479000,"log":"[2026-02-17T14:04:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bpd4hbhhvx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719480000,"log":"[2026-02-17T14:41:27Z INFO tonic::transport::server] grpc ingester connected client_id=1qwgt5kw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vlpiynh0ne","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719481000,"log":"[2026-02-17T14:12:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 979 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3xgo4xs2tq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719482000,"log":"[2026-02-17T13:15:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1452823928622835000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4kacz0z55c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719483000,"log":"[2026-02-17T14:46:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=ner48cxx","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fvaaj02b1o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719484000,"log":"[2026-02-17T12:42:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wzi5y4v1t6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719485000,"log":"[2026-02-17T13:52:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 239ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q5o98mnla6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719486000,"log":"[2026-02-17T13:21:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6712143893541819000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-li2guza9h5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719487000,"log":"[2026-02-17T14:11:33Z DEBUG sqlx::query] INSERT FROM users returned 132 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-46vclbrnyc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719488000,"log":"[2026-02-17T14:21:57Z DEBUG sqlx::query] SELECT FROM streams returned 789 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bomnxkwwuo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719489000,"log":"[2026-02-17T15:01:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6od7eqnoko","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719490000,"log":"[2026-02-17T13:37:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4548329522331837400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wch2w7z86d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719491000,"log":"[2026-02-17T13:56:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 343ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-atbmwy4l4d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719492000,"log":"[2026-02-17T13:07:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nd5yzeb6fs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719493000,"log":"[2026-02-17T14:53:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3vty6zmpks","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719494000,"log":"[2026-02-17T14:27:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 790 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qwdvc42xjr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719495000,"log":"[2026-02-17T14:55:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-44br389nhh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719496000,"log":"[2026-02-17T13:34:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1405293235551314700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5tx6dhbicg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719497000,"log":"[2026-02-17T15:13:39Z TRACE cache::redis] cache set key=stream:schema:9949","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xncmhgag1y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719498000,"log":"[2026-02-17T13:04:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-21bxsmazat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719499000,"log":"[2026-02-17T13:08:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9897073859459211000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fnyk9k7rxi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719500000,"log":"[2026-02-17T14:40:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=n480z2aj","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5wv2rtv7do","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719501000,"log":"[2026-02-17T13:01:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f4rgla2m72","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719502000,"log":"[2026-02-17T13:21:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3527941143468509000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1ktrxz2htx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719503000,"log":"[2026-02-17T14:07:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 920 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-stcr1oyhbb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719504000,"log":"[2026-02-17T14:37:13Z INFO tonic::transport::server] grpc querier disconnected client_id=p7og69cf","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hdo6bp1hai","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719505000,"log":"[2026-02-17T13:02:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o82nzzi0wx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719506000,"log":"[2026-02-17T15:02:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bo67ftwecl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719507000,"log":"[2026-02-17T12:52:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y2q8d82bl9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719508000,"log":"[2026-02-17T15:19:07Z TRACE cache::redis] cache miss key=user:session:5077","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kj5sxj2gvp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719509000,"log":"[2026-02-17T14:03:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 128ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nzv9z6qgmd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719510000,"log":"[2026-02-17T14:05:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-imrnm5uedx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719511000,"log":"[2026-02-17T13:11:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5832292997583699000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4ihse367gw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719512000,"log":"[2026-02-17T14:49:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tarmxe30ae","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719513000,"log":"[2026-02-17T13:26:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5705542157549668000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-64bbxwluqs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719514000,"log":"[2026-02-17T14:36:04Z INFO tonic::transport::server] grpc router request_received client_id=e53mw4au","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e4d1aw6d76","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719515000,"log":"[2026-02-17T14:25:22Z DEBUG sqlx::query] SELECT FROM streams returned 133 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i8exxc0hor","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719516000,"log":"[2026-02-17T14:28:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 690 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xqnw1zwai1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719517000,"log":"[2026-02-17T13:36:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7718111686082829000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w85us14fqp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719518000,"log":"[2026-02-17T12:46:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ywjbj1myu6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719519000,"log":"[2026-02-17T13:51:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 312ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qc4ndvja6f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719520000,"log":"[2026-02-17T13:43:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 415ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m9j5xzkv82","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719521000,"log":"[2026-02-17T12:40:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fi6oktfv7z","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719522000,"log":"[2026-02-17T14:56:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jo1tgi4h87","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719523000,"log":"[2026-02-17T13:48:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 333ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u1l7655i4w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719524000,"log":"[2026-02-17T14:53:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xj4kaz07h7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719525000,"log":"[2026-02-17T14:47:33Z INFO tonic::transport::server] grpc querier disconnected client_id=7yvi19xe","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d0khnzactv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719526000,"log":"[2026-02-17T15:06:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8475","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-elposcdn8a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719527000,"log":"[2026-02-17T14:44:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=a989moms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vfcsufkp9w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719528000,"log":"[2026-02-17T13:34:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4687847490157042000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-odtb5uujmw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719529000,"log":"[2026-02-17T12:59:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2gxeij2czv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719530000,"log":"[2026-02-17T13:27:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/38684916429916340.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ujndiyrtf3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719531000,"log":"[2026-02-17T13:17:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2558700559021778000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cdm2aesdfk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719532000,"log":"[2026-02-17T14:47:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=nbmn5to9","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hvbz1z3l6c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719533000,"log":"[2026-02-17T14:56:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h83l4smprh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719534000,"log":"[2026-02-17T14:13:03Z DEBUG sqlx::query] INSERT FROM users returned 736 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sxnyvzi0e3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719535000,"log":"[2026-02-17T12:44:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pk5owuoczg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719536000,"log":"[2026-02-17T13:56:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 172ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kgac66fpd5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719537000,"log":"[2026-02-17T14:22:04Z DEBUG sqlx::query] UPDATE FROM organizations returned 573 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9puph8j57d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719538000,"log":"[2026-02-17T13:03:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-068lknsc70","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719539000,"log":"[2026-02-17T15:20:35Z TRACE cache::redis] cache expire key=query:result:5160","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5ycjmmk7ai","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719540000,"log":"[2026-02-17T13:21:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8451559869711453000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-acertbi6nm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719541000,"log":"[2026-02-17T14:23:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 571 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w2smowv02y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719542000,"log":"[2026-02-17T13:34:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2864520777534958000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eckkscffo1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719543000,"log":"[2026-02-17T13:12:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/978995296067448600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2jz74mbbb0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719544000,"log":"[2026-02-17T13:57:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 261ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yc2or5ut3t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719545000,"log":"[2026-02-17T12:49:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n0xjaqyh14","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719546000,"log":"[2026-02-17T15:17:54Z TRACE cache::redis] cache hit key=stream:schema:4509","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-acs5ptteiw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719547000,"log":"[2026-02-17T13:01:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b3okluc9yy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719548000,"log":"[2026-02-17T14:31:48Z DEBUG sqlx::query] INSERT FROM users returned 351 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ge65x40r6g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719549000,"log":"[2026-02-17T13:26:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/535133356088741700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rbrp769bin","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719550000,"log":"[2026-02-17T14:39:42Z INFO tonic::transport::server] grpc ingester connected client_id=gpf8vreq","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wvmrs7zvvv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719551000,"log":"[2026-02-17T14:26:38Z DEBUG sqlx::query] INSERT FROM users returned 415 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pnwqzer1il","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719552000,"log":"[2026-02-17T13:23:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5777627371260050000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hi2kysq1da","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719553000,"log":"[2026-02-17T14:39:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=hefnll9s","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-derm6x2nin","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719554000,"log":"[2026-02-17T14:34:14Z INFO tonic::transport::server] grpc router request_received client_id=ebxurq2w","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2h5squvkko","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719555000,"log":"[2026-02-17T14:22:23Z DEBUG sqlx::query] INSERT FROM users returned 97 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jfhps0p2cl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719556000,"log":"[2026-02-17T13:33:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3397838113048566000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uxryzsnqcw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719557000,"log":"[2026-02-17T13:03:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f6whdhm2ej","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719558000,"log":"[2026-02-17T14:39:04Z INFO tonic::transport::server] grpc router request_received client_id=e4l86rqt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ofj14tuv67","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719559000,"log":"[2026-02-17T13:15:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2302298191134171100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-241fga2lid","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719560000,"log":"[2026-02-17T14:06:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 70ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bdof4mkeyk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719561000,"log":"[2026-02-17T13:09:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9033446127058378000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8gwpolxx3n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719562000,"log":"[2026-02-17T14:54:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fdr14jx8g6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719563000,"log":"[2026-02-17T12:45:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lsc8m75ba2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719564000,"log":"[2026-02-17T12:52:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ulyswrzqts","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719565000,"log":"[2026-02-17T12:44:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ql10x596cn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719566000,"log":"[2026-02-17T14:44:47Z INFO tonic::transport::server] grpc ingester connected client_id=yf0db14u","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y8eqmpnayt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719567000,"log":"[2026-02-17T12:40:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-twt6vps7ae","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719568000,"log":"[2026-02-17T13:01:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-npwn733wqi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719569000,"log":"[2026-02-17T14:33:17Z INFO tonic::transport::server] grpc ingester connected client_id=fw7djh3e","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q890n25vxm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719570000,"log":"[2026-02-17T15:04:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5d9kn3zg4u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719571000,"log":"[2026-02-17T13:36:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2635042618464058400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-keomoe9yv4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719572000,"log":"[2026-02-17T13:14:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9880824994172088000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qo9m0m6sjr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719573000,"log":"[2026-02-17T14:49:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-35j3lm7wr0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719574000,"log":"[2026-02-17T14:01:25Z INFO actix_web::middleware::logger] GET /api/_search 400 242ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yylrakgbbs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719575000,"log":"[2026-02-17T14:13:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 286 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6nycyumiq8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719576000,"log":"[2026-02-17T15:01:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3i8gj0zc3y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719577000,"log":"[2026-02-17T13:13:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1428452710072813800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h5cznwp31o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719578000,"log":"[2026-02-17T15:06:40Z DEBUG metrics::reporter] recorded metric active_connections value=6062","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ypw8tddg4t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719579000,"log":"[2026-02-17T13:33:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2574532265865834500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x22uzoj2kq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719580000,"log":"[2026-02-17T13:57:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 403 20ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tbly852hf9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719581000,"log":"[2026-02-17T13:32:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9586471049160395000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-orksvfl892","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719582000,"log":"[2026-02-17T14:25:58Z DEBUG sqlx::query] INSERT FROM users returned 587 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zys1cnausv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719583000,"log":"[2026-02-17T14:28:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 695 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r87wgdysm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719584000,"log":"[2026-02-17T14:52:22Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qh7l0yesz6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719585000,"log":"[2026-02-17T14:11:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 687 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9f63my4irb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719586000,"log":"[2026-02-17T13:42:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 218ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pojpn8qa55","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719587000,"log":"[2026-02-17T14:57:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d056ctylmj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719588000,"log":"[2026-02-17T12:58:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m06rcaaa83","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719589000,"log":"[2026-02-17T14:06:52Z DEBUG sqlx::query] SELECT FROM streams returned 43 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ptxnhbi2pa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719590000,"log":"[2026-02-17T13:12:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2940494835263156700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2jaegtcth6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719591000,"log":"[2026-02-17T13:36:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6037083064248918000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b0swporo5c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719592000,"log":"[2026-02-17T13:32:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9738125736725103000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4zf815hbjg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719593000,"log":"[2026-02-17T14:12:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 783 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0cs96vkgff","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719594000,"log":"[2026-02-17T14:45:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=x9nh956s","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2ao0qnbrt5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719595000,"log":"[2026-02-17T14:09:10Z DEBUG sqlx::query] DELETE FROM alerts returned 577 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gg0osoqcm6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719596000,"log":"[2026-02-17T13:32:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7536356388671570000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cokfrsu6t8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719597000,"log":"[2026-02-17T12:54:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hg8p121jra","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719598000,"log":"[2026-02-17T12:43:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qd7ipvd4kb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719599000,"log":"[2026-02-17T15:10:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=7843","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gmb1ta3bl4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719600000,"log":"[2026-02-17T14:16:12Z DEBUG sqlx::query] SELECT FROM streams returned 486 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u3loqlnlnn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719601000,"log":"[2026-02-17T14:01:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 400 239ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ip9tl08x44","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719602000,"log":"[2026-02-17T13:10:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6738722177595790000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z5ehucclpd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719603000,"log":"[2026-02-17T14:30:20Z DEBUG sqlx::query] DELETE FROM alerts returned 875 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5z2r6sbr18","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719604000,"log":"[2026-02-17T14:26:18Z DEBUG sqlx::query] INSERT FROM users returned 756 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lfmliqvxo1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719605000,"log":"[2026-02-17T14:31:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 454 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-px87ugclht","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719606000,"log":"[2026-02-17T13:23:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7473195550252578000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-75ixqcukpp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719607000,"log":"[2026-02-17T14:10:53Z DEBUG sqlx::query] INSERT FROM users returned 369 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-379uyikaee","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719608000,"log":"[2026-02-17T14:55:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hgzyisnphf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719609000,"log":"[2026-02-17T12:42:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-77s53prx9y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719610000,"log":"[2026-02-17T14:42:24Z INFO tonic::transport::server] grpc router request_received client_id=z6d6s4a6","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gvhbwautvn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719611000,"log":"[2026-02-17T14:35:48Z INFO tonic::transport::server] grpc querier disconnected client_id=x3gknbvv","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gckg06ln7l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719612000,"log":"[2026-02-17T13:12:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4731025039331162000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-czs3efzgx6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719613000,"log":"[2026-02-17T13:00:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dy9fa0nh7y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719614000,"log":"[2026-02-17T13:51:45Z INFO actix_web::middleware::logger] GET /api/_search 503 45ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-32dmfmk60z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719615000,"log":"[2026-02-17T12:59:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-naeu0rez1t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719616000,"log":"[2026-02-17T14:59:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mvwxx662z9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719617000,"log":"[2026-02-17T14:37:23Z INFO tonic::transport::server] grpc querier disconnected client_id=bh1t05ga","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-itfbcq68r0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719618000,"log":"[2026-02-17T13:57:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 364ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x3gmrjo1jo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719619000,"log":"[2026-02-17T12:54:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kk9r43fk2v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719620000,"log":"[2026-02-17T12:47:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fgtskn3weh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719621000,"log":"[2026-02-17T15:03:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-94800kj7dv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719622000,"log":"[2026-02-17T13:45:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 238ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m2ae9tpla6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719623000,"log":"[2026-02-17T14:38:24Z INFO tonic::transport::server] grpc router request_received client_id=pkp7wcyv","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2g8a2md0vk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719624000,"log":"[2026-02-17T12:36:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2ivolczv89","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719625000,"log":"[2026-02-17T14:05:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 494ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pjvewc0hgx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719626000,"log":"[2026-02-17T13:35:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2677667589301613000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wj5kdpw4n5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719627000,"log":"[2026-02-17T12:39:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1uwz02gv9b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719628000,"log":"[2026-02-17T14:12:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 107 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g2tb7gdj6k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719629000,"log":"[2026-02-17T13:07:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f96xohmz08","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719630000,"log":"[2026-02-17T13:39:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4365713487019596000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9ym7w5agh1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719631000,"log":"[2026-02-17T14:54:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w0cgd9icuc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719632000,"log":"[2026-02-17T14:56:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fj1xkeyhcx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719633000,"log":"[2026-02-17T12:51:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cjaetzpr03","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719634000,"log":"[2026-02-17T14:11:35Z DEBUG sqlx::query] DELETE FROM alerts returned 687 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fzmm0dvt73","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719635000,"log":"[2026-02-17T13:26:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/86794773587961500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-varalvdguu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719636000,"log":"[2026-02-17T13:40:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7542251750769506000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6nk3gu5y1a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719637000,"log":"[2026-02-17T14:57:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m3pe7izmjm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719638000,"log":"[2026-02-17T14:44:32Z INFO tonic::transport::server] grpc ingester connected client_id=rod8gsrw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-okhtr4kayh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719639000,"log":"[2026-02-17T13:16:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3039105744048060400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x3mqbr4l2m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719640000,"log":"[2026-02-17T13:14:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5960332674347468000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a4oeogs8j1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719641000,"log":"[2026-02-17T13:13:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3203156408056447000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ur7rb655yg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719642000,"log":"[2026-02-17T12:59:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gkt9kynlf3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719643000,"log":"[2026-02-17T15:06:55Z DEBUG metrics::reporter] recorded metric active_connections value=618","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lye5pj0o56","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719644000,"log":"[2026-02-17T13:06:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8swq6ecors","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719645000,"log":"[2026-02-17T15:05:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6060","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-18pofk6m9u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719646000,"log":"[2026-02-17T13:00:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kzf2aesq0a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719647000,"log":"[2026-02-17T13:27:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5113642289358300000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rpkcav2ag2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719648000,"log":"[2026-02-17T13:36:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5977632170155873000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ljf0pkuogw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719649000,"log":"[2026-02-17T14:58:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eeqf18ltcp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719650000,"log":"[2026-02-17T13:01:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r7yzy4ac38","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719651000,"log":"[2026-02-17T15:07:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5786","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zbkvp7qy2p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719652000,"log":"[2026-02-17T12:43:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ip31j3f90e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719653000,"log":"[2026-02-17T13:06:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ftddomr02c","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719654000,"log":"[2026-02-17T14:34:04Z INFO tonic::transport::server] grpc router request_received client_id=54t3s2k4","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x2c45iwyi2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719655000,"log":"[2026-02-17T15:16:37Z TRACE cache::redis] cache miss key=user:session:111","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-we2j6nb9y3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719656000,"log":"[2026-02-17T13:59:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 169ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-74aemldkn9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719657000,"log":"[2026-02-17T14:52:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cfq0ebztqj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719658000,"log":"[2026-02-17T13:00:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qp8tr4mshg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719659000,"log":"[2026-02-17T12:42:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1vjd4uh6id","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719660000,"log":"[2026-02-17T13:32:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1945979823756862700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2aroawuekb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719661000,"log":"[2026-02-17T14:30:15Z DEBUG sqlx::query] DELETE FROM alerts returned 776 rows in 61ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3wu2490aok","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719662000,"log":"[2026-02-17T13:50:05Z INFO actix_web::middleware::logger] GET /api/_search 500 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4r57vjm8km","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719663000,"log":"[2026-02-17T14:43:44Z INFO tonic::transport::server] grpc router request_received client_id=9uocwwr2","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nc3n4p7svk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719664000,"log":"[2026-02-17T15:08:27Z DEBUG metrics::reporter] recorded metric http_requests_total value=6022","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dmd1erofsp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719665000,"log":"[2026-02-17T12:48:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-opom2hqxba","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719666000,"log":"[2026-02-17T14:02:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 421ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u6xm13xn0x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719667000,"log":"[2026-02-17T14:06:20Z INFO actix_web::middleware::logger] POST /api/_search 201 311ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-quai0l5y2x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719668000,"log":"[2026-02-17T13:43:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 490ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tzhus6cen2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719669000,"log":"[2026-02-17T12:51:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-90ktued5um","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719670000,"log":"[2026-02-17T15:07:30Z DEBUG metrics::reporter] recorded metric active_connections value=4015","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-84bhs96wfg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719671000,"log":"[2026-02-17T12:47:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hfu4eezc5m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719672000,"log":"[2026-02-17T12:49:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-47aw1axqrg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719673000,"log":"[2026-02-17T13:50:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 296ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-opy69ayj9f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719674000,"log":"[2026-02-17T12:46:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kesncn30es","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719675000,"log":"[2026-02-17T13:43:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 48ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dcdhl4yqai","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719676000,"log":"[2026-02-17T14:27:47Z DEBUG sqlx::query] SELECT FROM streams returned 991 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zaut7k7nyd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719677000,"log":"[2026-02-17T13:31:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3896877928336712700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n0pw62d3d4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719678000,"log":"[2026-02-17T13:46:45Z INFO actix_web::middleware::logger] GET /api/_search 403 74ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jvrfrlbggu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719679000,"log":"[2026-02-17T13:47:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 121ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0fhsc75y70","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719680000,"log":"[2026-02-17T13:12:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3413076468477083000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-344jg1dm5e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719681000,"log":"[2026-02-17T15:02:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5jvmwa5aqf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719682000,"log":"[2026-02-17T15:17:13Z TRACE cache::redis] cache miss key=org:config:7195","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r5kdnlzui1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719683000,"log":"[2026-02-17T14:07:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 812 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-099j36frmw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719684000,"log":"[2026-02-17T14:48:07Z INFO tonic::transport::server] grpc ingester connected client_id=ng5pgsqv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-svgge8ubce","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719685000,"log":"[2026-02-17T12:48:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x1ce8i51fh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719686000,"log":"[2026-02-17T14:35:32Z INFO tonic::transport::server] grpc ingester connected client_id=pn5rwgg5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fpsyufloj1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719687000,"log":"[2026-02-17T15:18:34Z TRACE cache::redis] cache delete key=stream:schema:4303","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h7ul3lao70","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719688000,"log":"[2026-02-17T15:00:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-djy4zigwci","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719689000,"log":"[2026-02-17T13:49:05Z INFO actix_web::middleware::logger] GET /api/_search 201 483ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3dgye0bbvj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719690000,"log":"[2026-02-17T15:15:04Z TRACE cache::redis] cache delete key=stream:schema:89","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-it8dt1jc5k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719691000,"log":"[2026-02-17T13:57:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wvsyqli8bj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719692000,"log":"[2026-02-17T12:46:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g2jmurk8x2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719693000,"log":"[2026-02-17T12:39:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wwb45v2czy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719694000,"log":"[2026-02-17T14:20:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 959 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n667qzq2v6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719695000,"log":"[2026-02-17T12:43:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7gd6xtdes0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719696000,"log":"[2026-02-17T14:15:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 848 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5j2az678ol","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719697000,"log":"[2026-02-17T13:22:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7664128780913812000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9snae2glxz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719698000,"log":"[2026-02-17T14:46:04Z INFO tonic::transport::server] grpc router request_received client_id=t5gw2p68","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-al9i0le4oc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719699000,"log":"[2026-02-17T12:48:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ipnbccmi2j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719700000,"log":"[2026-02-17T12:51:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-afepp124ta","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719701000,"log":"[2026-02-17T14:12:22Z DEBUG sqlx::query] SELECT FROM streams returned 4 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0svcsos0q2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719702000,"log":"[2026-02-17T14:37:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=5k9or213","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mx6ql1gzxn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719703000,"log":"[2026-02-17T12:57:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nspkccr8dr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719704000,"log":"[2026-02-17T14:14:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 633 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m14nefdutm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719705000,"log":"[2026-02-17T13:44:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 204 162ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mfaua6uqjt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719706000,"log":"[2026-02-17T12:56:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-of1kp2bc4z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719707000,"log":"[2026-02-17T15:18:19Z TRACE cache::redis] cache miss key=stream:schema:2165","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9i284e3kri","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719708000,"log":"[2026-02-17T14:00:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 9ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-82i30rojov","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719709000,"log":"[2026-02-17T13:26:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1872381776517477600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2eul1r2vt0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719710000,"log":"[2026-02-17T15:07:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=5875","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1f8qv8rlda","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719711000,"log":"[2026-02-17T12:59:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tu0tzh1a34","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719712000,"log":"[2026-02-17T13:49:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 40ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zdz8mwqbsx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719713000,"log":"[2026-02-17T14:20:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 885 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9ujuxn7gf5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719714000,"log":"[2026-02-17T13:17:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1972116790258788900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zcodd6slog","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719715000,"log":"[2026-02-17T13:57:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 261ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q4n349ubxc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719716000,"log":"[2026-02-17T13:55:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 460ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-scoxtah5fh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719717000,"log":"[2026-02-17T13:24:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/550232892011253100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6b48iqr1hg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719718000,"log":"[2026-02-17T15:19:10Z TRACE cache::redis] cache delete key=query:result:7384","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9uro0gz4xq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719719000,"log":"[2026-02-17T14:37:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=auon32gh","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y1ejxc50bb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719720000,"log":"[2026-02-17T13:13:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3587985507834434600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-55c1domy9q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719721000,"log":"[2026-02-17T13:17:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7717939966119005000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cwr4kuhdlm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719722000,"log":"[2026-02-17T14:24:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 366 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l82lhawix6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719723000,"log":"[2026-02-17T15:18:00Z TRACE cache::redis] cache hit key=query:result:2367","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vw1ikoslzj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719724000,"log":"[2026-02-17T13:14:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7925352978996476000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dk4tikic8f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719725000,"log":"[2026-02-17T12:46:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c36bz1s0sp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719726000,"log":"[2026-02-17T14:31:52Z INFO tonic::transport::server] grpc ingester connected client_id=gnmm3bwz","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-joap5qjxyt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719727000,"log":"[2026-02-17T15:04:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-50adz44gq7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719728000,"log":"[2026-02-17T14:52:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rdt8zv9c1s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719729000,"log":"[2026-02-17T14:18:13Z DEBUG sqlx::query] INSERT FROM users returned 102 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ls8czz64kv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719730000,"log":"[2026-02-17T12:45:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wl0x3ukm5f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719731000,"log":"[2026-02-17T13:17:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5946383904711412000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jfo4hz2umh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719732000,"log":"[2026-02-17T14:19:55Z DEBUG sqlx::query] DELETE FROM alerts returned 319 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8bl2phiy4t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719733000,"log":"[2026-02-17T15:11:50Z DEBUG metrics::reporter] recorded metric active_connections value=1558","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s69ptv9nj3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719734000,"log":"[2026-02-17T12:45:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oo19vnlx48","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719735000,"log":"[2026-02-17T14:45:48Z INFO tonic::transport::server] grpc querier disconnected client_id=vos6hmhv","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-az2kywbetf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719736000,"log":"[2026-02-17T13:25:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9845874972153830000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hwuwlu82r0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719737000,"log":"[2026-02-17T15:16:23Z TRACE cache::redis] cache expire key=org:config:2942","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-irs1hljqpd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719738000,"log":"[2026-02-17T15:15:46Z TRACE cache::redis] cache delete key=auth:token:6510","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4qknk0rf6t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719739000,"log":"[2026-02-17T15:02:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-exwzafx7zk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719740000,"log":"[2026-02-17T12:53:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zgwp0z4bpg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719741000,"log":"[2026-02-17T13:23:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/275545774568053400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cszwzm6lp4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719742000,"log":"[2026-02-17T15:16:15Z TRACE cache::redis] cache set key=query:result:1488","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-29ee9jcpem","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719743000,"log":"[2026-02-17T12:55:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5xs4yowij4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719744000,"log":"[2026-02-17T14:31:37Z DEBUG sqlx::query] SELECT FROM streams returned 772 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w0mq3i6qke","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719745000,"log":"[2026-02-17T14:09:17Z DEBUG sqlx::query] SELECT FROM streams returned 222 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m4h32c4tvc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719746000,"log":"[2026-02-17T12:49:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1f7znbxkay","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719747000,"log":"[2026-02-17T13:05:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jeh0zet12b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719748000,"log":"[2026-02-17T14:04:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 503 451ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mxuqgbfs9q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719749000,"log":"[2026-02-17T12:43:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pv8bto7jpo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719750000,"log":"[2026-02-17T14:22:40Z DEBUG sqlx::query] DELETE FROM alerts returned 135 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t4hrzodlnj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719751000,"log":"[2026-02-17T15:03:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b8vs3gkb83","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719752000,"log":"[2026-02-17T15:19:03Z TRACE cache::redis] cache set key=org:config:5741","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qy1badkdc0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719753000,"log":"[2026-02-17T13:37:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1756973798376009200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gq1qdyxeen","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719754000,"log":"[2026-02-17T13:20:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6189440900900274000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-45ukvffugu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719755000,"log":"[2026-02-17T12:46:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ny69tgvsao","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719756000,"log":"[2026-02-17T14:34:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=cqo5b8el","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h1zyr0vo41","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719757000,"log":"[2026-02-17T13:20:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1651260618465029000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dnxo67gr5u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719758000,"log":"[2026-02-17T14:17:33Z DEBUG sqlx::query] INSERT FROM users returned 799 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yg3ztddzim","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719759000,"log":"[2026-02-17T15:18:55Z TRACE cache::redis] cache miss key=query:result:1120","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sj3f0qmyms","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719760000,"log":"[2026-02-17T13:11:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4056674291607813600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c3zdpyywg7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719761000,"log":"[2026-02-17T13:43:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 255ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-psshebnko7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719762000,"log":"[2026-02-17T15:06:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=2963","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ymsz8v1y7b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719763000,"log":"[2026-02-17T14:39:47Z INFO tonic::transport::server] grpc ingester connected client_id=rq9cqz9x","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-24cnnu12ia","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719764000,"log":"[2026-02-17T13:14:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4110824807042663400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wiuihaapit","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719765000,"log":"[2026-02-17T15:01:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tfup382aub","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719766000,"log":"[2026-02-17T15:10:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=2827","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vuq44lim6z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719767000,"log":"[2026-02-17T14:35:34Z INFO tonic::transport::server] grpc router request_received client_id=fi2uggu7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fzf61xz857","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719768000,"log":"[2026-02-17T12:47:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wl6chw71ey","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719769000,"log":"[2026-02-17T14:00:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 370ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-14bnkc7mz6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719770000,"log":"[2026-02-17T14:22:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 864 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g82s8ezd68","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719771000,"log":"[2026-02-17T13:54:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 389ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-owjdi5iwco","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719772000,"log":"[2026-02-17T14:22:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 92 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u4e2kqrzyg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719773000,"log":"[2026-02-17T13:49:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 13ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5la9pbawlz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719774000,"log":"[2026-02-17T13:43:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 80ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uux2bsw0kc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719775000,"log":"[2026-02-17T14:29:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 341 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ngkyavfe3k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719776000,"log":"[2026-02-17T13:22:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1261889022164055600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-25bsyci0f0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719777000,"log":"[2026-02-17T13:15:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9675173174644689000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hyqp2w436y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719778000,"log":"[2026-02-17T13:21:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2421929806406406700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8yph7uo3ex","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719779000,"log":"[2026-02-17T14:34:33Z INFO tonic::transport::server] grpc querier disconnected client_id=jo1xalhm","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yyjxl95jir","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719780000,"log":"[2026-02-17T13:55:25Z INFO actix_web::middleware::logger] GET /api/_search 400 136ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b92wfn9uhx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719781000,"log":"[2026-02-17T14:56:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-khnfwhjdmh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719782000,"log":"[2026-02-17T12:57:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kpnivh7mj2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719783000,"log":"[2026-02-17T14:27:52Z DEBUG sqlx::query] SELECT FROM streams returned 661 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rfa9p35os4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719784000,"log":"[2026-02-17T14:26:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 97 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4yhmzainy4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719785000,"log":"[2026-02-17T14:19:22Z DEBUG sqlx::query] SELECT FROM streams returned 663 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9udgrq9npa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719786000,"log":"[2026-02-17T13:27:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8861917929334311000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-us6iiktvhp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719787000,"log":"[2026-02-17T14:28:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 671 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rid669xgxd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719788000,"log":"[2026-02-17T15:13:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=6093","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fetxhu5h0t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719789000,"log":"[2026-02-17T14:15:00Z DEBUG sqlx::query] DELETE FROM alerts returned 479 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p2fypxsmui","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719790000,"log":"[2026-02-17T14:34:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=k8dwwwls","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ixvwi2t88i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719791000,"log":"[2026-02-17T14:05:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 309ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f2yn2p6suy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719792000,"log":"[2026-02-17T13:53:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 493ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-11sme4rqt7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719793000,"log":"[2026-02-17T14:31:53Z INFO tonic::transport::server] grpc querier disconnected client_id=k5q4i4k1","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-czr4lj2g6f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719794000,"log":"[2026-02-17T14:56:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0wfauqg5hw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719795000,"log":"[2026-02-17T12:38:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ljceseenmm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719796000,"log":"[2026-02-17T13:31:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/339457103778169100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tt3zkcxnp4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719797000,"log":"[2026-02-17T13:50:21Z INFO actix_web::middleware::logger] GET /api/_bulk 403 457ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-igo3wfog2g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719798000,"log":"[2026-02-17T13:29:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1675407678493141500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iw5qo2cedi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719799000,"log":"[2026-02-17T15:11:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=6571","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1txb2vhalu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719800000,"log":"[2026-02-17T13:37:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2026865416917058600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pisp774nr1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719801000,"log":"[2026-02-17T13:49:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 14ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-93b22unmsu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719802000,"log":"[2026-02-17T14:39:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ieu8e862","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vsz0ile0z9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719803000,"log":"[2026-02-17T13:23:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4747596412580057000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y14qosn6d0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719804000,"log":"[2026-02-17T13:07:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qu41zlw3wu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719805000,"log":"[2026-02-17T14:09:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 807 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q8p6ey9h1s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719806000,"log":"[2026-02-17T13:45:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 200 448ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-auibs6mknt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719807000,"log":"[2026-02-17T13:10:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/179680460468678600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ofa423h5qh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719808000,"log":"[2026-02-17T13:03:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-axjsd45017","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719809000,"log":"[2026-02-17T13:02:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c7ni8ve7se","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719810000,"log":"[2026-02-17T14:48:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i0l40b5l9v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719811000,"log":"[2026-02-17T14:42:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=tblpdpq5","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4fgg64u97q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719812000,"log":"[2026-02-17T13:39:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4991930539924012000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-krekduu66d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719813000,"log":"[2026-02-17T13:18:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6559654187322817000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t76j382wn8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719814000,"log":"[2026-02-17T15:04:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2oms4nhihs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719815000,"log":"[2026-02-17T14:24:22Z DEBUG sqlx::query] SELECT FROM streams returned 914 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t1izp54iu3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719816000,"log":"[2026-02-17T14:03:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 359ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r41ltz1j6w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719817000,"log":"[2026-02-17T13:02:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-klnh2c7b6j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719818000,"log":"[2026-02-17T13:56:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 314ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wwmpbs8zvp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719819000,"log":"[2026-02-17T12:57:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zf2m362gco","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719820000,"log":"[2026-02-17T12:59:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xl2mkb0pkl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719821000,"log":"[2026-02-17T13:59:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 150ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a50cla4rat","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719822000,"log":"[2026-02-17T13:19:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1644347387455826700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cfnxwktn00","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719823000,"log":"[2026-02-17T15:15:27Z TRACE cache::redis] cache set key=user:session:2263","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g8lliw2jt6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719824000,"log":"[2026-02-17T13:56:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 176ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x8sbsyf7wz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719825000,"log":"[2026-02-17T13:35:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5678147284711911000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nf0yi3lj8u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719826000,"log":"[2026-02-17T13:48:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 107ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m1cd1uf0rs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719827000,"log":"[2026-02-17T15:16:01Z TRACE cache::redis] cache miss key=auth:token:9978","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7lcxobr3y9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719828000,"log":"[2026-02-17T15:01:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rtzrx8vqiq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719829000,"log":"[2026-02-17T12:41:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ptkyxjmr7f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719830000,"log":"[2026-02-17T13:46:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 422ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-um14sxvtv5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719831000,"log":"[2026-02-17T13:44:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 166ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zu3l4ojkr3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719832000,"log":"[2026-02-17T12:38:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-axer24q4e9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719833000,"log":"[2026-02-17T14:47:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=vfz1c07n","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qurrm60rvo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719834000,"log":"[2026-02-17T13:37:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8094207616014935000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jirenpe4ae","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719835000,"log":"[2026-02-17T13:36:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7812482786338382000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c8roqxmf02","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719836000,"log":"[2026-02-17T12:56:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bze0jk3wcc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719837000,"log":"[2026-02-17T13:29:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1822886888119159600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1wt6chckn3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719838000,"log":"[2026-02-17T13:42:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 178ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5g7720zibo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719839000,"log":"[2026-02-17T13:14:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6920941180029978000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x0cov8i6ls","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719840000,"log":"[2026-02-17T13:11:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2490371349579949600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o6sgnrft3k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719841000,"log":"[2026-02-17T14:15:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 981 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eo7u3vxf8w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719842000,"log":"[2026-02-17T13:14:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1661788288358272500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fm131fi47q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719843000,"log":"[2026-02-17T15:15:38Z TRACE cache::redis] cache get key=org:config:9401","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uzbk7luywv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719844000,"log":"[2026-02-17T14:37:04Z INFO tonic::transport::server] grpc router request_received client_id=nzu1lhc0","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pou20rna3g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719845000,"log":"[2026-02-17T12:36:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6fzlpu04u8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719846000,"log":"[2026-02-17T12:58:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gfobjn37jq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719847000,"log":"[2026-02-17T13:02:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xll1vx9vkr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719848000,"log":"[2026-02-17T13:17:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6400503788015446000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b64pchovt5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719849000,"log":"[2026-02-17T14:01:45Z INFO actix_web::middleware::logger] GET /api/_search 403 94ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wxda82h7l9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719850000,"log":"[2026-02-17T13:27:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3027321024544211500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mtuz3i0pr7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719851000,"log":"[2026-02-17T14:58:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8mthu1znsn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719852000,"log":"[2026-02-17T13:51:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 1ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-83djfn3dty","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719853000,"log":"[2026-02-17T14:15:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 829 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7ydc8shmim","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719854000,"log":"[2026-02-17T15:18:01Z TRACE cache::redis] cache miss key=auth:token:4347","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-904zk9s3pe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719855000,"log":"[2026-02-17T13:28:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1838457234089667600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tnmy23f99n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719856000,"log":"[2026-02-17T14:02:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 22ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c33mlad510","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719857000,"log":"[2026-02-17T14:25:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 378 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-68ic72ge5c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719858000,"log":"[2026-02-17T13:49:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 57ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yoojhyjqmg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719859000,"log":"[2026-02-17T12:59:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-633ksixj88","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719860000,"log":"[2026-02-17T14:51:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-erfikoaxez","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719861000,"log":"[2026-02-17T13:47:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 154ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jzy6slydfm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719862000,"log":"[2026-02-17T13:58:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 200 24ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4f7mlx052c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719863000,"log":"[2026-02-17T12:52:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q0soco8i84","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719864000,"log":"[2026-02-17T13:17:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3755276613062501400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-40pvakuri3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719865000,"log":"[2026-02-17T14:11:20Z DEBUG sqlx::query] DELETE FROM alerts returned 294 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q91esk7ctb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719866000,"log":"[2026-02-17T13:44:20Z INFO actix_web::middleware::logger] POST /api/_search 401 348ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-o3dm0jtgox","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719867000,"log":"[2026-02-17T13:12:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7922611562682897000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hwr814frbc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719868000,"log":"[2026-02-17T13:26:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4350686487647721500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m7183txiav","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719869000,"log":"[2026-02-17T14:13:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 785 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b1fb45zfy0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719870000,"log":"[2026-02-17T15:14:04Z TRACE cache::redis] cache delete key=stream:schema:1863","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9liggrleib","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719871000,"log":"[2026-02-17T12:45:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jieojv38cr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719872000,"log":"[2026-02-17T14:37:19Z INFO tonic::transport::server] grpc router request_received client_id=3cw9skgn","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aaly2yiy4o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719873000,"log":"[2026-02-17T13:45:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 370ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d899yfbs7q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719874000,"log":"[2026-02-17T13:54:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 111ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i158uwf5gs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719875000,"log":"[2026-02-17T14:29:58Z DEBUG sqlx::query] INSERT FROM users returned 648 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mb6jo4dyc3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719876000,"log":"[2026-02-17T15:07:16Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3693","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-faopkcq63h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719877000,"log":"[2026-02-17T14:18:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 839 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q8sv6ol9qg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719878000,"log":"[2026-02-17T14:44:33Z INFO tonic::transport::server] grpc querier disconnected client_id=1k4bb3a8","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ejsdl6jrt2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719879000,"log":"[2026-02-17T14:58:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fjxk7gxjp8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719880000,"log":"[2026-02-17T13:07:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-96m2p8xlxg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719881000,"log":"[2026-02-17T14:40:24Z INFO tonic::transport::server] grpc router request_received client_id=to50ncqf","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-52tj4pgbx1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719882000,"log":"[2026-02-17T14:44:08Z INFO tonic::transport::server] grpc querier disconnected client_id=ubak5q75","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-webtq7vw75","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719883000,"log":"[2026-02-17T14:08:02Z DEBUG sqlx::query] SELECT FROM streams returned 854 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gm3lqj5tke","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719884000,"log":"[2026-02-17T12:36:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qqcebd6o0w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719885000,"log":"[2026-02-17T13:37:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8623871834933510000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o2dcm42m4e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719886000,"log":"[2026-02-17T12:35:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-emwblhvy0q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719887000,"log":"[2026-02-17T13:01:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-092hn2eb4o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719888000,"log":"[2026-02-17T14:58:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5542owvhqh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719889000,"log":"[2026-02-17T13:28:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8956757714730887000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2a2psvz4gp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719890000,"log":"[2026-02-17T13:18:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/621608531637545200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t2ahul0xlv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719891000,"log":"[2026-02-17T13:47:40Z INFO actix_web::middleware::logger] POST /api/_search 404 445ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tgq2lxdqro","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719892000,"log":"[2026-02-17T14:41:12Z INFO tonic::transport::server] grpc ingester connected client_id=snskushi","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wfauawuqju","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719893000,"log":"[2026-02-17T14:45:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=nkdvyun9","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-95dwkqjpcx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719894000,"log":"[2026-02-17T13:27:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2981139660030021000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gfzrnduegc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719895000,"log":"[2026-02-17T13:02:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1yxb3wtbsv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719896000,"log":"[2026-02-17T14:23:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 766 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pgznphtt07","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719897000,"log":"[2026-02-17T12:55:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-epladcr1iy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719898000,"log":"[2026-02-17T14:42:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=f8jdvmrq","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w8kdh7wz4q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719899000,"log":"[2026-02-17T13:24:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8068469044198639000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bewknr8wr1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719900000,"log":"[2026-02-17T14:09:43Z DEBUG sqlx::query] INSERT FROM users returned 630 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-inhdbstgxl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719901000,"log":"[2026-02-17T14:23:38Z DEBUG sqlx::query] INSERT FROM users returned 668 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yoymobnofd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719902000,"log":"[2026-02-17T13:23:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3352979304780306000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-umnagos7px","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719903000,"log":"[2026-02-17T13:01:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qo0iz0063t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719904000,"log":"[2026-02-17T14:12:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 341 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jq9du5mdxk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719905000,"log":"[2026-02-17T15:15:19Z TRACE cache::redis] cache miss key=stream:schema:6323","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xnwfbybfik","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719906000,"log":"[2026-02-17T14:36:19Z INFO tonic::transport::server] grpc router request_received client_id=ebmfq5qm","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6r9ncr9mwa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719907000,"log":"[2026-02-17T13:32:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/624269747784869100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i7hm0zsf3k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719908000,"log":"[2026-02-17T15:12:00Z DEBUG metrics::reporter] recorded metric active_connections value=7567","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-glkizp1yp0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719909000,"log":"[2026-02-17T14:47:32Z INFO tonic::transport::server] grpc ingester connected client_id=eljd4zko","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v6mrerrl14","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719910000,"log":"[2026-02-17T14:31:30Z DEBUG sqlx::query] DELETE FROM alerts returned 479 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yaukqqq2rm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719911000,"log":"[2026-02-17T12:39:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ozkxefcyqn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719912000,"log":"[2026-02-17T14:20:00Z DEBUG sqlx::query] DELETE FROM alerts returned 706 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g0xw2v0jry","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719913000,"log":"[2026-02-17T14:58:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lz42ogh9tg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719914000,"log":"[2026-02-17T13:19:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3646076815256800000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2jj6p9cyo2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719915000,"log":"[2026-02-17T13:41:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7113969242998798000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6xlvr73mqx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719916000,"log":"[2026-02-17T14:16:33Z DEBUG sqlx::query] INSERT FROM users returned 96 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pl0tkk4f2f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719917000,"log":"[2026-02-17T13:05:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4q1g42co9a","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719918000,"log":"[2026-02-17T13:40:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2098664478313199000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7uv6wmjnmz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719919000,"log":"[2026-02-17T12:49:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-34za4rsh9o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719920000,"log":"[2026-02-17T13:22:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6913486975651504000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-488a2kzn1j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719921000,"log":"[2026-02-17T13:22:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4017949285539724000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-udirlfcjgv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719922000,"log":"[2026-02-17T13:49:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 200 387ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-t86ynidnfx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719923000,"log":"[2026-02-17T14:22:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 935 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f87c83vxoh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719924000,"log":"[2026-02-17T13:57:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 489ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4s7pt19j3e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719925000,"log":"[2026-02-17T13:56:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 123ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jetrhd8iag","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719926000,"log":"[2026-02-17T14:56:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-410d13c5vl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719927000,"log":"[2026-02-17T15:19:00Z TRACE cache::redis] cache hit key=query:result:5639","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uejxavrn6g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719928000,"log":"[2026-02-17T14:51:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8qqwt8r728","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719929000,"log":"[2026-02-17T14:07:02Z DEBUG sqlx::query] SELECT FROM streams returned 953 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-do8oud4h2y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719930000,"log":"[2026-02-17T15:08:30Z DEBUG metrics::reporter] recorded metric active_connections value=6102","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-upalm97s79","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719931000,"log":"[2026-02-17T12:56:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ps20emntty","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719932000,"log":"[2026-02-17T14:25:37Z DEBUG sqlx::query] SELECT FROM streams returned 492 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-63vqxjbvzs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719933000,"log":"[2026-02-17T13:39:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1409693358740440300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nzwb893mw0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719934000,"log":"[2026-02-17T14:58:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-26ni60ph1t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719935000,"log":"[2026-02-17T13:41:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1376898487562261500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t5pjvolbhi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719936000,"log":"[2026-02-17T13:50:00Z INFO actix_web::middleware::logger] POST /api/_search 204 433ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4se56kza8u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719937000,"log":"[2026-02-17T12:53:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fx1q52nxxm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719938000,"log":"[2026-02-17T13:47:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 490ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bslsd38fc3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719939000,"log":"[2026-02-17T14:43:24Z INFO tonic::transport::server] grpc router request_received client_id=f1owrynb","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4lv88kgt81","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719940000,"log":"[2026-02-17T13:16:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6194208527125926000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9sm8yn5xlt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719941000,"log":"[2026-02-17T13:11:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9822186399130757000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r6ze0jlny3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719942000,"log":"[2026-02-17T14:37:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=18luzk6h","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1a877nvni2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719943000,"log":"[2026-02-17T14:13:40Z DEBUG sqlx::query] DELETE FROM alerts returned 162 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n4n9vnzigq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719944000,"log":"[2026-02-17T13:47:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 400 337ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p7qi207r1w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719945000,"log":"[2026-02-17T13:18:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3518704899202953700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-09v34u86n1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719946000,"log":"[2026-02-17T14:21:23Z DEBUG sqlx::query] INSERT FROM users returned 357 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-led9eac128","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719947000,"log":"[2026-02-17T14:35:23Z INFO tonic::transport::server] grpc querier disconnected client_id=y4jr3lbd","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-scn9sfjjof","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719948000,"log":"[2026-02-17T15:02:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i9bvglt19d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719949000,"log":"[2026-02-17T15:08:45Z DEBUG metrics::reporter] recorded metric active_connections value=1922","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fdyqjn05co","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719950000,"log":"[2026-02-17T13:38:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2250925619698693600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-da5m2g180s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719951000,"log":"[2026-02-17T13:47:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 125ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-np66by12yp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719952000,"log":"[2026-02-17T13:11:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1545948629812230400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hdv6mts51w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719953000,"log":"[2026-02-17T12:37:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mrxnly6m9a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719954000,"log":"[2026-02-17T15:16:38Z TRACE cache::redis] cache get key=org:config:3299","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-124dv5w9dm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719955000,"log":"[2026-02-17T14:56:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rkr1q59toa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719956000,"log":"[2026-02-17T12:45:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k5g991cdpx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719957000,"log":"[2026-02-17T13:11:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9311692861957894000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-flynif6ddm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719958000,"log":"[2026-02-17T13:40:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6467935954273036000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wofkwsnkve","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719959000,"log":"[2026-02-17T13:02:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-on9b12a4xx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719960000,"log":"[2026-02-17T12:46:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-76g70ccayb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719961000,"log":"[2026-02-17T13:08:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ufz3d45fnq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719962000,"log":"[2026-02-17T12:37:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e3ci97ag8e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719963000,"log":"[2026-02-17T14:59:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-abqcalh9nv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719964000,"log":"[2026-02-17T14:16:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 242 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ban3ushi18","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719965000,"log":"[2026-02-17T15:05:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=9812","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wyxn71yacw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719966000,"log":"[2026-02-17T14:09:32Z DEBUG sqlx::query] SELECT FROM streams returned 38 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tc8sex7duh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719967000,"log":"[2026-02-17T14:03:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 39ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gl5gyqng7n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719968000,"log":"[2026-02-17T15:13:54Z TRACE cache::redis] cache hit key=stream:schema:7383","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5iu5xkle29","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719969000,"log":"[2026-02-17T13:56:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 61ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dwqff8fm8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719970000,"log":"[2026-02-17T12:58:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rst6ofbcvq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719971000,"log":"[2026-02-17T13:59:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 431ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yg9rp7rgbq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719972000,"log":"[2026-02-17T15:04:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wx6zjmf5yk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719973000,"log":"[2026-02-17T13:56:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 429ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6prskyubzr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719974000,"log":"[2026-02-17T13:29:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4940603984543246000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sgng41ft7k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719975000,"log":"[2026-02-17T13:51:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 414ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uhprh628sn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719976000,"log":"[2026-02-17T15:13:35Z TRACE cache::redis] cache expire key=query:result:8361","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xququfqir3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719977000,"log":"[2026-02-17T14:59:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8y0l6wm13e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719978000,"log":"[2026-02-17T13:56:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 318ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-blparw0rbg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719979000,"log":"[2026-02-17T14:48:36Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5y6wfwx5kr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719980000,"log":"[2026-02-17T13:11:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7470596325508918000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b6ptd4yus9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719981000,"log":"[2026-02-17T14:32:34Z INFO tonic::transport::server] grpc router request_received client_id=rh61rted","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8pjikufsv1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719982000,"log":"[2026-02-17T14:19:42Z DEBUG sqlx::query] SELECT FROM streams returned 222 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l5s4phe7gf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719983000,"log":"[2026-02-17T14:35:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=w4ry6z9l","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ni31lvi4ym","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719984000,"log":"[2026-02-17T15:00:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ydukxplfwq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719985000,"log":"[2026-02-17T13:15:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6645236700347468000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nsjircexdl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719986000,"log":"[2026-02-17T14:54:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b30tfi2itl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719987000,"log":"[2026-02-17T12:41:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2lwasqyd49","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719988000,"log":"[2026-02-17T13:22:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1839422359586859800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fmlrlyway1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719989000,"log":"[2026-02-17T14:17:43Z DEBUG sqlx::query] INSERT FROM users returned 188 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ljh3t9c346","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719990000,"log":"[2026-02-17T12:38:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c5efsrcu7w","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719991000,"log":"[2026-02-17T13:32:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7129754215748689000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e982ooth1p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719992000,"log":"[2026-02-17T15:06:34Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1919","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7moxtl95qk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719993000,"log":"[2026-02-17T14:04:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 135ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kxjragymfc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719994000,"log":"[2026-02-17T15:13:31Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=343","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xq50xje5nu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331719995000,"log":"[2026-02-17T15:00:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ai9gp67sbv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331719996000,"log":"[2026-02-17T12:53:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lgogz50s0t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719997000,"log":"[2026-02-17T13:09:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7392402171493953000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v5fpi9vkys","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719998000,"log":"[2026-02-17T15:02:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b03t7j62ef","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331719999000,"log":"[2026-02-17T15:17:18Z TRACE cache::redis] cache hit key=org:config:9406","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yfuqg5z3l6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720000000,"log":"[2026-02-17T14:51:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i2bxbxu0ux","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720001000,"log":"[2026-02-17T14:45:28Z INFO tonic::transport::server] grpc querier disconnected client_id=0lmxi1oo","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8bw4k5lw9h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720002000,"log":"[2026-02-17T13:00:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gszx0bwftl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720003000,"log":"[2026-02-17T13:18:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/915125766917779200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7a90qdzcc4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720004000,"log":"[2026-02-17T13:31:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4419370100595376600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rhud5dh8pf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720005000,"log":"[2026-02-17T14:55:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u3md9f4vjb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720006000,"log":"[2026-02-17T14:44:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=jlv501y4","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u33gu9ke4j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720007000,"log":"[2026-02-17T15:11:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9501","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k0p204drkb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720008000,"log":"[2026-02-17T14:41:19Z INFO tonic::transport::server] grpc router request_received client_id=oznvvxzf","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zcdvereea8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720009000,"log":"[2026-02-17T14:54:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mxjt43xa88","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720010000,"log":"[2026-02-17T13:30:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1794728044215901000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vv5v8nll1a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720011000,"log":"[2026-02-17T13:44:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 403 338ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-loccyzy274","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720012000,"log":"[2026-02-17T13:40:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3604488641022065700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1rzxdar8jc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720013000,"log":"[2026-02-17T14:24:03Z DEBUG sqlx::query] INSERT FROM users returned 153 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dr4o91tonz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720014000,"log":"[2026-02-17T14:30:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 784 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uf52eg36kr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720015000,"log":"[2026-02-17T12:49:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hbbs5s4oav","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720016000,"log":"[2026-02-17T13:01:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-honf3m10wq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720017000,"log":"[2026-02-17T15:01:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tq0wwz9xwk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720018000,"log":"[2026-02-17T13:25:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5922669795519262000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ewz7qcn2en","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720019000,"log":"[2026-02-17T13:56:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 204 71ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mnnpnhwoto","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720020000,"log":"[2026-02-17T15:10:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8810","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rhv3x90w82","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720021000,"log":"[2026-02-17T14:17:53Z DEBUG sqlx::query] INSERT FROM users returned 307 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6f6ob5z8xy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720022000,"log":"[2026-02-17T13:52:35Z INFO actix_web::middleware::logger] DELETE /api/_search 401 463ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-caasvdvqu5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720023000,"log":"[2026-02-17T14:22:45Z DEBUG sqlx::query] DELETE FROM alerts returned 438 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ao3qln9bng","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720024000,"log":"[2026-02-17T13:00:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0jb5wr0ddw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720025000,"log":"[2026-02-17T13:47:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 408ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-guz8gxda7q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720026000,"log":"[2026-02-17T13:07:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n6sygmq4gz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720027000,"log":"[2026-02-17T12:40:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i0pmf5tx49","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720028000,"log":"[2026-02-17T13:42:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 279ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jj0d57zkhx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720029000,"log":"[2026-02-17T13:44:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 318ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-okxc47ylmd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720030000,"log":"[2026-02-17T12:57:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6sop1v0192","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720031000,"log":"[2026-02-17T15:09:37Z DEBUG metrics::reporter] recorded metric http_requests_total value=6159","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xjsdfzhacx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720032000,"log":"[2026-02-17T14:40:58Z INFO tonic::transport::server] grpc querier disconnected client_id=sjf0z3ja","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tsy7kqijgl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720033000,"log":"[2026-02-17T15:17:10Z TRACE cache::redis] cache delete key=query:result:84","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ziquski155","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720034000,"log":"[2026-02-17T14:33:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=uqfzvop4","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8xu1rzgxpe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720035000,"log":"[2026-02-17T14:10:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 874 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kor6l3x0ae","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720036000,"log":"[2026-02-17T13:39:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1630855713561281000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0uajr4xur6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720037000,"log":"[2026-02-17T14:28:08Z DEBUG sqlx::query] INSERT FROM users returned 678 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-13p8wuuzra","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720038000,"log":"[2026-02-17T12:52:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4s8r6z7irk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720039000,"log":"[2026-02-17T14:14:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 872 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iq193u4m2v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720040000,"log":"[2026-02-17T13:11:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3293621823224641500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7f3uo6d11m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720041000,"log":"[2026-02-17T13:37:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1792708715347739400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j8i1qt7hns","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720042000,"log":"[2026-02-17T13:46:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 37ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0425dhjc66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720043000,"log":"[2026-02-17T14:58:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8qwmic4uzb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720044000,"log":"[2026-02-17T14:43:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=8zuvqr5q","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xfk9gr46r7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720045000,"log":"[2026-02-17T13:57:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tp7ruahkcu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720046000,"log":"[2026-02-17T14:22:03Z DEBUG sqlx::query] INSERT FROM users returned 536 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vqaiyecft6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720047000,"log":"[2026-02-17T13:52:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 503 400ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5p5web9rmb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720048000,"log":"[2026-02-17T13:12:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4547561303457432600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h6ntl5invp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720049000,"log":"[2026-02-17T15:02:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vustm87hy7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720050000,"log":"[2026-02-17T12:50:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d661lnhpra","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720051000,"log":"[2026-02-17T13:50:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 503 359ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vnh23ef5ke","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720052000,"log":"[2026-02-17T13:18:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3403185096954853000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-89rtu8t95s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720053000,"log":"[2026-02-17T12:58:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tgtekng42s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720054000,"log":"[2026-02-17T13:16:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6031903259223097000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-22oqi6lxdu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720055000,"log":"[2026-02-17T13:39:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7111541920598501000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g1a5e61abd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720056000,"log":"[2026-02-17T15:09:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=3064","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d76b959nbg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720057000,"log":"[2026-02-17T15:19:51Z TRACE cache::redis] cache set key=auth:token:8890","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2uvuh3mvfc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720058000,"log":"[2026-02-17T14:51:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kqkpl4hbq9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720059000,"log":"[2026-02-17T14:57:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pkqoepqidt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720060000,"log":"[2026-02-17T14:26:48Z DEBUG sqlx::query] INSERT FROM users returned 152 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b56yvszhaf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720061000,"log":"[2026-02-17T13:15:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2014526896959776300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mfti72ud5m","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720062000,"log":"[2026-02-17T13:07:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ygq70my24h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720063000,"log":"[2026-02-17T14:06:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 182ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a98bwhkwx3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720064000,"log":"[2026-02-17T13:39:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7647667633357487000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-55vaksxp8c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720065000,"log":"[2026-02-17T13:05:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-du5b4tqbbl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720066000,"log":"[2026-02-17T12:57:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qoddhtajuc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720067000,"log":"[2026-02-17T13:04:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x10obe6stn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720068000,"log":"[2026-02-17T13:48:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 204 392ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5rcxsifkap","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720069000,"log":"[2026-02-17T13:54:41Z INFO actix_web::middleware::logger] GET /api/_bulk 401 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-b7gmupjdfm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720070000,"log":"[2026-02-17T15:09:50Z DEBUG metrics::reporter] recorded metric active_connections value=259","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zcgu2d8vcb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720071000,"log":"[2026-02-17T14:25:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 5 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sjgndt43qg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720072000,"log":"[2026-02-17T13:20:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9188422281415893000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-trc1c8q91y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720073000,"log":"[2026-02-17T14:18:10Z DEBUG sqlx::query] DELETE FROM alerts returned 278 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-82noek5uz1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720074000,"log":"[2026-02-17T15:09:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=1061","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-74abq8271d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720075000,"log":"[2026-02-17T13:27:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6400215002130221000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tawmbi0xt7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720076000,"log":"[2026-02-17T13:13:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/12381844621680572.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bfipu84obo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720077000,"log":"[2026-02-17T14:49:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nkfmk8kl8f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720078000,"log":"[2026-02-17T12:51:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wwfjxcuufb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720079000,"log":"[2026-02-17T14:27:33Z DEBUG sqlx::query] INSERT FROM users returned 384 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-94y2rqfbaq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720080000,"log":"[2026-02-17T13:14:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6619753462927917000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lcr5gggh79","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720081000,"log":"[2026-02-17T13:13:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3130701045544786000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kb1pjc82tj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720082000,"log":"[2026-02-17T14:36:47Z INFO tonic::transport::server] grpc ingester connected client_id=k092otf8","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-406u2ea898","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720083000,"log":"[2026-02-17T12:37:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1v731ocs75","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720084000,"log":"[2026-02-17T14:59:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-avt2pe2mx2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720085000,"log":"[2026-02-17T12:57:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0exjx63cwp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720086000,"log":"[2026-02-17T13:29:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/714804524666479100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a6h3853i35","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720087000,"log":"[2026-02-17T14:01:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 12ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-arx8uwaazg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720088000,"log":"[2026-02-17T13:46:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 201 399ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wu1rpp0ia0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720089000,"log":"[2026-02-17T13:40:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/291647568601700600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k90jsw6c91","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720090000,"log":"[2026-02-17T14:31:05Z DEBUG sqlx::query] DELETE FROM alerts returned 300 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wyxl7k4t7o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720091000,"log":"[2026-02-17T13:49:16Z INFO actix_web::middleware::logger] POST /api/_bulk 400 18ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q5hbg5i1k2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720092000,"log":"[2026-02-17T14:45:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=ldeeh67q","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oiyerxnok9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720093000,"log":"[2026-02-17T13:05:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h179z1885w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720094000,"log":"[2026-02-17T14:10:38Z DEBUG sqlx::query] INSERT FROM users returned 873 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h0l1l0bgzd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720095000,"log":"[2026-02-17T13:15:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2472071508998099500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pnhrh45l3d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720096000,"log":"[2026-02-17T14:56:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-spznpb7vib","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720097000,"log":"[2026-02-17T14:42:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=y9ybrrd1","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n4misedpmr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720098000,"log":"[2026-02-17T14:24:23Z DEBUG sqlx::query] INSERT FROM users returned 21 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-axens46wj8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720099000,"log":"[2026-02-17T15:04:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tcsouqym95","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720100000,"log":"[2026-02-17T14:21:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 392 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0giuoxv4sc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720101000,"log":"[2026-02-17T15:05:54Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2809","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c78qvjfb7h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720102000,"log":"[2026-02-17T14:51:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ac7poefk9f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720103000,"log":"[2026-02-17T12:39:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-roskak4ryy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720104000,"log":"[2026-02-17T15:09:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6309","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oqc4p22x7g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720105000,"log":"[2026-02-17T13:00:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8nietujt8m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720106000,"log":"[2026-02-17T13:02:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kks0lypt5q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720107000,"log":"[2026-02-17T15:20:52Z TRACE cache::redis] cache delete key=user:session:2582","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5xd8sqehv2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720108000,"log":"[2026-02-17T12:59:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aabxqf9vrv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720109000,"log":"[2026-02-17T13:53:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 258ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-plavxk8yvx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720110000,"log":"[2026-02-17T13:57:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 355ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ydcfvsacwr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720111000,"log":"[2026-02-17T14:31:23Z DEBUG sqlx::query] INSERT FROM users returned 607 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vz5xdn1l1q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720112000,"log":"[2026-02-17T13:12:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2659297783077865500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ln0qj0d4p6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720113000,"log":"[2026-02-17T13:14:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9914751346337298000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7fhqbwiyfo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720114000,"log":"[2026-02-17T13:58:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 318ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qf8xk6vqmi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720115000,"log":"[2026-02-17T14:12:13Z DEBUG sqlx::query] INSERT FROM users returned 933 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7p0rvhpwvc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720116000,"log":"[2026-02-17T13:49:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 277ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ncuqosmu3i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720117000,"log":"[2026-02-17T14:13:45Z DEBUG sqlx::query] DELETE FROM alerts returned 386 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gg3o1dmiwy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720118000,"log":"[2026-02-17T15:19:55Z TRACE cache::redis] cache miss key=query:result:5447","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5fx9sycly8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720119000,"log":"[2026-02-17T15:09:05Z DEBUG metrics::reporter] recorded metric active_connections value=6544","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t2hvglrc2p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720120000,"log":"[2026-02-17T13:18:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2248466804238134000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i6nbbil4d0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720121000,"log":"[2026-02-17T13:48:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 321ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5smo56797k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720122000,"log":"[2026-02-17T15:21:37Z TRACE cache::redis] cache miss key=user:session:3362","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q2spibnxxt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720123000,"log":"[2026-02-17T15:13:20Z DEBUG metrics::reporter] recorded metric active_connections value=5278","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nmqx7qpgzp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720124000,"log":"[2026-02-17T14:58:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n96ssboq2u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720125000,"log":"[2026-02-17T13:20:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6302054553252643000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bashc4hdct","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720126000,"log":"[2026-02-17T14:06:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 503 150ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mti4r2kg86","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720127000,"log":"[2026-02-17T15:20:13Z TRACE cache::redis] cache miss key=org:config:7482","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sltpvksh63","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720128000,"log":"[2026-02-17T14:25:48Z DEBUG sqlx::query] INSERT FROM users returned 17 rows in 28ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-myo4qriv1p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720129000,"log":"[2026-02-17T14:55:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9w39p3i7jt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720130000,"log":"[2026-02-17T13:06:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tjg1hhilua","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720131000,"log":"[2026-02-17T13:48:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 168ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zz22tyu2v7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720132000,"log":"[2026-02-17T14:37:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=q0rl3j11","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d2i868uu0m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720133000,"log":"[2026-02-17T15:08:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3093","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2eo3e1qiar","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720134000,"log":"[2026-02-17T14:15:05Z DEBUG sqlx::query] DELETE FROM alerts returned 415 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uvchl22dxb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720135000,"log":"[2026-02-17T14:02:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 442ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f7wjhgbzon","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720136000,"log":"[2026-02-17T12:35:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9bwoaun7ba","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720137000,"log":"[2026-02-17T13:26:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1952229729402930000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vk9k4kq74y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720138000,"log":"[2026-02-17T13:10:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8961869287079729000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8tebfq5iym","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720139000,"log":"[2026-02-17T12:47:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x0dygfjtdl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720140000,"log":"[2026-02-17T13:12:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9767550587353438000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-srk15gdqbc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720141000,"log":"[2026-02-17T14:52:13Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tmfwlta8lm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720142000,"log":"[2026-02-17T15:13:32Z TRACE cache::redis] cache get key=user:session:3278","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6f1zuqut64","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720143000,"log":"[2026-02-17T14:07:28Z DEBUG sqlx::query] INSERT FROM users returned 318 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qyrueiob03","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720144000,"log":"[2026-02-17T15:18:02Z TRACE cache::redis] cache get key=user:session:5079","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b8nb71izpg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720145000,"log":"[2026-02-17T14:34:24Z INFO tonic::transport::server] grpc router request_received client_id=f3vphund","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-su8epyfyix","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720146000,"log":"[2026-02-17T13:38:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8570209564926601000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kjze6i12wl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720147000,"log":"[2026-02-17T14:39:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=ofgju51x","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dzjx2pvwq5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720148000,"log":"[2026-02-17T14:16:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 552 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e55m5l5asi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720149000,"log":"[2026-02-17T14:00:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 184ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9pje33fbe2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720150000,"log":"[2026-02-17T12:35:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4er1k9fc4m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720151000,"log":"[2026-02-17T13:47:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 48ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mgmodj2q5n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720152000,"log":"[2026-02-17T13:10:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7248681520985758000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5umpufwfb0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720153000,"log":"[2026-02-17T14:55:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tbism6jz2s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720154000,"log":"[2026-02-17T15:06:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=5042","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jnrx81ytkx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720155000,"log":"[2026-02-17T13:29:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3519507936375545300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9b6dt543j0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720156000,"log":"[2026-02-17T12:38:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-huosskpg4h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720157000,"log":"[2026-02-17T13:29:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7978281078901772000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qx5fb5z0ni","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720158000,"log":"[2026-02-17T13:51:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 1ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d80fn9v3cm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720159000,"log":"[2026-02-17T14:19:05Z DEBUG sqlx::query] DELETE FROM alerts returned 227 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n5e9j0ts2y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720160000,"log":"[2026-02-17T14:04:40Z INFO actix_web::middleware::logger] POST /api/_search 200 216ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iade29vl4d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720161000,"log":"[2026-02-17T14:47:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=k4fj8ofy","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5ywx66yzcz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720162000,"log":"[2026-02-17T13:54:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 243ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-75t3akwl01","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720163000,"log":"[2026-02-17T13:45:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 56ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9qvirvq923","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720164000,"log":"[2026-02-17T14:10:18Z DEBUG sqlx::query] INSERT FROM users returned 485 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k9mwo60g4r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720165000,"log":"[2026-02-17T13:03:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tx10msfmr1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720166000,"log":"[2026-02-17T12:55:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8p0s8is31l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720167000,"log":"[2026-02-17T13:42:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 400 244ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ylzvuhbqwq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720168000,"log":"[2026-02-17T15:19:52Z TRACE cache::redis] cache delete key=user:session:4151","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bu1bzegsyc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720169000,"log":"[2026-02-17T12:58:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jc2z2ze48c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720170000,"log":"[2026-02-17T14:21:20Z DEBUG sqlx::query] DELETE FROM alerts returned 487 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vzf1wk46lw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720171000,"log":"[2026-02-17T14:27:15Z DEBUG sqlx::query] DELETE FROM alerts returned 558 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-iatz66sb3l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720172000,"log":"[2026-02-17T14:13:02Z DEBUG sqlx::query] SELECT FROM streams returned 853 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l9ndptrh08","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720173000,"log":"[2026-02-17T14:57:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-54710mkf08","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720174000,"log":"[2026-02-17T13:19:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3963155749649654000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eyfla4d0xk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720175000,"log":"[2026-02-17T13:04:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2za7vmys1t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720176000,"log":"[2026-02-17T13:03:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mq85aw0icr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720177000,"log":"[2026-02-17T13:36:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7199605806422296000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m2f2vizs9o","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720178000,"log":"[2026-02-17T14:10:22Z DEBUG sqlx::query] SELECT FROM streams returned 514 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-aey7l0cc0r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720179000,"log":"[2026-02-17T13:31:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2378969882947033000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0autah6ivh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720180000,"log":"[2026-02-17T14:35:52Z INFO tonic::transport::server] grpc ingester connected client_id=5ipm8d6s","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f7ioynu8ye","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720181000,"log":"[2026-02-17T15:01:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vraqcrxgbn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720182000,"log":"[2026-02-17T14:46:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=1603ugfo","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5l8l2ednuk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720183000,"log":"[2026-02-17T13:00:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nedp9i1r71","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720184000,"log":"[2026-02-17T13:40:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2137188037799373600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q6p8a5ocnr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720185000,"log":"[2026-02-17T13:45:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 288ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nmppu3poho","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720186000,"log":"[2026-02-17T13:00:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3c6rbwkbs7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720187000,"log":"[2026-02-17T13:29:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5147492908908724000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-icnovx8w4i","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720188000,"log":"[2026-02-17T14:38:28Z INFO tonic::transport::server] grpc querier disconnected client_id=84o95tnk","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gnmzbjd4d4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720189000,"log":"[2026-02-17T12:36:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k64onlihf4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720190000,"log":"[2026-02-17T14:19:47Z DEBUG sqlx::query] SELECT FROM streams returned 626 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m7mza2l9za","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720191000,"log":"[2026-02-17T13:26:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6325029425114020000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sn6nephkzb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720192000,"log":"[2026-02-17T13:54:00Z INFO actix_web::middleware::logger] POST /api/_search 503 478ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2qfvnr1me5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720193000,"log":"[2026-02-17T12:56:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ji4nbig7cf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720194000,"log":"[2026-02-17T13:49:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 250ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ggqm8mefze","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720195000,"log":"[2026-02-17T13:41:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9173659458899140000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p47yq3efvd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720196000,"log":"[2026-02-17T15:21:21Z TRACE cache::redis] cache set key=auth:token:5517","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1bgw9qf04h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720197000,"log":"[2026-02-17T12:54:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qm5ea2skmp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720198000,"log":"[2026-02-17T12:54:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h2e7zgg33m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720199000,"log":"[2026-02-17T13:36:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8145319733439377000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-io0lboogvy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720200000,"log":"[2026-02-17T12:56:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ag9vdl83mf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720201000,"log":"[2026-02-17T14:07:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 734 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-huxz2h8f9l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720202000,"log":"[2026-02-17T13:08:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8mpywauzi6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720203000,"log":"[2026-02-17T13:00:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hcj9zcnv17","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720204000,"log":"[2026-02-17T14:06:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 132ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4cioih2tqv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720205000,"log":"[2026-02-17T13:11:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6824452122800802000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c4z02t6xrn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720206000,"log":"[2026-02-17T12:41:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-71poxl7gcy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720207000,"log":"[2026-02-17T15:20:45Z TRACE cache::redis] cache set key=query:result:9554","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ubponkqr4u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720208000,"log":"[2026-02-17T13:47:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 122ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mvwxgpw8sy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720209000,"log":"[2026-02-17T14:48:22Z INFO tonic::transport::server] grpc ingester connected client_id=7vcwjpzq","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z7o6lmq0ud","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720210000,"log":"[2026-02-17T13:29:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1003456554750619100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zfsrf7w9ti","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720211000,"log":"[2026-02-17T14:50:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-atadssw4i9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720212000,"log":"[2026-02-17T14:31:35Z DEBUG sqlx::query] DELETE FROM alerts returned 767 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3ur97szdlp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720213000,"log":"[2026-02-17T15:17:05Z TRACE cache::redis] cache expire key=query:result:2686","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e0muagkttz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720214000,"log":"[2026-02-17T13:23:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5004914315770957000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y7r0z5p1ik","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720215000,"log":"[2026-02-17T13:13:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9831121488351834000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xejyrm2faw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720216000,"log":"[2026-02-17T13:06:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qfhpzo6ezk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720217000,"log":"[2026-02-17T13:44:25Z INFO actix_web::middleware::logger] GET /api/_search 200 417ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-scrfe8htwl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720218000,"log":"[2026-02-17T14:49:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-439pcqxu1w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720219000,"log":"[2026-02-17T13:27:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9117985117145116000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5zgerhqfi9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720220000,"log":"[2026-02-17T13:24:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3484184958688510000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-smy9q1dewq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720221000,"log":"[2026-02-17T12:45:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g2gjhzxhfw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720222000,"log":"[2026-02-17T14:03:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 359ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0c4t1ns7hv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720223000,"log":"[2026-02-17T13:50:45Z INFO actix_web::middleware::logger] GET /api/_search 204 247ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-85r1p239uj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720224000,"log":"[2026-02-17T14:30:18Z DEBUG sqlx::query] INSERT FROM users returned 111 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7wwzsjxcvj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720225000,"log":"[2026-02-17T13:47:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 274ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9x4ednyhek","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720226000,"log":"[2026-02-17T14:22:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 610 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i6w90rgvnw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720227000,"log":"[2026-02-17T14:42:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=8bkus164","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h3kmq2lk3y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720228000,"log":"[2026-02-17T15:12:21Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5583","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6i64xaphnt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720229000,"log":"[2026-02-17T15:21:11Z TRACE cache::redis] cache expire key=auth:token:3133","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v9q9rmrscv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720230000,"log":"[2026-02-17T13:17:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3894241231259823000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s2va656tvt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720231000,"log":"[2026-02-17T14:04:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 503 131ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fk65x9vl7b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720232000,"log":"[2026-02-17T12:38:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xiplfegc3h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720233000,"log":"[2026-02-17T13:32:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4675460189028465000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vbrcm3nfsp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720234000,"log":"[2026-02-17T14:36:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=6g3qulbm","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y43hsda8lg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720235000,"log":"[2026-02-17T14:54:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ekgp2yvub1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720236000,"log":"[2026-02-17T13:11:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5345165076697416000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l83rya0yml","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720237000,"log":"[2026-02-17T13:51:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 145ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a00scxv7r2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720238000,"log":"[2026-02-17T15:15:49Z TRACE cache::redis] cache miss key=stream:schema:991","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3ltqzv9qip","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720239000,"log":"[2026-02-17T14:43:07Z INFO tonic::transport::server] grpc ingester connected client_id=gt85ie9f","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-445srpmxkk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720240000,"log":"[2026-02-17T13:05:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tsf5nofik1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720241000,"log":"[2026-02-17T12:50:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nc72pigane","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720242000,"log":"[2026-02-17T14:18:35Z DEBUG sqlx::query] DELETE FROM alerts returned 612 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sso18k1w05","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720243000,"log":"[2026-02-17T14:23:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 790 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9vp59n5vbu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720244000,"log":"[2026-02-17T15:05:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5087","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1h61njq508","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720245000,"log":"[2026-02-17T13:21:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6248240262679785000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-madnz1zk0l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720246000,"log":"[2026-02-17T13:21:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1838479550090388200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-842kss3mfo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720247000,"log":"[2026-02-17T14:03:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 99ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yym2d11ann","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720248000,"log":"[2026-02-17T13:09:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5091208074095701000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hlenujtidz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720249000,"log":"[2026-02-17T15:08:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9526","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ue41m8kx0l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720250000,"log":"[2026-02-17T13:50:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 8ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6xx9dlg19i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720251000,"log":"[2026-02-17T14:12:23Z DEBUG sqlx::query] INSERT FROM users returned 305 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bz9wyaznbw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720252000,"log":"[2026-02-17T13:03:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n0x8ds8gfh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720253000,"log":"[2026-02-17T14:01:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 436ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7j0izo9pfa","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720254000,"log":"[2026-02-17T14:52:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xu2euvt6dn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720255000,"log":"[2026-02-17T13:18:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1691740841706324000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3hy5mgh63d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720256000,"log":"[2026-02-17T14:51:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0c5laosu43","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720257000,"log":"[2026-02-17T13:14:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2483268768346627000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s6za641uzu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720258000,"log":"[2026-02-17T15:21:17Z TRACE cache::redis] cache expire key=user:session:7132","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pehzvyxua5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720259000,"log":"[2026-02-17T13:25:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3371448701568198000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vuxwlghhmd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720260000,"log":"[2026-02-17T13:54:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 418ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ppl91mk9mc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720261000,"log":"[2026-02-17T13:40:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3060047472308058600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-78llmoe90u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720262000,"log":"[2026-02-17T13:24:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/197130271254741800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a82w6ule8p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720263000,"log":"[2026-02-17T15:01:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0j3ap0cb1j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720264000,"log":"[2026-02-17T13:58:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 152ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iutcdn3cuf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720265000,"log":"[2026-02-17T14:39:17Z INFO tonic::transport::server] grpc ingester connected client_id=e0b927g5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u686k3dr3p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720266000,"log":"[2026-02-17T15:20:47Z TRACE cache::redis] cache expire key=user:session:8252","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g6eswn46b1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720267000,"log":"[2026-02-17T14:53:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ussv615ufy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720268000,"log":"[2026-02-17T13:37:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9880612503871103000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-afc2vkcnhn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720269000,"log":"[2026-02-17T13:38:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8217192210279649000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-apyr84njad","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720270000,"log":"[2026-02-17T13:34:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6325941038619753000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9izwvfsaon","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720271000,"log":"[2026-02-17T13:23:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3749051444135654000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pdcmc1sump","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720272000,"log":"[2026-02-17T14:30:10Z DEBUG sqlx::query] DELETE FROM alerts returned 520 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rzc74g0ga1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720273000,"log":"[2026-02-17T12:37:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pksaawr9ki","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720274000,"log":"[2026-02-17T13:31:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5832613898515286000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cvdtpopvhy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720275000,"log":"[2026-02-17T13:57:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 432ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dsgme8vglh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720276000,"log":"[2026-02-17T12:41:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n3hngrjvpd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720277000,"log":"[2026-02-17T14:39:33Z INFO tonic::transport::server] grpc querier disconnected client_id=8pe2w08l","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h82mqa2t98","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720278000,"log":"[2026-02-17T13:03:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ve1afnhls2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720279000,"log":"[2026-02-17T14:36:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=n8f5bqoo","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kgfq11sv46","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720280000,"log":"[2026-02-17T12:39:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nrwqkrit0g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720281000,"log":"[2026-02-17T13:52:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 46ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lqf5s1niki","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720282000,"log":"[2026-02-17T14:04:56Z INFO actix_web::middleware::logger] POST /api/_bulk 500 183ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lkex1kasba","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720283000,"log":"[2026-02-17T12:47:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p6rlnf33ac","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720284000,"log":"[2026-02-17T13:36:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5767028916740879000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sto8xvqfgk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720285000,"log":"[2026-02-17T13:18:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7886391339992880000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n8hu6aw3ul","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720286000,"log":"[2026-02-17T15:14:20Z TRACE cache::redis] cache get key=query:result:9588","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6whujape7w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720287000,"log":"[2026-02-17T13:41:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 143ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hzi065lzhy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720288000,"log":"[2026-02-17T12:36:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-es2nca0n72","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720289000,"log":"[2026-02-17T13:44:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 25ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fu55l6o261","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720290000,"log":"[2026-02-17T14:48:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-geulrwbmyc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720291000,"log":"[2026-02-17T14:47:18Z INFO tonic::transport::server] grpc querier disconnected client_id=4kpg7n3c","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r3jsre6s00","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720292000,"log":"[2026-02-17T14:01:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 310ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-blqymoxewr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720293000,"log":"[2026-02-17T13:43:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 225ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-js29z41pfr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720294000,"log":"[2026-02-17T13:09:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9642132936902547000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bowpc6pyo0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720295000,"log":"[2026-02-17T13:05:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9ek9gxgwoy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720296000,"log":"[2026-02-17T12:38:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vkr68hkrmh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720297000,"log":"[2026-02-17T14:45:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=4i8nurht","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0obr8in2gx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720298000,"log":"[2026-02-17T13:28:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1862087937580734200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5hedaxnlg8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720299000,"log":"[2026-02-17T14:57:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v6pvwopj2v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720300000,"log":"[2026-02-17T14:37:24Z INFO tonic::transport::server] grpc router request_received client_id=1wz7zxd4","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m6w1rzlzuc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720301000,"log":"[2026-02-17T14:40:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=mz0c3tbt","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vpvo39eovs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720302000,"log":"[2026-02-17T14:28:22Z DEBUG sqlx::query] SELECT FROM streams returned 398 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3r9taqvjw8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720303000,"log":"[2026-02-17T14:11:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 920 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u8sivlzv2g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720304000,"log":"[2026-02-17T13:40:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5652349800641312000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1n1yq1nipm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720305000,"log":"[2026-02-17T14:19:13Z DEBUG sqlx::query] INSERT FROM users returned 427 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8bdnkxihat","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720306000,"log":"[2026-02-17T13:03:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qch9swo8fc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720307000,"log":"[2026-02-17T13:28:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9607093717034590000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u6pv3sfjyp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720308000,"log":"[2026-02-17T14:51:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wiajz2kejh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720309000,"log":"[2026-02-17T13:01:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kwmtl1uvdu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720310000,"log":"[2026-02-17T15:03:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-04gldl2kg1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720311000,"log":"[2026-02-17T14:28:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 626 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dz0yquee5p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720312000,"log":"[2026-02-17T13:48:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 269ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dnaae5nyb3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720313000,"log":"[2026-02-17T14:46:54Z INFO tonic::transport::server] grpc router request_received client_id=whnn389b","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iuhqrj9mrt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720314000,"log":"[2026-02-17T15:05:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=8993","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-khuiv8ux9d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720315000,"log":"[2026-02-17T14:52:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2xd4xe1ggk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720316000,"log":"[2026-02-17T14:34:02Z INFO tonic::transport::server] grpc ingester connected client_id=vqzjkvc4","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nl8qypays","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720317000,"log":"[2026-02-17T14:32:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=pz3vonkb","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s7zd733a5c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720318000,"log":"[2026-02-17T14:57:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qeumtx03uo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720319000,"log":"[2026-02-17T15:03:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d5q5608t43","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720320000,"log":"[2026-02-17T13:59:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 120ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8taxg7gb57","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720321000,"log":"[2026-02-17T12:59:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wzgimf0pll","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720322000,"log":"[2026-02-17T13:03:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bcw6cak201","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720323000,"log":"[2026-02-17T12:45:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hcb2cz06bk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720324000,"log":"[2026-02-17T14:08:48Z DEBUG sqlx::query] INSERT FROM users returned 561 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gpv7bqsx9f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720325000,"log":"[2026-02-17T14:05:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 227ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bkhebm3bvl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720326000,"log":"[2026-02-17T13:46:05Z INFO actix_web::middleware::logger] GET /api/_search 201 116ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pb1uw7xvqs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720327000,"log":"[2026-02-17T13:59:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 134ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cz5zqjf6ld","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720328000,"log":"[2026-02-17T13:30:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2607392772229367000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qmc7ntqs6v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720329000,"log":"[2026-02-17T14:31:47Z DEBUG sqlx::query] SELECT FROM streams returned 474 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1l45ot1iwq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720330000,"log":"[2026-02-17T14:52:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u0rb8pvie6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720331000,"log":"[2026-02-17T13:22:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6586694213883537000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pf3so4gana","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720332000,"log":"[2026-02-17T15:21:50Z TRACE cache::redis] cache get key=query:result:4020","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-chd5z0eev3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720333000,"log":"[2026-02-17T14:09:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 899 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-kvdr0248n9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720334000,"log":"[2026-02-17T15:17:53Z TRACE cache::redis] cache expire key=org:config:4222","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7e631ei7w3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720335000,"log":"[2026-02-17T13:56:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 401 350ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h0xy281i5l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720336000,"log":"[2026-02-17T15:10:32Z DEBUG metrics::reporter] recorded metric http_requests_total value=8603","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n8qw8f7rh2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720337000,"log":"[2026-02-17T12:35:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e7udqkt70o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720338000,"log":"[2026-02-17T15:10:40Z DEBUG metrics::reporter] recorded metric active_connections value=7942","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5tym0lpp0u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720339000,"log":"[2026-02-17T13:40:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7530945975171138000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g01j5m5ko4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720340000,"log":"[2026-02-17T13:09:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4760720111790857000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mehh6z9kw5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720341000,"log":"[2026-02-17T14:37:33Z INFO tonic::transport::server] grpc querier disconnected client_id=pz1xdogw","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-64btknvtke","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720342000,"log":"[2026-02-17T13:02:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-svr87obmfy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720343000,"log":"[2026-02-17T12:50:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9z5lir08qq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720344000,"log":"[2026-02-17T13:36:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7958749473030526000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ebho48q4ih","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720345000,"log":"[2026-02-17T14:31:50Z DEBUG sqlx::query] DELETE FROM alerts returned 452 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tpre5unyx9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720346000,"log":"[2026-02-17T13:29:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8984202566508925000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fnx17t04ml","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720347000,"log":"[2026-02-17T14:42:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=3yjg6b2n","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zjym2a89ba","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720348000,"log":"[2026-02-17T12:46:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0qqf5e9971","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720349000,"log":"[2026-02-17T13:02:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-98jd9kcbuw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720350000,"log":"[2026-02-17T15:02:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-frwymaz7fe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720351000,"log":"[2026-02-17T14:44:48Z INFO tonic::transport::server] grpc querier disconnected client_id=fk2zrs9u","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dn9dvxwk23","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720352000,"log":"[2026-02-17T13:25:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2622009026877223400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yw3fx7bhby","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720353000,"log":"[2026-02-17T14:25:12Z DEBUG sqlx::query] SELECT FROM streams returned 557 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vfonxicpsh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720354000,"log":"[2026-02-17T12:49:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3z3j6j0wzr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720355000,"log":"[2026-02-17T14:52:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hzem9nky9v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720356000,"log":"[2026-02-17T15:19:24Z TRACE cache::redis] cache hit key=stream:schema:1203","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xz5qrmn8b2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720357000,"log":"[2026-02-17T13:48:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 26ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h6tpm7d3sk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720358000,"log":"[2026-02-17T13:33:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2005132641100133600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7e9btqf1e4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720359000,"log":"[2026-02-17T15:00:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j64wzpki54","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720360000,"log":"[2026-02-17T14:32:44Z INFO tonic::transport::server] grpc router request_received client_id=zlnai1pz","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-elvuuwjnkd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720361000,"log":"[2026-02-17T12:53:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xjde1o3l8u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720362000,"log":"[2026-02-17T13:19:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1408567033258063600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hhx2ejavgh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720363000,"log":"[2026-02-17T14:54:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1kk4305yk1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720364000,"log":"[2026-02-17T13:47:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 394ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-58bokh19yh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720365000,"log":"[2026-02-17T14:49:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0xcm9vx2ff","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720366000,"log":"[2026-02-17T15:20:29Z TRACE cache::redis] cache expire key=stream:schema:3047","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xxloihq6v1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720367000,"log":"[2026-02-17T13:32:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8218079301834935000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gjh3tt7c6s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720368000,"log":"[2026-02-17T14:24:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 652 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b4oyvo2ovj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720369000,"log":"[2026-02-17T14:39:57Z INFO tonic::transport::server] grpc ingester connected client_id=nzbd66k3","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h5fh07n207","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720370000,"log":"[2026-02-17T14:47:09Z INFO tonic::transport::server] grpc router request_received client_id=oxgxxgq0","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nlgv6xif9v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720371000,"log":"[2026-02-17T14:15:30Z DEBUG sqlx::query] DELETE FROM alerts returned 430 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sce2ndfqjn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720372000,"log":"[2026-02-17T13:15:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2931738946312265700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-27u4ivj8g5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720373000,"log":"[2026-02-17T13:41:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9172390308168441000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zp444ash6l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720374000,"log":"[2026-02-17T14:53:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7i6pqp6li1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720375000,"log":"[2026-02-17T13:02:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uzbkcumcm9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720376000,"log":"[2026-02-17T13:52:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 267ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dlhvwlhp73","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720377000,"log":"[2026-02-17T14:22:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 934 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q6hs792697","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720378000,"log":"[2026-02-17T12:50:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bu4gw93b2x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720379000,"log":"[2026-02-17T12:41:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7bfcm92kf8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720380000,"log":"[2026-02-17T13:28:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6676275810785763000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xqgghtu97b","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720381000,"log":"[2026-02-17T15:04:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kftr8t7rq5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720382000,"log":"[2026-02-17T13:30:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7501589095376979000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ec5che4kti","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720383000,"log":"[2026-02-17T13:51:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5jrnsaspkj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720384000,"log":"[2026-02-17T14:12:15Z DEBUG sqlx::query] DELETE FROM alerts returned 706 rows in 53ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-69tc3pw9ai","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720385000,"log":"[2026-02-17T13:14:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7024254178788516000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w9n4vsbkmy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720386000,"log":"[2026-02-17T14:32:19Z INFO tonic::transport::server] grpc router request_received client_id=b1jbig8p","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yx8lvd4elv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720387000,"log":"[2026-02-17T14:00:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 333ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fujn6n5fbs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720388000,"log":"[2026-02-17T14:35:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=xdv5gk6b","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0h1286c45g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720389000,"log":"[2026-02-17T13:42:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 101ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d3zxocn795","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720390000,"log":"[2026-02-17T14:22:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 771 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4xq1ln4ayk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720391000,"log":"[2026-02-17T14:38:29Z INFO tonic::transport::server] grpc router request_received client_id=dxklli1z","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-as66o6szl5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720392000,"log":"[2026-02-17T14:14:38Z DEBUG sqlx::query] INSERT FROM users returned 363 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-74u04kfbkt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720393000,"log":"[2026-02-17T14:49:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mk4mi7310r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720394000,"log":"[2026-02-17T13:54:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 487ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t1pfabbnsf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720395000,"log":"[2026-02-17T14:41:17Z INFO tonic::transport::server] grpc ingester connected client_id=uzv10fpe","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ht460ybgap","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720396000,"log":"[2026-02-17T14:29:28Z DEBUG sqlx::query] INSERT FROM users returned 609 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f0tjx0m9ys","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720397000,"log":"[2026-02-17T14:45:57Z INFO tonic::transport::server] grpc ingester connected client_id=lyxt4nj7","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-og3ooaak4k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720398000,"log":"[2026-02-17T14:54:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tsh0dxlfky","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720399000,"log":"[2026-02-17T12:43:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t0uiqlijcu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720400000,"log":"[2026-02-17T13:43:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 392ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c9b6dg3apt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720401000,"log":"[2026-02-17T13:05:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-geuqd90gt5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720402000,"log":"[2026-02-17T12:47:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gqm6v7ww5u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720403000,"log":"[2026-02-17T13:06:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mh5mbbzdl6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720404000,"log":"[2026-02-17T12:50:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lv5d91bq90","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720405000,"log":"[2026-02-17T13:30:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6751891180366813000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kmzmlldvr5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720406000,"log":"[2026-02-17T14:55:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-177wg7a1dj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720407000,"log":"[2026-02-17T14:49:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zkwc9py9bb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720408000,"log":"[2026-02-17T12:46:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4y4hd26334","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720409000,"log":"[2026-02-17T12:36:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-thjlab4pi4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720410000,"log":"[2026-02-17T12:51:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xmv558h343","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720411000,"log":"[2026-02-17T13:27:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/113121646694029900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ft2opuvs8j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720412000,"log":"[2026-02-17T12:50:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-x78010gphn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720413000,"log":"[2026-02-17T13:44:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 1ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t4c5tnsl6n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720414000,"log":"[2026-02-17T13:48:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 492ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tfjrb5fl7d","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720415000,"log":"[2026-02-17T13:22:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5058084529301230000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zw3e7hqkxc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720416000,"log":"[2026-02-17T14:13:52Z DEBUG sqlx::query] SELECT FROM streams returned 576 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jpzqyfn43q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720417000,"log":"[2026-02-17T15:21:45Z TRACE cache::redis] cache set key=query:result:9235","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-os778djkzx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720418000,"log":"[2026-02-17T13:11:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6712632819966271000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1ghgwbw79z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720419000,"log":"[2026-02-17T13:29:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1636311238146357200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p26wl65j06","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720420000,"log":"[2026-02-17T12:52:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jn5sc5l8jp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720421000,"log":"[2026-02-17T13:39:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3377203830630659600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y9uxgxkqm5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720422000,"log":"[2026-02-17T15:00:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gv7ij9wtw2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720423000,"log":"[2026-02-17T14:17:08Z DEBUG sqlx::query] INSERT FROM users returned 679 rows in 52ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-288mwxcqxz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720424000,"log":"[2026-02-17T14:56:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q4ctjuehn8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720425000,"log":"[2026-02-17T14:07:58Z DEBUG sqlx::query] INSERT FROM users returned 993 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ynq3zxnvup","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720426000,"log":"[2026-02-17T14:52:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d6jko0q1td","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720427000,"log":"[2026-02-17T12:53:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q8hmmebpwm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720428000,"log":"[2026-02-17T13:31:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1925886395816065500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-trsihjvx1w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720429000,"log":"[2026-02-17T13:16:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5454440956227616000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wixjr0341g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720430000,"log":"[2026-02-17T13:59:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 175ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g2rf213892","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720431000,"log":"[2026-02-17T13:02:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-03ilzi0q38","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720432000,"log":"[2026-02-17T13:15:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6485625800198413000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-exkncr6rd1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720433000,"log":"[2026-02-17T13:16:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1692614026113517000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sgm8vh7e2m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720434000,"log":"[2026-02-17T12:58:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-re11n3d7b5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720435000,"log":"[2026-02-17T12:45:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jjwc4u9obp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720436000,"log":"[2026-02-17T13:46:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 190ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8pjm47r8eu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720437000,"log":"[2026-02-17T13:27:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2514395075250597400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vxoga1g0p8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720438000,"log":"[2026-02-17T13:20:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2017090579086866400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h5fnzeeaf0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720439000,"log":"[2026-02-17T13:11:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9286274996862384000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d1bdpoxzkk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720440000,"log":"[2026-02-17T13:50:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 228ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j2mu3son06","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720441000,"log":"[2026-02-17T12:37:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uoyjqyfs7x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720442000,"log":"[2026-02-17T14:27:13Z DEBUG sqlx::query] INSERT FROM users returned 757 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hnvpshha9r","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720443000,"log":"[2026-02-17T14:11:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 63 rows in 23ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ylq9w2wikg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720444000,"log":"[2026-02-17T14:51:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cofqwxm2e7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720445000,"log":"[2026-02-17T13:20:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3900895308276841500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8g6dh1icnx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720446000,"log":"[2026-02-17T15:20:21Z TRACE cache::redis] cache set key=auth:token:3021","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zs20d92tvq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720447000,"log":"[2026-02-17T14:15:58Z DEBUG sqlx::query] INSERT FROM users returned 274 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-15j7ystiai","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720448000,"log":"[2026-02-17T14:22:22Z DEBUG sqlx::query] SELECT FROM streams returned 870 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-545s211sg5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720449000,"log":"[2026-02-17T14:05:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 352ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ksch62pogp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720450000,"log":"[2026-02-17T13:42:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 340ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8cxytn0qdg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720451000,"log":"[2026-02-17T13:59:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 201 444ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zyo35m200w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720452000,"log":"[2026-02-17T15:05:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m83yoiz7eq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720453000,"log":"[2026-02-17T14:14:30Z DEBUG sqlx::query] DELETE FROM alerts returned 815 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wuwyyb67t2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720454000,"log":"[2026-02-17T13:33:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1206040607435303700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0tmklhsddj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720455000,"log":"[2026-02-17T14:14:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 161 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i1lismdhgz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720456000,"log":"[2026-02-17T14:25:25Z DEBUG sqlx::query] DELETE FROM alerts returned 722 rows in 20ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9x7mpferyl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720457000,"log":"[2026-02-17T13:04:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rnlvcspcel","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720458000,"log":"[2026-02-17T14:11:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 780 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zpc5alju7d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720459000,"log":"[2026-02-17T14:06:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 268ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8roimjvlki","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720460000,"log":"[2026-02-17T14:25:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 305 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rl0w28snvk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720461000,"log":"[2026-02-17T13:32:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3185141060967163000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-du0wb077en","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720462000,"log":"[2026-02-17T13:52:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 500ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gn4vgomf0a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720463000,"log":"[2026-02-17T12:43:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2na6i7abxw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720464000,"log":"[2026-02-17T15:19:44Z TRACE cache::redis] cache get key=stream:schema:258","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ocke9vul05","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720465000,"log":"[2026-02-17T14:31:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 277 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wfuifqo25b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720466000,"log":"[2026-02-17T13:07:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0gcab9kxbs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720467000,"log":"[2026-02-17T14:41:59Z INFO tonic::transport::server] grpc router request_received client_id=rf58zlqu","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1ny3gk5jhl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720468000,"log":"[2026-02-17T13:48:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 160ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8t3k82y1bj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720469000,"log":"[2026-02-17T15:10:55Z DEBUG metrics::reporter] recorded metric active_connections value=7976","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hb23k11c89","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720470000,"log":"[2026-02-17T12:43:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l2vigvt56c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720471000,"log":"[2026-02-17T15:13:34Z TRACE cache::redis] cache delete key=stream:schema:6140","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g4usonhsel","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720472000,"log":"[2026-02-17T15:14:42Z TRACE cache::redis] cache hit key=user:session:7630","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uq8e5xd470","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720473000,"log":"[2026-02-17T15:21:26Z TRACE cache::redis] cache get key=auth:token:6329","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2ngdlyrmfb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720474000,"log":"[2026-02-17T14:13:28Z DEBUG sqlx::query] INSERT FROM users returned 423 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4sybtn3us6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720475000,"log":"[2026-02-17T15:10:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7114","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fmtnxwkrx5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720476000,"log":"[2026-02-17T14:52:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o85hek7rf9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720477000,"log":"[2026-02-17T14:01:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 379ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qfdb5p0kad","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720478000,"log":"[2026-02-17T14:35:29Z INFO tonic::transport::server] grpc router request_received client_id=t5f91h4y","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ej4ew5uzdp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720479000,"log":"[2026-02-17T14:51:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rbkzmfsvlo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720480000,"log":"[2026-02-17T14:08:57Z DEBUG sqlx::query] SELECT FROM streams returned 159 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-cm7nd408g4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720481000,"log":"[2026-02-17T13:10:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6893164654743369000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t8q6s2y4s7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720482000,"log":"[2026-02-17T14:58:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4uf7sagvot","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720483000,"log":"[2026-02-17T12:57:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yd36un60ai","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720484000,"log":"[2026-02-17T15:21:25Z TRACE cache::redis] cache miss key=query:result:5249","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u9zz7kup6o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720485000,"log":"[2026-02-17T14:27:17Z DEBUG sqlx::query] SELECT FROM streams returned 936 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jd6fusv2y6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720486000,"log":"[2026-02-17T14:33:12Z INFO tonic::transport::server] grpc ingester connected client_id=wteethxw","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7n6pkp1895","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720487000,"log":"[2026-02-17T13:57:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 313ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fg8kil3u66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720488000,"log":"[2026-02-17T13:18:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4494098803143284700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lt9flmxza5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720489000,"log":"[2026-02-17T12:48:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lhtz53do5y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720490000,"log":"[2026-02-17T14:59:55Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i13pj0szmj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720491000,"log":"[2026-02-17T13:21:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1080012817110438400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fdumqo5bm7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720492000,"log":"[2026-02-17T12:46:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iij2f81ky2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720493000,"log":"[2026-02-17T13:06:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ciynbyrs62","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720494000,"log":"[2026-02-17T14:43:43Z INFO tonic::transport::server] grpc querier disconnected client_id=deb5v7o5","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9n2wj9eq7l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720495000,"log":"[2026-02-17T13:33:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/165963597677354370.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-u48de7l7ug","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720496000,"log":"[2026-02-17T13:27:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3823759331719352000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-voipka3mad","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720497000,"log":"[2026-02-17T13:42:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 285ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tpcu7a69yh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720498000,"log":"[2026-02-17T13:30:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3529522577542412300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xlk1upx2gm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720499000,"log":"[2026-02-17T12:54:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2487345vhc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720500000,"log":"[2026-02-17T13:17:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5695111366820132000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zdeud31xlk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720501000,"log":"[2026-02-17T14:24:17Z DEBUG sqlx::query] SELECT FROM streams returned 193 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-14tain4div","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720502000,"log":"[2026-02-17T13:46:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 500 82ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-frfrizorpz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720503000,"log":"[2026-02-17T13:30:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7289300491941091000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rhjvhc44k9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720504000,"log":"[2026-02-17T15:04:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yk80g0v9q5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720505000,"log":"[2026-02-17T14:44:09Z INFO tonic::transport::server] grpc router request_received client_id=p5l50wbp","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p5uf2utghm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720506000,"log":"[2026-02-17T13:04:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5g1065by06","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720507000,"log":"[2026-02-17T15:12:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=9539","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-010fjbdrhj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720508000,"log":"[2026-02-17T14:57:12Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ogy59i1joi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720509000,"log":"[2026-02-17T12:35:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jux58nd2f5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720510000,"log":"[2026-02-17T13:04:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5mh4pht1cm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720511000,"log":"[2026-02-17T14:42:07Z INFO tonic::transport::server] grpc ingester connected client_id=0pz880ft","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hwjhk6td6o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720512000,"log":"[2026-02-17T12:39:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sjam7o1086","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720513000,"log":"[2026-02-17T14:22:12Z DEBUG sqlx::query] SELECT FROM streams returned 520 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eyeruyqajc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720514000,"log":"[2026-02-17T14:15:15Z DEBUG sqlx::query] DELETE FROM alerts returned 634 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e4a4lt206f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720515000,"log":"[2026-02-17T15:20:49Z TRACE cache::redis] cache miss key=stream:schema:6014","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hmzi48zvxd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720516000,"log":"[2026-02-17T13:39:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8516136661248701000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aauu3bji4f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720517000,"log":"[2026-02-17T14:23:10Z DEBUG sqlx::query] DELETE FROM alerts returned 827 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3b2sdkxk1y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720518000,"log":"[2026-02-17T14:04:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 497ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9jx3reul2g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720519000,"log":"[2026-02-17T13:40:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2043452731696715500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rqddjo4qyz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720520000,"log":"[2026-02-17T13:21:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9029597237030575000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q0ft2jcbmz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720521000,"log":"[2026-02-17T13:43:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 373ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-53k8h5gvym","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720522000,"log":"[2026-02-17T14:40:59Z INFO tonic::transport::server] grpc router request_received client_id=tdzjg4bp","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-osij5jw2b0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720523000,"log":"[2026-02-17T14:22:58Z DEBUG sqlx::query] INSERT FROM users returned 409 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lsir70nit1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720524000,"log":"[2026-02-17T14:14:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 116 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-986eeld39k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720525000,"log":"[2026-02-17T13:19:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9881198047499364000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gamjpjle8x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720526000,"log":"[2026-02-17T13:04:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g7q4j3wbse","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720527000,"log":"[2026-02-17T14:07:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 468 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-a9i3z4ml3c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720528000,"log":"[2026-02-17T15:13:13Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3786","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1nko94q2f2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720529000,"log":"[2026-02-17T13:25:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7233882161237761000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-08f6opyjov","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720530000,"log":"[2026-02-17T13:57:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 370ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kbyxvcs0yf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720531000,"log":"[2026-02-17T13:26:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9463683745619728000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-recswq24cb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720532000,"log":"[2026-02-17T13:07:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ee6deshda9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720533000,"log":"[2026-02-17T13:59:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 201 309ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pz1wmg41f4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720534000,"log":"[2026-02-17T14:21:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 977 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w0wcmrntrz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720535000,"log":"[2026-02-17T13:24:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3565121263066924000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iq40ra4dbu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720536000,"log":"[2026-02-17T13:23:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4699366386949513000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d14pcjndev","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720537000,"log":"[2026-02-17T14:54:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f2i7q44y2y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720538000,"log":"[2026-02-17T12:58:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-225mew8j5q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720539000,"log":"[2026-02-17T14:02:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 106ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y11b7kxd0d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720540000,"log":"[2026-02-17T13:46:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 395ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mx59iutuhn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720541000,"log":"[2026-02-17T13:34:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3648998640474494500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-unlez115ox","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720542000,"log":"[2026-02-17T14:13:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 47 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-srpcq768gb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720543000,"log":"[2026-02-17T14:44:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ts0c8u0x","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ev3y2e0mrk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720544000,"log":"[2026-02-17T14:00:40Z INFO actix_web::middleware::logger] POST /api/_search 400 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hoqwsdfzda","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720545000,"log":"[2026-02-17T14:20:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 965 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mx6s9sw0gy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720546000,"log":"[2026-02-17T13:51:00Z INFO actix_web::middleware::logger] POST /api/_search 503 60ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uavdmb1zyo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720547000,"log":"[2026-02-17T15:10:46Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=185","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zxmbj0plw7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720548000,"log":"[2026-02-17T13:44:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 400 350ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mv5x4k40ty","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720549000,"log":"[2026-02-17T14:06:57Z DEBUG sqlx::query] SELECT FROM streams returned 949 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-04v20fily8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720550000,"log":"[2026-02-17T15:06:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7130","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cwty7jo7zz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720551000,"log":"[2026-02-17T13:45:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 437ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hlxil5d85r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720552000,"log":"[2026-02-17T14:26:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 130 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9o05rlwqim","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720553000,"log":"[2026-02-17T13:50:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 362ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oh07j5vquv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720554000,"log":"[2026-02-17T13:14:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3023759321035803600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sly55osufh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720555000,"log":"[2026-02-17T15:08:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7291","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ibxy06ndyr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720556000,"log":"[2026-02-17T14:36:49Z INFO tonic::transport::server] grpc router request_received client_id=v6d163jo","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zwmflsl7m4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720557000,"log":"[2026-02-17T13:12:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7752736713468200000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3sjl495jeu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720558000,"log":"[2026-02-17T13:39:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8859567649182904000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gdy5nvcvj3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720559000,"log":"[2026-02-17T13:35:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7255124251858247000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hfcvmzptx0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720560000,"log":"[2026-02-17T15:20:19Z TRACE cache::redis] cache miss key=stream:schema:4081","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j4a164029r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720561000,"log":"[2026-02-17T13:43:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 442ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pedht4nai6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720562000,"log":"[2026-02-17T13:13:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1083345743115509100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-56kdd3fqa0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720563000,"log":"[2026-02-17T13:30:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5175770630942029000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zujtrl4oaz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720564000,"log":"[2026-02-17T13:02:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6iawwoklx5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720565000,"log":"[2026-02-17T14:55:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cn47q3ckgr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720566000,"log":"[2026-02-17T13:04:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wcyg8jkmz7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720567000,"log":"[2026-02-17T13:50:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 82ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yytd6dawqx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720568000,"log":"[2026-02-17T14:22:10Z DEBUG sqlx::query] DELETE FROM alerts returned 643 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ihhsmjd1w8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720569000,"log":"[2026-02-17T13:07:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j5bu3zdm0m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720570000,"log":"[2026-02-17T14:14:50Z DEBUG sqlx::query] DELETE FROM alerts returned 468 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1xyhck8foz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720571000,"log":"[2026-02-17T12:38:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2ulybdyr7y","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720572000,"log":"[2026-02-17T13:34:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3115168733537647600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qnssa3020e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720573000,"log":"[2026-02-17T12:58:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0rtkfo3cza","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720574000,"log":"[2026-02-17T15:12:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1888","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u6ql9v8mg7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720575000,"log":"[2026-02-17T12:35:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-72m3fjm0ow","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720576000,"log":"[2026-02-17T12:44:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-so0985btaw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720577000,"log":"[2026-02-17T14:35:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=qgru9fl1","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-194wtdq1ih","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720578000,"log":"[2026-02-17T14:09:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 692 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qrka4np7z9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720579000,"log":"[2026-02-17T12:36:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0mgbasczvf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720580000,"log":"[2026-02-17T14:01:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 325ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v860t9rviz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720581000,"log":"[2026-02-17T13:50:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 496ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s8cmn4h7pv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720582000,"log":"[2026-02-17T15:14:18Z TRACE cache::redis] cache hit key=org:config:5015","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xefpw79c88","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720583000,"log":"[2026-02-17T13:08:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1186852382165777200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fnlrh4cujp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720584000,"log":"[2026-02-17T14:12:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 63 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zpa61zferk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720585000,"log":"[2026-02-17T12:45:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5brrd1f2wx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720586000,"log":"[2026-02-17T13:41:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5509642188356498000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jeq8g89wb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720587000,"log":"[2026-02-17T14:47:23Z INFO tonic::transport::server] grpc querier disconnected client_id=9fw2qm4c","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j74gzd3g9h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720588000,"log":"[2026-02-17T13:07:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fzdl9aq98s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720589000,"log":"[2026-02-17T14:08:03Z DEBUG sqlx::query] INSERT FROM users returned 309 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z1jbm9827q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720590000,"log":"[2026-02-17T14:13:57Z DEBUG sqlx::query] SELECT FROM streams returned 945 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b3byxsxskp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720591000,"log":"[2026-02-17T14:28:52Z DEBUG sqlx::query] SELECT FROM streams returned 709 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w5gb099lnw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720592000,"log":"[2026-02-17T13:53:25Z INFO actix_web::middleware::logger] GET /api/_search 200 219ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m4cdm5ek8r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720593000,"log":"[2026-02-17T14:47:47Z INFO tonic::transport::server] grpc ingester connected client_id=5o5vg8k5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mh3u7e5x92","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720594000,"log":"[2026-02-17T12:40:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yj290uqcec","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720595000,"log":"[2026-02-17T13:33:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8956828569440414000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sd4lfboj7d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720596000,"log":"[2026-02-17T13:57:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 64ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0e8bga83pq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720597000,"log":"[2026-02-17T13:55:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 200 317ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-39bussfyow","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720598000,"log":"[2026-02-17T15:13:59Z TRACE cache::redis] cache expire key=stream:schema:2810","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8nicthpdo0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720599000,"log":"[2026-02-17T12:40:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-aimziarbff","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720600000,"log":"[2026-02-17T13:42:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 401 11ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kreevc0oco","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720601000,"log":"[2026-02-17T14:43:39Z INFO tonic::transport::server] grpc router request_received client_id=n2eh5m5p","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hrvqn5x51p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720602000,"log":"[2026-02-17T13:56:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 447ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cj58vpqm84","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720603000,"log":"[2026-02-17T14:51:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c6w3bwqql1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720604000,"log":"[2026-02-17T14:40:09Z INFO tonic::transport::server] grpc router request_received client_id=qlp25gcc","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i5rm2cribr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720605000,"log":"[2026-02-17T13:05:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3mrpxh67qv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720606000,"log":"[2026-02-17T15:10:05Z DEBUG metrics::reporter] recorded metric active_connections value=3113","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-itpvq2j3fs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720607000,"log":"[2026-02-17T14:35:18Z INFO tonic::transport::server] grpc querier disconnected client_id=uysimy2f","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f6cj1g7q9v","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720608000,"log":"[2026-02-17T14:11:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 644 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5wl0i677sc","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720609000,"log":"[2026-02-17T15:10:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=7568","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rgt66l2g0y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720610000,"log":"[2026-02-17T15:12:12Z DEBUG metrics::reporter] recorded metric http_requests_total value=7336","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gka09olsjz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720611000,"log":"[2026-02-17T15:21:08Z TRACE cache::redis] cache get key=org:config:1088","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t3kmglgen9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720612000,"log":"[2026-02-17T12:55:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zfjfvp1t7i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720613000,"log":"[2026-02-17T14:56:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-btsvwz613q","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720614000,"log":"[2026-02-17T12:42:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-70wtkszi60","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720615000,"log":"[2026-02-17T14:13:42Z DEBUG sqlx::query] SELECT FROM streams returned 596 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hbtpj6gc6i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720616000,"log":"[2026-02-17T13:09:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8110523538657657000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r48vwi2i46","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720617000,"log":"[2026-02-17T15:13:49Z TRACE cache::redis] cache miss key=stream:schema:6172","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t1rh1bgs3o","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720618000,"log":"[2026-02-17T15:00:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xnf2t34de","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720619000,"log":"[2026-02-17T12:52:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-znl08yv3ga","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720620000,"log":"[2026-02-17T14:22:38Z DEBUG sqlx::query] INSERT FROM users returned 24 rows in 21ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-78tiev84l4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720621000,"log":"[2026-02-17T12:57:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vu340ghgd7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720622000,"log":"[2026-02-17T12:48:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-40zyfktlvi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720623000,"log":"[2026-02-17T14:50:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f4e8hvkjf8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720624000,"log":"[2026-02-17T14:13:47Z DEBUG sqlx::query] SELECT FROM streams returned 923 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lr8q6iht82","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720625000,"log":"[2026-02-17T14:38:57Z INFO tonic::transport::server] grpc ingester connected client_id=8a5foum7","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j6vru2rp8s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720626000,"log":"[2026-02-17T15:11:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=6384","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-je7kzfj0ho","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720627000,"log":"[2026-02-17T13:46:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 499ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v2yabao120","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720628000,"log":"[2026-02-17T14:08:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 231 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uh42a9h0a3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720629000,"log":"[2026-02-17T13:37:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9854839498574455000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-sj4xint3j6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720630000,"log":"[2026-02-17T13:20:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2892048470678332400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-otmay026hu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720631000,"log":"[2026-02-17T13:25:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4193451075225654000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zrjezqvpfc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720632000,"log":"[2026-02-17T15:07:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=3760","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9p0plwp1uk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720633000,"log":"[2026-02-17T15:11:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=1943","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i52johvgoc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720634000,"log":"[2026-02-17T14:15:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 950 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mwdvkq9gfk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720635000,"log":"[2026-02-17T14:36:07Z INFO tonic::transport::server] grpc ingester connected client_id=mnoxjuex","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-npj22l9v33","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720636000,"log":"[2026-02-17T15:19:57Z TRACE cache::redis] cache set key=user:session:3047","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-75xhqqc8x6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720637000,"log":"[2026-02-17T14:15:27Z DEBUG sqlx::query] SELECT FROM streams returned 697 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vanlbdbp6l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720638000,"log":"[2026-02-17T14:18:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 216 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-36ccrs16sl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720639000,"log":"[2026-02-17T13:53:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 5ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uprqa47q48","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720640000,"log":"[2026-02-17T14:57:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-quxj5aotd3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720641000,"log":"[2026-02-17T13:04:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ipsid5dpkc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720642000,"log":"[2026-02-17T15:21:09Z TRACE cache::redis] cache set key=stream:schema:6857","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c9ts2niasf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720643000,"log":"[2026-02-17T15:03:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jjg4tam44j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720644000,"log":"[2026-02-17T15:19:42Z TRACE cache::redis] cache hit key=user:session:8734","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3phwr1vctg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720645000,"log":"[2026-02-17T14:09:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 542 rows in 92ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mcmzk7dk97","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720646000,"log":"[2026-02-17T14:38:23Z INFO tonic::transport::server] grpc querier disconnected client_id=yw9mh8fe","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4pgr8slvpy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720647000,"log":"[2026-02-17T14:43:33Z INFO tonic::transport::server] grpc querier disconnected client_id=wwfleaam","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lljp68xoz4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720648000,"log":"[2026-02-17T13:16:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5292430001836168000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pzy9r81i81","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720649000,"log":"[2026-02-17T13:36:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5187897185556145000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2dmsklxrza","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720650000,"log":"[2026-02-17T14:28:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 605 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m160z357vr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720651000,"log":"[2026-02-17T14:31:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 424 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1q35tzrlkw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720652000,"log":"[2026-02-17T12:39:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-c9m3pgvipi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720653000,"log":"[2026-02-17T14:10:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 898 rows in 74ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6tsn62xl68","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720654000,"log":"[2026-02-17T15:19:38Z TRACE cache::redis] cache get key=org:config:7332","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fffnyahq52","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720655000,"log":"[2026-02-17T13:55:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 251ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ksx20aq4tp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720656000,"log":"[2026-02-17T12:49:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r8x3gp4ggi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720657000,"log":"[2026-02-17T14:48:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mgc3tnmxmy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720658000,"log":"[2026-02-17T13:15:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5966996979135653000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zacgl4loul","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720659000,"log":"[2026-02-17T14:34:38Z INFO tonic::transport::server] grpc querier disconnected client_id=tqpl9rz4","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1d5or3oad6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720660000,"log":"[2026-02-17T14:27:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 56 rows in 11ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jy79c1jy66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720661000,"log":"[2026-02-17T15:06:41Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5295","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h5xi97mr2s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720662000,"log":"[2026-02-17T13:43:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 376ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b41f1ue57s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720663000,"log":"[2026-02-17T14:09:12Z DEBUG sqlx::query] SELECT FROM streams returned 781 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5170octmpf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720664000,"log":"[2026-02-17T14:53:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q8z2bwbp9h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720665000,"log":"[2026-02-17T14:47:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=bjmhuvf4","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-62tjkkz99j","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720666000,"log":"[2026-02-17T12:48:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1w0j2kvnw6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720667000,"log":"[2026-02-17T12:42:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9db1jt86xl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720668000,"log":"[2026-02-17T12:58:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7vet3m1r51","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720669000,"log":"[2026-02-17T14:06:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 64ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rdwzn2zlh7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720670000,"log":"[2026-02-17T14:38:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=8tpj75ht","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9pnn0uzrr6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720671000,"log":"[2026-02-17T13:00:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vity0e3v8y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720672000,"log":"[2026-02-17T14:27:12Z DEBUG sqlx::query] SELECT FROM streams returned 476 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xaraf3kb2u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720673000,"log":"[2026-02-17T14:46:58Z INFO tonic::transport::server] grpc querier disconnected client_id=3qt25p2z","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-muft6akvqr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720674000,"log":"[2026-02-17T14:43:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=88n3rrjc","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hwzri2jl3c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720675000,"log":"[2026-02-17T13:28:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3104439914102825500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bvzloqqbpd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720676000,"log":"[2026-02-17T13:14:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6513931374517230000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-raj9fh7zqs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720677000,"log":"[2026-02-17T15:21:15Z TRACE cache::redis] cache set key=query:result:2771","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0mowbxkgzo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720678000,"log":"[2026-02-17T13:39:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7276955382171821000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bfctvo5su5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720679000,"log":"[2026-02-17T14:13:07Z DEBUG sqlx::query] SELECT FROM streams returned 338 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-49f05vj44d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720680000,"log":"[2026-02-17T12:35:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-97m794p9gv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720681000,"log":"[2026-02-17T14:09:58Z DEBUG sqlx::query] INSERT FROM users returned 42 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g3e4e9dclr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720682000,"log":"[2026-02-17T12:55:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e6jl3fsj9e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720683000,"log":"[2026-02-17T14:50:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-do510z3thk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720684000,"log":"[2026-02-17T15:19:47Z TRACE cache::redis] cache expire key=user:session:2289","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-778uwye4ym","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720685000,"log":"[2026-02-17T14:29:52Z DEBUG sqlx::query] SELECT FROM streams returned 710 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xdobqofqgd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720686000,"log":"[2026-02-17T12:48:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i0vb1qlelx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720687000,"log":"[2026-02-17T13:02:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0rt2d1cxsk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720688000,"log":"[2026-02-17T14:52:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pesiv2oxte","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720689000,"log":"[2026-02-17T13:28:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2103015659890954500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zocdwbb4tk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720690000,"log":"[2026-02-17T15:20:55Z TRACE cache::redis] cache miss key=query:result:2064","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ka473rfu86","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720691000,"log":"[2026-02-17T12:46:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zd0818o2lr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720692000,"log":"[2026-02-17T14:41:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=py6jpv8p","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hju4jknugx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720693000,"log":"[2026-02-17T15:20:09Z TRACE cache::redis] cache set key=stream:schema:790","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-avk8200bhg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720694000,"log":"[2026-02-17T12:53:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vum1jj69ei","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720695000,"log":"[2026-02-17T14:15:35Z DEBUG sqlx::query] DELETE FROM alerts returned 554 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ba2dx5kxja","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720696000,"log":"[2026-02-17T14:13:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 254 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b4umsqbgz1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720697000,"log":"[2026-02-17T14:00:39Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 204 114ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nbqwshui6x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720698000,"log":"[2026-02-17T12:48:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jrujgtn0pl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720699000,"log":"[2026-02-17T14:38:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=kla4l8c1","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jzzhyseedh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720700000,"log":"[2026-02-17T12:52:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u4svwq4347","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720701000,"log":"[2026-02-17T12:59:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tihyy3t1et","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720702000,"log":"[2026-02-17T15:14:44Z TRACE cache::redis] cache get key=stream:schema:5041","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bsg0cneltx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720703000,"log":"[2026-02-17T14:35:09Z INFO tonic::transport::server] grpc router request_received client_id=e9ik6o4l","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-clgp5xa4dx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720704000,"log":"[2026-02-17T14:17:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 831 rows in 79ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7108v2jql3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720705000,"log":"[2026-02-17T12:41:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ilkem92pw9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720706000,"log":"[2026-02-17T14:15:03Z DEBUG sqlx::query] INSERT FROM users returned 553 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-31f298q038","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720707000,"log":"[2026-02-17T13:36:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2568457363532679000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8v92gioih7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720708000,"log":"[2026-02-17T13:05:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p38bbxavfl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720709000,"log":"[2026-02-17T13:11:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/307558602982780350.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-f2i434o2y4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720710000,"log":"[2026-02-17T12:48:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nyf2ypeqhw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720711000,"log":"[2026-02-17T13:02:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xzv65pxalg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720712000,"log":"[2026-02-17T13:42:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 159ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5ukxzw3n5g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720713000,"log":"[2026-02-17T13:48:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 313ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-51atzh7f26","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720714000,"log":"[2026-02-17T13:25:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4076459638810486000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w4tbd505nk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720715000,"log":"[2026-02-17T14:35:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=4yo27qh6","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-680rlbd78e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720716000,"log":"[2026-02-17T14:47:28Z INFO tonic::transport::server] grpc querier disconnected client_id=zcf17va8","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ljzm9nccuv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720717000,"log":"[2026-02-17T13:01:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v52m7a54yp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720718000,"log":"[2026-02-17T15:05:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=735","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7ytnsb8t4n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720719000,"log":"[2026-02-17T14:07:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 496 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-danf6kj6sw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720720000,"log":"[2026-02-17T12:36:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-63d6r507gi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720721000,"log":"[2026-02-17T14:06:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 226ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5gq59kz8qt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720722000,"log":"[2026-02-17T13:08:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6892205926486714000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-oyqmoe2ynh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720723000,"log":"[2026-02-17T12:51:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uvwta4bwow","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720724000,"log":"[2026-02-17T14:10:00Z DEBUG sqlx::query] DELETE FROM alerts returned 714 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l8sdv9nn6y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720725000,"log":"[2026-02-17T14:31:28Z DEBUG sqlx::query] INSERT FROM users returned 219 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-os7szjl5ye","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720726000,"log":"[2026-02-17T14:34:59Z INFO tonic::transport::server] grpc router request_received client_id=yi70v9dw","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qlfp3uhfb3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720727000,"log":"[2026-02-17T12:56:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xbllvwvsgy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720728000,"log":"[2026-02-17T12:55:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jflqaon67r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720729000,"log":"[2026-02-17T13:23:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3242661573466146000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w7wrmmpw8p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720730000,"log":"[2026-02-17T13:20:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6867672832988152000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-an4xeil3kk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720731000,"log":"[2026-02-17T14:24:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 665 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gkqavbyb2i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720732000,"log":"[2026-02-17T14:08:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 606 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-00k3rsuf6a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720733000,"log":"[2026-02-17T14:31:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 140 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-toak4a6ylw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720734000,"log":"[2026-02-17T14:41:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ilcbzkvx","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1q2y1u060h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720735000,"log":"[2026-02-17T12:42:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5im2sjxysu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720736000,"log":"[2026-02-17T13:26:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5094123759604030000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5w3a5nmitm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720737000,"log":"[2026-02-17T14:40:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=l7rkjo4x","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nrbq380l30","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720738000,"log":"[2026-02-17T14:53:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jghmzm90ol","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720739000,"log":"[2026-02-17T12:56:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-68bbp48gib","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720740000,"log":"[2026-02-17T14:52:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0o19km87y9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720741000,"log":"[2026-02-17T14:47:08Z INFO tonic::transport::server] grpc querier disconnected client_id=hj5dzmaw","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l7n0r4g072","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720742000,"log":"[2026-02-17T13:14:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2373958474132653600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s4afmtv9n5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720743000,"log":"[2026-02-17T14:19:48Z DEBUG sqlx::query] INSERT FROM users returned 740 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wxw6txamgp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720744000,"log":"[2026-02-17T14:26:05Z DEBUG sqlx::query] DELETE FROM alerts returned 835 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e55b52crcx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720745000,"log":"[2026-02-17T14:07:38Z DEBUG sqlx::query] INSERT FROM users returned 461 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pyu8fri71q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720746000,"log":"[2026-02-17T14:20:47Z DEBUG sqlx::query] SELECT FROM streams returned 901 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8cfa2dk556","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720747000,"log":"[2026-02-17T13:32:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9348078043670536000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d988kjbiu5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720748000,"log":"[2026-02-17T13:22:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3257827415070278000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b9ad83swe1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720749000,"log":"[2026-02-17T14:46:07Z INFO tonic::transport::server] grpc ingester connected client_id=jk9es65s","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wsgrn2yem4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720750000,"log":"[2026-02-17T13:43:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 99ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-34fm6uhw3s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720751000,"log":"[2026-02-17T12:53:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gfy4b5p0r0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720752000,"log":"[2026-02-17T13:51:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 403 120ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7h30vg4rcs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720753000,"log":"[2026-02-17T13:10:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7603979331779382000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bca4igb7ny","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720754000,"log":"[2026-02-17T13:01:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2gs7lpcv4f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720755000,"log":"[2026-02-17T12:35:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a75ym0g39w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720756000,"log":"[2026-02-17T12:52:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vsgad0kwug","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720757000,"log":"[2026-02-17T13:39:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2011804834552384300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k8afmn0tcx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720758000,"log":"[2026-02-17T13:14:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/534791155397009700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-obyanuwvr9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720759000,"log":"[2026-02-17T14:56:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zrg9xolxce","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720760000,"log":"[2026-02-17T15:11:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4462","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-il0sstz6gx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720761000,"log":"[2026-02-17T12:51:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eik7du8rwk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720762000,"log":"[2026-02-17T13:40:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1894356496910290000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5kpqfxuzfq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720763000,"log":"[2026-02-17T14:51:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8z04mgv7vw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720764000,"log":"[2026-02-17T13:46:20Z INFO actix_web::middleware::logger] POST /api/_search 500 132ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mgiy7uqo7h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720765000,"log":"[2026-02-17T14:20:10Z DEBUG sqlx::query] DELETE FROM alerts returned 75 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xi2un7zay6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720766000,"log":"[2026-02-17T14:35:22Z INFO tonic::transport::server] grpc ingester connected client_id=ugognkpk","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bt48gfi8of","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720767000,"log":"[2026-02-17T13:04:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wiw9kbehm8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720768000,"log":"[2026-02-17T14:02:56Z INFO actix_web::middleware::logger] POST /api/_bulk 401 75ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-243ywucjs0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720769000,"log":"[2026-02-17T15:09:55Z DEBUG metrics::reporter] recorded metric active_connections value=5253","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-emvgy8lt79","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720770000,"log":"[2026-02-17T12:37:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-71biga2j0z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720771000,"log":"[2026-02-17T13:20:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9116267728547640000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bkog0od9wf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720772000,"log":"[2026-02-17T15:00:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pv1g6y5jki","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720773000,"log":"[2026-02-17T14:05:41Z INFO actix_web::middleware::logger] GET /api/_bulk 500 83ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-16cql5orw5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720774000,"log":"[2026-02-17T12:45:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c2t8pho4bv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720775000,"log":"[2026-02-17T14:24:33Z DEBUG sqlx::query] INSERT FROM users returned 757 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1jx6epgo54","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720776000,"log":"[2026-02-17T13:12:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4563797328213030400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fg3f3trw3p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720777000,"log":"[2026-02-17T13:42:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 281ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uillwshxeo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720778000,"log":"[2026-02-17T12:55:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3cg1804er1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720779000,"log":"[2026-02-17T14:09:38Z DEBUG sqlx::query] INSERT FROM users returned 560 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q99qge0hbl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720780000,"log":"[2026-02-17T15:15:11Z TRACE cache::redis] cache expire key=auth:token:7975","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iip83x1jml","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720781000,"log":"[2026-02-17T14:17:35Z DEBUG sqlx::query] DELETE FROM alerts returned 871 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bgptftvf20","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720782000,"log":"[2026-02-17T13:26:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3621436866820595000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o4mgqj0no7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720783000,"log":"[2026-02-17T13:34:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3785367974898428000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-430lyoue4n","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720784000,"log":"[2026-02-17T14:13:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 247 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cpxdyb5lyd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720785000,"log":"[2026-02-17T12:50:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n3sn0d9znj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720786000,"log":"[2026-02-17T14:15:10Z DEBUG sqlx::query] DELETE FROM alerts returned 496 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ktdf56rhxp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720787000,"log":"[2026-02-17T13:07:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wymd2wk7aa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720788000,"log":"[2026-02-17T12:44:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l3fgvymxwz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720789000,"log":"[2026-02-17T13:53:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 45ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kqt7hy2l9g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720790000,"log":"[2026-02-17T14:24:07Z DEBUG sqlx::query] SELECT FROM streams returned 655 rows in 84ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0u44ignsbf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720791000,"log":"[2026-02-17T13:41:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5861852374303031000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g2o8dnkkq6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720792000,"log":"[2026-02-17T13:28:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5660011924177297000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rugqh1grn3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720793000,"log":"[2026-02-17T15:14:43Z TRACE cache::redis] cache miss key=org:config:1493","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gp4iznlg5n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720794000,"log":"[2026-02-17T13:42:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 408ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8ae7c3w81u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720795000,"log":"[2026-02-17T13:29:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/350256919881974900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gclccoy3b1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720796000,"log":"[2026-02-17T13:06:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3idmb5gkuz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720797000,"log":"[2026-02-17T15:04:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-afeyzz2ze2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720798000,"log":"[2026-02-17T14:14:02Z DEBUG sqlx::query] SELECT FROM streams returned 861 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-06jssqsb4y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720799000,"log":"[2026-02-17T14:39:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=b2cvwcsb","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tke65oeo09","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720800000,"log":"[2026-02-17T15:16:45Z TRACE cache::redis] cache set key=query:result:4488","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6trd18mkra","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720801000,"log":"[2026-02-17T13:42:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 459ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uosd88bh19","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720802000,"log":"[2026-02-17T12:47:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qr5jhcb7uj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720803000,"log":"[2026-02-17T15:02:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wke1g2eqfp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720804000,"log":"[2026-02-17T12:55:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7cmlpj64op","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720805000,"log":"[2026-02-17T13:06:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b7ry90cpc3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720806000,"log":"[2026-02-17T14:34:42Z INFO tonic::transport::server] grpc ingester connected client_id=kwwhcylv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kypp4czjhn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720807000,"log":"[2026-02-17T13:08:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4146065829543066000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lfvtyrpfbo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720808000,"log":"[2026-02-17T14:19:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 422 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bn26joxpid","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720809000,"log":"[2026-02-17T15:07:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=6174","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pawrvxu4ji","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720810000,"log":"[2026-02-17T14:49:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-urig793dxu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720811000,"log":"[2026-02-17T14:54:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lnf31m9gpl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720812000,"log":"[2026-02-17T12:40:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h48ig8t38b","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720813000,"log":"[2026-02-17T15:03:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dqibbo14cg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720814000,"log":"[2026-02-17T14:27:35Z DEBUG sqlx::query] DELETE FROM alerts returned 304 rows in 41ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-barxor1b4r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720815000,"log":"[2026-02-17T13:55:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 354ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9s7rmhu8u8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720816000,"log":"[2026-02-17T13:53:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 377ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b3f2xgmz8j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720817000,"log":"[2026-02-17T14:11:13Z DEBUG sqlx::query] INSERT FROM users returned 775 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-r6hwfw2ezv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720818000,"log":"[2026-02-17T14:17:42Z DEBUG sqlx::query] SELECT FROM streams returned 35 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vgcnch38gc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720819000,"log":"[2026-02-17T14:17:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 236 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pw1ufkh8ya","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720820000,"log":"[2026-02-17T12:40:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9secdq52u7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720821000,"log":"[2026-02-17T14:46:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=rm4kv8cr","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yyj1lasec1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720822000,"log":"[2026-02-17T12:59:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x6xkncc9w3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720823000,"log":"[2026-02-17T14:17:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 203 rows in 64ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fg75t59gbk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720824000,"log":"[2026-02-17T13:51:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 109ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wwztw0q3mv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720825000,"log":"[2026-02-17T13:34:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9699524689461598000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2j2t73ikxh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720826000,"log":"[2026-02-17T14:05:20Z INFO actix_web::middleware::logger] POST /api/_search 401 246ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-92n6i6sxu5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720827000,"log":"[2026-02-17T15:14:13Z TRACE cache::redis] cache miss key=org:config:7796","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-b5sdol8dps","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720828000,"log":"[2026-02-17T14:10:28Z DEBUG sqlx::query] INSERT FROM users returned 349 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-otdewfzaqf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720829000,"log":"[2026-02-17T12:41:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-equu1h6d9t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720830000,"log":"[2026-02-17T14:36:52Z INFO tonic::transport::server] grpc ingester connected client_id=so2uabxv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kk88eqfb7m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720831000,"log":"[2026-02-17T14:43:49Z INFO tonic::transport::server] grpc router request_received client_id=7wnlv9ye","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-99t1l2djaq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720832000,"log":"[2026-02-17T13:41:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5183472699269851000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-em9oa13s1b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720833000,"log":"[2026-02-17T14:40:54Z INFO tonic::transport::server] grpc router request_received client_id=t48ubqfb","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jysmnviu8p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720834000,"log":"[2026-02-17T13:58:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 346ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-66pyxo0aaz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720835000,"log":"[2026-02-17T15:16:10Z TRACE cache::redis] cache delete key=query:result:7699","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9arwwfyyxl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720836000,"log":"[2026-02-17T13:19:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8542274182830767000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lscehnvkbt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720837000,"log":"[2026-02-17T12:53:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wxy7obf7jh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720838000,"log":"[2026-02-17T13:12:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7604408971870080000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sycqqfr0rk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720839000,"log":"[2026-02-17T15:14:41Z TRACE cache::redis] cache expire key=auth:token:5227","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uszp7vj2p9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720840000,"log":"[2026-02-17T14:39:13Z INFO tonic::transport::server] grpc querier disconnected client_id=6e8a7eml","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c0yi7uaf6q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720841000,"log":"[2026-02-17T14:24:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 617 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a3qfo6uy2z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720842000,"log":"[2026-02-17T14:26:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 90 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v6we39rt3h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720843000,"log":"[2026-02-17T14:02:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 128ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qczgm6tza1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720844000,"log":"[2026-02-17T14:05:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 409ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-572pqok9kq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720845000,"log":"[2026-02-17T12:58:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v3okyaa9k5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720846000,"log":"[2026-02-17T13:28:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1536817651821902300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5f308prkm6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720847000,"log":"[2026-02-17T13:25:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9838170463858584000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wuh0ek8vop","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720848000,"log":"[2026-02-17T13:56:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 42ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s4kc3ua4co","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720849000,"log":"[2026-02-17T14:10:15Z DEBUG sqlx::query] DELETE FROM alerts returned 211 rows in 39ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gdhrayfyxn","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720850000,"log":"[2026-02-17T13:56:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 244ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5j2864q38y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720851000,"log":"[2026-02-17T14:56:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ogg2uxpwps","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720852000,"log":"[2026-02-17T12:51:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-b0a14e5lqt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720853000,"log":"[2026-02-17T13:12:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/968516027991648300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9n6rkij1er","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720854000,"log":"[2026-02-17T14:47:27Z INFO tonic::transport::server] grpc ingester connected client_id=0drzkdox","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jlcel69apu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720855000,"log":"[2026-02-17T13:40:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9407566393018714000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y63pp28umd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720856000,"log":"[2026-02-17T15:03:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5plvzkl964","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720857000,"log":"[2026-02-17T14:59:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h16dgpngwt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720858000,"log":"[2026-02-17T15:10:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=1","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4u18tuyr0u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720859000,"log":"[2026-02-17T13:41:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1638698688241813000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-g199eu9ylx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720860000,"log":"[2026-02-17T13:12:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4005619650941061000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ltgxrasf8p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720861000,"log":"[2026-02-17T13:48:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 500 230ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pyesio8tc1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720862000,"log":"[2026-02-17T14:28:15Z DEBUG sqlx::query] DELETE FROM alerts returned 932 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cetbx93zwi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720863000,"log":"[2026-02-17T14:01:30Z INFO actix_web::middleware::logger] PUT /api/_search 503 243ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e2eq3ch19f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720864000,"log":"[2026-02-17T14:55:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-38o988u1cl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720865000,"log":"[2026-02-17T14:27:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 50 rows in 40ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zap8xor6d6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720866000,"log":"[2026-02-17T13:48:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 200 124ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5qo5xaxj15","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720867000,"log":"[2026-02-17T14:36:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=zamicj9j","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-llgdg3oyz9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720868000,"log":"[2026-02-17T13:03:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zmjvzli341","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720869000,"log":"[2026-02-17T13:54:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 447ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6eusknzr27","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720870000,"log":"[2026-02-17T13:37:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9542084490509600000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-67x913hmwb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720871000,"log":"[2026-02-17T15:04:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ingf405czt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720872000,"log":"[2026-02-17T12:35:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9pajv6i4zu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720873000,"log":"[2026-02-17T12:48:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ee2e8pa502","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720874000,"log":"[2026-02-17T12:53:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qffacy94hn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720875000,"log":"[2026-02-17T13:02:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4ytktvft72","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720876000,"log":"[2026-02-17T12:56:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-89bqi2gnoz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720877000,"log":"[2026-02-17T13:50:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 12ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7pucxepg0l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720878000,"log":"[2026-02-17T13:42:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 291ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-07tz9khp3p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720879000,"log":"[2026-02-17T15:15:24Z TRACE cache::redis] cache hit key=stream:schema:4117","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yf31978e0x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720880000,"log":"[2026-02-17T13:21:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8651065433901933000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ki2dofikun","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720881000,"log":"[2026-02-17T14:35:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=tlzvd12o","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rgk01t451z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720882000,"log":"[2026-02-17T14:29:48Z DEBUG sqlx::query] INSERT FROM users returned 51 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nj7b1t109p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720883000,"log":"[2026-02-17T12:50:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7aktdk1x0r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720884000,"log":"[2026-02-17T15:09:38Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9668","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5sl9c91xdj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720885000,"log":"[2026-02-17T13:41:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 405ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rrnwpo3297","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720886000,"log":"[2026-02-17T13:01:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hkgfzfc87a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720887000,"log":"[2026-02-17T15:14:37Z TRACE cache::redis] cache miss key=user:session:2856","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jydhx8s6jx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720888000,"log":"[2026-02-17T12:52:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rjfa18h7ij","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720889000,"log":"[2026-02-17T14:37:17Z INFO tonic::transport::server] grpc ingester connected client_id=o7gzovwh","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rtqkoy8n97","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720890000,"log":"[2026-02-17T13:56:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 281ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2huqfmxza0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720891000,"log":"[2026-02-17T14:38:02Z INFO tonic::transport::server] grpc ingester connected client_id=bqnxcknv","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7m8tnx1nzf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720892000,"log":"[2026-02-17T12:52:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r0ji8zggag","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720893000,"log":"[2026-02-17T13:24:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1018486372154294700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0dquupvpmf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720894000,"log":"[2026-02-17T14:10:23Z DEBUG sqlx::query] INSERT FROM users returned 180 rows in 13ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e88amfx14a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720895000,"log":"[2026-02-17T15:19:05Z TRACE cache::redis] cache expire key=query:result:9299","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ymzcha67i1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720896000,"log":"[2026-02-17T12:44:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a7y6x8oxos","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720897000,"log":"[2026-02-17T14:43:32Z INFO tonic::transport::server] grpc ingester connected client_id=yq5lukdt","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hp7k4uisms","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720898000,"log":"[2026-02-17T13:15:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5299864052470206000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vxu0uec2rq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720899000,"log":"[2026-02-17T14:30:55Z DEBUG sqlx::query] DELETE FROM alerts returned 260 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zatwd7hups","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720900000,"log":"[2026-02-17T13:25:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5877179313684844000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rs8hfx8g66","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720901000,"log":"[2026-02-17T12:59:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-263fom6trf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720902000,"log":"[2026-02-17T13:50:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 204 373ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-67lakd69yh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720903000,"log":"[2026-02-17T14:25:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 439 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r6qolsus35","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720904000,"log":"[2026-02-17T13:24:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/631320678914493200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qawibz8dwe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720905000,"log":"[2026-02-17T13:22:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2168239477221400800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-98kk6ht9zv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720906000,"log":"[2026-02-17T13:40:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4078069451366011000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4j7ylob4dk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720907000,"log":"[2026-02-17T14:28:05Z DEBUG sqlx::query] DELETE FROM alerts returned 30 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4gozpf2vo0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720908000,"log":"[2026-02-17T14:20:40Z DEBUG sqlx::query] DELETE FROM alerts returned 890 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-e9vleh0w3h","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720909000,"log":"[2026-02-17T13:04:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-44ouxsp23j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720910000,"log":"[2026-02-17T15:01:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6yw3yscf4w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720911000,"log":"[2026-02-17T13:18:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5621755626679212000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nzxsptidth","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720912000,"log":"[2026-02-17T14:06:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 431ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9jtdo3uyiu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720913000,"log":"[2026-02-17T13:00:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u865yr6lxd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720914000,"log":"[2026-02-17T13:47:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 65ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e25dc8f7u8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720915000,"log":"[2026-02-17T13:28:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4713803816728728000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sb3yqhfy9l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720916000,"log":"[2026-02-17T15:05:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xurpfbdl6f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720917000,"log":"[2026-02-17T14:37:07Z INFO tonic::transport::server] grpc ingester connected client_id=bxzq2oeb","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wd9rq9vcpv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720918000,"log":"[2026-02-17T12:52:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rp0osl1tfr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720919000,"log":"[2026-02-17T14:36:38Z INFO tonic::transport::server] grpc querier disconnected client_id=uhrfnmng","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8uhq7rsmig","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720920000,"log":"[2026-02-17T14:44:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=x18505i0","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nmy1xz1toq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720921000,"log":"[2026-02-17T13:16:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7886051615027560000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-en8l6e9jia","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720922000,"log":"[2026-02-17T14:44:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=yrjf720l","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lwn259x6lg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720923000,"log":"[2026-02-17T14:32:13Z INFO tonic::transport::server] grpc querier disconnected client_id=42ly6jgt","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mwdampgj38","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720924000,"log":"[2026-02-17T14:09:47Z DEBUG sqlx::query] SELECT FROM streams returned 229 rows in 5ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h3zmnzofan","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720925000,"log":"[2026-02-17T15:19:31Z TRACE cache::redis] cache miss key=auth:token:3401","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-86m12f937l","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720926000,"log":"[2026-02-17T14:24:48Z DEBUG sqlx::query] INSERT FROM users returned 872 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2pwlchpxg3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720927000,"log":"[2026-02-17T13:18:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9884000592931380000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-de12rxzg2z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720928000,"log":"[2026-02-17T13:05:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-z2fb7gptdm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720929000,"log":"[2026-02-17T12:57:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ueu1b5pffx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720930000,"log":"[2026-02-17T13:58:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 401 211ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tna5sjzn6o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720931000,"log":"[2026-02-17T14:22:13Z DEBUG sqlx::query] INSERT FROM users returned 711 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lbi0yap2lj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720932000,"log":"[2026-02-17T14:29:38Z DEBUG sqlx::query] INSERT FROM users returned 541 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p6qlukeghm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720933000,"log":"[2026-02-17T12:52:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yyy11h5n85","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720934000,"log":"[2026-02-17T14:45:58Z INFO tonic::transport::server] grpc querier disconnected client_id=x8sack0y","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8sd6jheu75","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720935000,"log":"[2026-02-17T14:59:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-35i69pchar","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720936000,"log":"[2026-02-17T14:41:24Z INFO tonic::transport::server] grpc router request_received client_id=z9obgupp","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-shp070gj58","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720937000,"log":"[2026-02-17T13:58:05Z INFO actix_web::middleware::logger] GET /api/_search 201 247ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4l4nlp11sg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720938000,"log":"[2026-02-17T14:02:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 327ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w8pjo0hyxd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720939000,"log":"[2026-02-17T12:55:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tzsia32zve","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720940000,"log":"[2026-02-17T13:22:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7668117486796666000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b3qta6ag8u","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720941000,"log":"[2026-02-17T13:48:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 500 397ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9zru3f946i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720942000,"log":"[2026-02-17T15:09:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1281","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z80rinort3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720943000,"log":"[2026-02-17T13:10:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2360901428973725000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s7s4e1aqod","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720944000,"log":"[2026-02-17T13:28:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2451796242114865000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8tbltjoy48","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720945000,"log":"[2026-02-17T13:56:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 288ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9nzp2a0id8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720946000,"log":"[2026-02-17T14:35:02Z INFO tonic::transport::server] grpc ingester connected client_id=j7657jsn","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-56jos45ydz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720947000,"log":"[2026-02-17T14:23:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 810 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-15nvqwdzpp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720948000,"log":"[2026-02-17T13:37:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/179737100415491300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-40jds7kkhe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720949000,"log":"[2026-02-17T14:27:55Z DEBUG sqlx::query] DELETE FROM alerts returned 691 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fonm2y3br1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720950000,"log":"[2026-02-17T13:29:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7720188665383700000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cmkrfzw10w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720951000,"log":"[2026-02-17T12:59:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o8uak9qai5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720952000,"log":"[2026-02-17T14:54:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g2ps4wt37b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720953000,"log":"[2026-02-17T12:42:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ah1pvsj1oa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720954000,"log":"[2026-02-17T12:46:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0wvapgwxlp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720955000,"log":"[2026-02-17T15:08:35Z DEBUG metrics::reporter] recorded metric active_connections value=7982","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-miu0yf51rf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720956000,"log":"[2026-02-17T14:52:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gsax0sfhoh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720957000,"log":"[2026-02-17T12:51:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4q5vj6h8ap","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720958000,"log":"[2026-02-17T13:14:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3450607081632571000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2bedb31gjh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720959000,"log":"[2026-02-17T14:22:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 626 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1ou5nyf9yy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720960000,"log":"[2026-02-17T13:06:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cktiv99zcq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720961000,"log":"[2026-02-17T13:18:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7339413876890514000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ada9wdsfr3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720962000,"log":"[2026-02-17T14:23:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 98 rows in 30ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-34hnokolak","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720963000,"log":"[2026-02-17T15:03:31Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mhvxhlg3rq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720964000,"log":"[2026-02-17T13:31:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5270564125846564000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s392d92cwb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720965000,"log":"[2026-02-17T14:15:50Z DEBUG sqlx::query] DELETE FROM alerts returned 405 rows in 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5x9tx533nc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720966000,"log":"[2026-02-17T14:16:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 614 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ertgd0waik","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720967000,"log":"[2026-02-17T13:33:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1242688141817236500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ue90g1zbcj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720968000,"log":"[2026-02-17T14:35:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=vmr21f6x","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w82olrtjrm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720969000,"log":"[2026-02-17T14:30:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 783 rows in 85ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zuty15zeg8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720970000,"log":"[2026-02-17T12:52:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c8kbbyor0y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720971000,"log":"[2026-02-17T13:41:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2255998337514253300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ao684zv2ut","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720972000,"log":"[2026-02-17T12:39:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ckchacrhph","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720973000,"log":"[2026-02-17T13:06:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g43cf365x6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720974000,"log":"[2026-02-17T13:52:05Z INFO actix_web::middleware::logger] GET /api/_search 201 41ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lnu9zmtqzf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720975000,"log":"[2026-02-17T14:49:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-locm4u4tt2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720976000,"log":"[2026-02-17T12:44:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wl6n8p9zqa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720977000,"log":"[2026-02-17T14:02:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 452ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-idx393y0w9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720978000,"log":"[2026-02-17T13:33:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9163399870492981000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7pflyjmseb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720979000,"log":"[2026-02-17T14:36:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=mmg99cp2","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dud7qxylf8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720980000,"log":"[2026-02-17T13:38:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/657550560089879000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j7zjj6tlrj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720981000,"log":"[2026-02-17T15:18:33Z TRACE cache::redis] cache set key=org:config:4114","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5sj85tjle5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720982000,"log":"[2026-02-17T12:40:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xkzqyy5ifq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720983000,"log":"[2026-02-17T13:58:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 109ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bddjhdwnaj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720984000,"log":"[2026-02-17T13:47:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 38ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-v8thvqmu04","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720985000,"log":"[2026-02-17T13:47:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 410ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rb0yy9knyk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720986000,"log":"[2026-02-17T12:51:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yadukambop","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720987000,"log":"[2026-02-17T13:11:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5098258215191762000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-otwu0uznhc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720988000,"log":"[2026-02-17T12:54:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x8temq3dw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720989000,"log":"[2026-02-17T13:23:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1188606226473079000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r7ohs1gct3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720990000,"log":"[2026-02-17T13:24:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9374294069359743000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s0t8zto3wv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720991000,"log":"[2026-02-17T14:50:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5ot4a1nmj7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720992000,"log":"[2026-02-17T12:57:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-e6bjbdmj89","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720993000,"log":"[2026-02-17T13:17:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/184274467959704160.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p8ekxgbp5w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720994000,"log":"[2026-02-17T12:37:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q6f4vlb9bm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720995000,"log":"[2026-02-17T13:49:41Z INFO actix_web::middleware::logger] GET /api/_bulk 201 212ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-9ank1518sv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720996000,"log":"[2026-02-17T14:30:33Z DEBUG sqlx::query] INSERT FROM users returned 868 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3sehl9vjx0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331720997000,"log":"[2026-02-17T13:42:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 401 463ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lh67qwtb1p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331720998000,"log":"[2026-02-17T12:39:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lsl5c01j8m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331720999000,"log":"[2026-02-17T13:31:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2907149167064054000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0fz8wsdp82","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721000000,"log":"[2026-02-17T12:51:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3283lwv76f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721001000,"log":"[2026-02-17T12:53:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w91blv1vu7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721002000,"log":"[2026-02-17T14:08:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 31 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rk805zbdlw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721003000,"log":"[2026-02-17T13:38:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/726718534879857700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2z3e5x3r7x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721004000,"log":"[2026-02-17T12:58:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s8y9jnhe2y","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721005000,"log":"[2026-02-17T14:12:48Z DEBUG sqlx::query] INSERT FROM users returned 351 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-66187aita8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721006000,"log":"[2026-02-17T12:40:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vfd0i06t7i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721007000,"log":"[2026-02-17T14:40:13Z INFO tonic::transport::server] grpc querier disconnected client_id=s0zl90rj","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yxmcfqvjn7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721008000,"log":"[2026-02-17T14:43:28Z INFO tonic::transport::server] grpc querier disconnected client_id=67g0xs9b","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7ql7ri0054","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721009000,"log":"[2026-02-17T14:20:50Z DEBUG sqlx::query] DELETE FROM alerts returned 15 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xzamki1pyc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721010000,"log":"[2026-02-17T13:16:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3067714228841906700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r56ip1klg4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721011000,"log":"[2026-02-17T14:06:21Z INFO actix_web::middleware::logger] GET /api/_bulk 204 3ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sh0kzjj5qs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721012000,"log":"[2026-02-17T14:41:39Z INFO tonic::transport::server] grpc router request_received client_id=sh46h7mt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rxseo2g5ve","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721013000,"log":"[2026-02-17T12:36:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lrngbk56kq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721014000,"log":"[2026-02-17T12:38:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4bw83j1skx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721015000,"log":"[2026-02-17T14:51:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ghgjynqtyy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721016000,"log":"[2026-02-17T13:15:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/650076552553553400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5gr8c50mgx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721017000,"log":"[2026-02-17T13:44:30Z INFO actix_web::middleware::logger] PUT /api/_search 403 488ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mdt61xkdkf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721018000,"log":"[2026-02-17T13:56:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 116ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k1a50kihc6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721019000,"log":"[2026-02-17T14:56:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p1927bxfso","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721020000,"log":"[2026-02-17T12:59:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6dh0hpvlz3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721021000,"log":"[2026-02-17T13:24:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9720121652276556000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rsip5b5sos","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721022000,"log":"[2026-02-17T14:48:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-igzscyddd9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721023000,"log":"[2026-02-17T14:25:15Z DEBUG sqlx::query] DELETE FROM alerts returned 170 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n0lvkc50oq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721024000,"log":"[2026-02-17T14:19:03Z DEBUG sqlx::query] INSERT FROM users returned 264 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l3imx2abzb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721025000,"log":"[2026-02-17T14:49:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qt3s2b0wzg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721026000,"log":"[2026-02-17T15:06:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=8793","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7u3fazejwc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721027000,"log":"[2026-02-17T13:06:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-hpc16ag7l1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721028000,"log":"[2026-02-17T12:38:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ljfrhtn18a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721029000,"log":"[2026-02-17T14:20:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 111 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p9xjtojarz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721030000,"log":"[2026-02-17T13:42:25Z INFO actix_web::middleware::logger] GET /api/_search 404 456ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lppnx820m0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721031000,"log":"[2026-02-17T14:27:48Z DEBUG sqlx::query] INSERT FROM users returned 132 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nsnnwg6zee","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721032000,"log":"[2026-02-17T12:49:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-y6fb67om1f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721033000,"log":"[2026-02-17T14:20:38Z DEBUG sqlx::query] INSERT FROM users returned 146 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n2pqx6ddfg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721034000,"log":"[2026-02-17T13:52:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 418ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-435ammgdvc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721035000,"log":"[2026-02-17T15:06:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1494","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tfz0d7hwl7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721036000,"log":"[2026-02-17T14:55:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-85opafghoq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721037000,"log":"[2026-02-17T14:44:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=v0gpmwrt","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-59rpfsyjc7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721038000,"log":"[2026-02-17T13:44:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 298ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-irps1layaf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721039000,"log":"[2026-02-17T12:43:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u3wwadck01","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721040000,"log":"[2026-02-17T13:23:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4390570435009322000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fxfjagkt2n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721041000,"log":"[2026-02-17T13:36:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5743292233171480000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nurelsx57a","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721042000,"log":"[2026-02-17T13:30:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2779438055257706000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d9kxih9cr4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721043000,"log":"[2026-02-17T12:35:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-69yuys5hhl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721044000,"log":"[2026-02-17T14:00:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 200 452ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r8vd9xuctk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721045000,"log":"[2026-02-17T14:18:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 386 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g42tj7t4kj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721046000,"log":"[2026-02-17T12:40:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nz2g1veoat","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721047000,"log":"[2026-02-17T12:55:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4l0c55fv35","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721048000,"log":"[2026-02-17T15:15:21Z TRACE cache::redis] cache set key=auth:token:5902","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k8atoyw3es","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721049000,"log":"[2026-02-17T14:19:07Z DEBUG sqlx::query] SELECT FROM streams returned 56 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dm01pqz6jh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721050000,"log":"[2026-02-17T14:39:49Z INFO tonic::transport::server] grpc router request_received client_id=c2j5n9kt","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p1sd8ofg2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721051000,"log":"[2026-02-17T14:30:30Z DEBUG sqlx::query] DELETE FROM alerts returned 64 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-cgnzfpugwk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721052000,"log":"[2026-02-17T13:48:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 102ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jlaos9ox0d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721053000,"log":"[2026-02-17T12:56:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ei95q5cknl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721054000,"log":"[2026-02-17T13:59:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 110ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yt0r95bhs7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721055000,"log":"[2026-02-17T13:55:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 413ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2mg482vqv2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721056000,"log":"[2026-02-17T12:51:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k3poq779cm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721057000,"log":"[2026-02-17T14:24:47Z DEBUG sqlx::query] SELECT FROM streams returned 69 rows in 38ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rpa9gub3fw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721058000,"log":"[2026-02-17T14:41:55Z INFO tonic::transport::server] grpc compactor response_sent client_id=tuvln1ua","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2w1l8546rs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721059000,"log":"[2026-02-17T13:59:10Z INFO actix_web::middleware::logger] PUT /api/_search 400 229ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7m29va8wgk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721060000,"log":"[2026-02-17T13:28:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/900430448047204700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wmz9xvt4aq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721061000,"log":"[2026-02-17T13:28:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1887007068311084000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-evj5kn9me8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721062000,"log":"[2026-02-17T14:54:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iriniw1a1e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721063000,"log":"[2026-02-17T15:19:13Z TRACE cache::redis] cache miss key=org:config:542","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kg0chil932","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721064000,"log":"[2026-02-17T13:29:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9934290050270079000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-757ik2d76o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721065000,"log":"[2026-02-17T13:40:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1133276924743584800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p6o7uxx3cr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721066000,"log":"[2026-02-17T13:37:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5158757933919826000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8wuum0gacr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721067000,"log":"[2026-02-17T13:30:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9460012348373221000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rz0nsvzt1a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721068000,"log":"[2026-02-17T14:09:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 87 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vbjxav22fy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721069000,"log":"[2026-02-17T15:17:51Z TRACE cache::redis] cache set key=auth:token:6203","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8r0x9fwlrl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721070000,"log":"[2026-02-17T13:47:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 483ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2wknovalp6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721071000,"log":"[2026-02-17T15:05:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=1649","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x33fsn1u18","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721072000,"log":"[2026-02-17T15:13:44Z TRACE cache::redis] cache get key=stream:schema:672","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e5s0945u99","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721073000,"log":"[2026-02-17T12:42:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ojmqbc4luz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721074000,"log":"[2026-02-17T13:55:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 195ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-387b2wtah4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721075000,"log":"[2026-02-17T13:15:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9377252037172810000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cwg8pvixwl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721076000,"log":"[2026-02-17T14:15:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 563 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-k8lknbx7ss","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721077000,"log":"[2026-02-17T15:17:33Z TRACE cache::redis] cache set key=org:config:3287","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-85w78g596y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721078000,"log":"[2026-02-17T14:34:17Z INFO tonic::transport::server] grpc ingester connected client_id=lry4jev9","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d1m7vx8l7i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721079000,"log":"[2026-02-17T15:12:47Z DEBUG metrics::reporter] recorded metric http_requests_total value=7589","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tcwk9qp6kv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721080000,"log":"[2026-02-17T15:14:48Z TRACE cache::redis] cache hit key=org:config:1197","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g47zpv91dx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721081000,"log":"[2026-02-17T15:20:10Z TRACE cache::redis] cache delete key=query:result:9264","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a8wdm1ruhx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721082000,"log":"[2026-02-17T15:06:10Z DEBUG metrics::reporter] recorded metric active_connections value=7820","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j9f1lrxfe","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721083000,"log":"[2026-02-17T14:57:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-68e7so4rjs","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721084000,"log":"[2026-02-17T14:38:44Z INFO tonic::transport::server] grpc router request_received client_id=go2qcezl","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eaaxbq42s3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721085000,"log":"[2026-02-17T15:21:18Z TRACE cache::redis] cache hit key=org:config:1178","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rhlljldetw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721086000,"log":"[2026-02-17T15:12:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=4823","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1sedsherlk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721087000,"log":"[2026-02-17T14:01:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 460ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j82e4542m5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721088000,"log":"[2026-02-17T13:29:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5174191139203028000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wr5v3b3s1r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721089000,"log":"[2026-02-17T13:57:14Z INFO actix_web::middleware::logger] PUT /api/v1/traces 401 205ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qvmzkvdzcp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721090000,"log":"[2026-02-17T12:42:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-epf8l58lba","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721091000,"log":"[2026-02-17T14:44:39Z INFO tonic::transport::server] grpc router request_received client_id=su5xgdv8","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nhxzojbhzy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721092000,"log":"[2026-02-17T15:10:30Z DEBUG metrics::reporter] recorded metric active_connections value=9117","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uq7wmphxbf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721093000,"log":"[2026-02-17T13:17:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4914218194196947000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dae4q95n1f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721094000,"log":"[2026-02-17T14:36:48Z INFO tonic::transport::server] grpc querier disconnected client_id=ognhxap7","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lcvpxbast7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721095000,"log":"[2026-02-17T13:49:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 1ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nqleidib0s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721096000,"log":"[2026-02-17T14:25:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 644 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ss0dfapfqk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721097000,"log":"[2026-02-17T12:43:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c2aad75062","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721098000,"log":"[2026-02-17T13:27:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1125247044507489200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pwavuripah","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721099000,"log":"[2026-02-17T15:20:41Z TRACE cache::redis] cache expire key=auth:token:8416","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wckgv3yhsd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721100000,"log":"[2026-02-17T13:27:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5597752340614657000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s4tyzf61h7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721101000,"log":"[2026-02-17T14:32:48Z INFO tonic::transport::server] grpc querier disconnected client_id=gf9q5m7t","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2kd3ctyqx5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721102000,"log":"[2026-02-17T14:54:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2x2h9sjtee","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721103000,"log":"[2026-02-17T14:56:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n7y4phu9v0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721104000,"log":"[2026-02-17T13:52:19Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 404 185ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iy4fy18l3i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721105000,"log":"[2026-02-17T14:55:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-17y3t31spp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721106000,"log":"[2026-02-17T14:22:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 225 rows in 73ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nwyl4ghmqi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721107000,"log":"[2026-02-17T12:47:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-94wyb4wlfx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721108000,"log":"[2026-02-17T13:00:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2dva021rmg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721109000,"log":"[2026-02-17T12:49:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w16kfktlh4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721110000,"log":"[2026-02-17T14:00:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 159ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2o4p60uomu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721111000,"log":"[2026-02-17T14:47:58Z INFO tonic::transport::server] grpc querier disconnected client_id=ek53lvf9","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ck6rw7ceo6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721112000,"log":"[2026-02-17T14:42:32Z INFO tonic::transport::server] grpc ingester connected client_id=2lubygqq","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4wm8ry1fu8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721113000,"log":"[2026-02-17T12:54:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ke909ds2yx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721114000,"log":"[2026-02-17T13:58:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 404 396ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gr59futq82","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721115000,"log":"[2026-02-17T14:34:03Z INFO tonic::transport::server] grpc querier disconnected client_id=v8ho7vz8","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sd7d27nmiu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721116000,"log":"[2026-02-17T13:52:27Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 403 362ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-82f22tm3ii","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721117000,"log":"[2026-02-17T13:12:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7676048313844730000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x6vb69l3c5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721118000,"log":"[2026-02-17T13:46:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 154ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7sr58cyqw1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721119000,"log":"[2026-02-17T15:18:22Z TRACE cache::redis] cache delete key=user:session:2104","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0s76j4zrtb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721120000,"log":"[2026-02-17T12:37:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cndk36g0h8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721121000,"log":"[2026-02-17T13:22:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7079684979509293000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-hfewtgaymo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721122000,"log":"[2026-02-17T13:13:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8986776588976886000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fff4eri3nw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721123000,"log":"[2026-02-17T14:43:13Z INFO tonic::transport::server] grpc querier disconnected client_id=xj8t8ten","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1adsdlby2e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721124000,"log":"[2026-02-17T14:11:18Z DEBUG sqlx::query] INSERT FROM users returned 55 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0w75wbtm5f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721125000,"log":"[2026-02-17T13:48:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 154ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-klaxij98wt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721126000,"log":"[2026-02-17T12:58:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5jsh65fyvu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721127000,"log":"[2026-02-17T13:10:09Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4156451538855656400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-761ycv4ta2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721128000,"log":"[2026-02-17T12:38:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9sk2ajj1i8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721129000,"log":"[2026-02-17T12:43:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a3e5hlrs2u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721130000,"log":"[2026-02-17T15:02:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vebqbc8lz5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721131000,"log":"[2026-02-17T15:00:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q5wlyyedm2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721132000,"log":"[2026-02-17T13:58:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 200 173ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m0z73dv0u9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721133000,"log":"[2026-02-17T13:07:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-abuwze8279","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721134000,"log":"[2026-02-17T13:21:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8587647444760504000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6i9vq821ur","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721135000,"log":"[2026-02-17T12:37:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bky9g2i4k2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721136000,"log":"[2026-02-17T13:17:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9153067469943362000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gia35ktrc3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721137000,"log":"[2026-02-17T14:51:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hb1ijm24ke","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721138000,"log":"[2026-02-17T13:15:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3943298247378359300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2f3ql4vkuc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721139000,"log":"[2026-02-17T15:01:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-14gtasl8jv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721140000,"log":"[2026-02-17T14:08:07Z DEBUG sqlx::query] SELECT FROM streams returned 8 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-i0ouosfhvd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721141000,"log":"[2026-02-17T15:11:10Z DEBUG metrics::reporter] recorded metric active_connections value=5240","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jx6gspp2dd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721142000,"log":"[2026-02-17T13:28:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4644824821572579000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ne2go1ak7l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721143000,"log":"[2026-02-17T13:53:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 404 128ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kbaoj1ie8z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721144000,"log":"[2026-02-17T13:38:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4024470727253793000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wrdx1w7tpj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721145000,"log":"[2026-02-17T15:14:00Z TRACE cache::redis] cache hit key=query:result:5742","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oj932q9cem","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721146000,"log":"[2026-02-17T13:46:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 139ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yyjr4h5uj4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721147000,"log":"[2026-02-17T13:15:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5978331568210526000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-miaao9t21h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721148000,"log":"[2026-02-17T12:45:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aofvj1jjkf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721149000,"log":"[2026-02-17T14:39:10Z INFO tonic::transport::server] grpc compactor response_sent client_id=oprh57uf","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-llbaza7zww","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721150000,"log":"[2026-02-17T13:57:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 65ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0xl86ze65p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721151000,"log":"[2026-02-17T12:37:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g2kkn9fjvn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721152000,"log":"[2026-02-17T12:40:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uluxsncx94","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721153000,"log":"[2026-02-17T15:18:41Z TRACE cache::redis] cache expire key=auth:token:9592","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ypcpu9c6ku","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721154000,"log":"[2026-02-17T12:36:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-thop379n2s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721155000,"log":"[2026-02-17T14:27:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 186 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4lcpyaty3l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721156000,"log":"[2026-02-17T14:32:14Z INFO tonic::transport::server] grpc router request_received client_id=jx8w9dxq","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-f1qc48tipe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721157000,"log":"[2026-02-17T13:35:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1037189606935498000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dw4u66a34g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721158000,"log":"[2026-02-17T15:07:42Z DEBUG metrics::reporter] recorded metric http_requests_total value=7677","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mjtot8aotb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721159000,"log":"[2026-02-17T14:00:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 201 48ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e94ivu7dat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721160000,"log":"[2026-02-17T14:26:53Z DEBUG sqlx::query] INSERT FROM users returned 18 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xjbm7c3sxp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721161000,"log":"[2026-02-17T12:48:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o1lz0igvzt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721162000,"log":"[2026-02-17T13:47:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 482ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qutc80cd7n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721163000,"log":"[2026-02-17T14:01:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 229ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-15xd0u9hjn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721164000,"log":"[2026-02-17T14:39:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=3d457jwu","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p7w84stg5d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721165000,"log":"[2026-02-17T12:40:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3dfd7n9e14","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721166000,"log":"[2026-02-17T13:26:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6296041553011063000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-01m0rdbuvo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721167000,"log":"[2026-02-17T13:57:05Z INFO actix_web::middleware::logger] GET /api/_search 401 264ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bef7qd924p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721168000,"log":"[2026-02-17T14:48:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-374vkiuwj8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721169000,"log":"[2026-02-17T13:00:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j4fjsc85wg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721170000,"log":"[2026-02-17T13:18:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/780072441965333800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c5nta95qu8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721171000,"log":"[2026-02-17T15:17:16Z TRACE cache::redis] cache delete key=auth:token:5912","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l2pl9qu2oj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721172000,"log":"[2026-02-17T14:15:45Z DEBUG sqlx::query] DELETE FROM alerts returned 606 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-buc2t8z9tw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721173000,"log":"[2026-02-17T13:12:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4586616474697533400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-j19y1cu4xz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721174000,"log":"[2026-02-17T14:33:44Z INFO tonic::transport::server] grpc router request_received client_id=rjxe2fd4","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1t1k9cqq1n","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721175000,"log":"[2026-02-17T13:14:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7015367406218680000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-9jzyujx9ma","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721176000,"log":"[2026-02-17T13:57:55Z INFO actix_web::middleware::logger] DELETE /api/_search 200 85ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d8nwz5xliq","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721177000,"log":"[2026-02-17T14:06:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 482ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dfwxk6sux7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721178000,"log":"[2026-02-17T13:30:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5850988048009345000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-glc41slzph","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721179000,"log":"[2026-02-17T14:13:37Z DEBUG sqlx::query] SELECT FROM streams returned 737 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g42trk3frd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721180000,"log":"[2026-02-17T14:14:03Z DEBUG sqlx::query] INSERT FROM users returned 14 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2ufc00puk7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721181000,"log":"[2026-02-17T13:15:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1331140848524301600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yiyt2hnnvh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721182000,"log":"[2026-02-17T14:51:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0paoff69c2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721183000,"log":"[2026-02-17T15:04:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1ee6d4k0mg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721184000,"log":"[2026-02-17T12:49:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mjrwzeuiwe","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721185000,"log":"[2026-02-17T15:00:02Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wnntxns477","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721186000,"log":"[2026-02-17T14:46:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=ahb9h8d6","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ino1b2f1ln","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721187000,"log":"[2026-02-17T15:10:52Z DEBUG metrics::reporter] recorded metric http_requests_total value=5598","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gejakb0zi3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721188000,"log":"[2026-02-17T13:23:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9238691537255879000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-zmr6scvx9p","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721189000,"log":"[2026-02-17T13:55:40Z INFO actix_web::middleware::logger] POST /api/_search 200 17ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qwgctinpwh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721190000,"log":"[2026-02-17T13:18:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6894929025755538000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tylk9jvirl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721191000,"log":"[2026-02-17T13:00:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qoq90c2bkb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721192000,"log":"[2026-02-17T13:36:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5424305616522125000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4ro2x2o3cv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721193000,"log":"[2026-02-17T12:55:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-oskfomxoi6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721194000,"log":"[2026-02-17T14:48:03Z INFO tonic::transport::server] grpc querier disconnected client_id=32ebkcbj","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jjp83zfry1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721195000,"log":"[2026-02-17T13:34:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7095754918417046000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4u90ved1mt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721196000,"log":"[2026-02-17T14:51:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2zjqi56jim","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721197000,"log":"[2026-02-17T13:23:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/673978805439869800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xjzpzp12dq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721198000,"log":"[2026-02-17T15:13:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=4796","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lq2dbltyf6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721199000,"log":"[2026-02-17T14:10:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 935 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bg25jpg9rf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721200000,"log":"[2026-02-17T14:53:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p2on7xp0j3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721201000,"log":"[2026-02-17T13:39:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9382710362785708000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vsnr05toz4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721202000,"log":"[2026-02-17T13:00:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jz6cii6y4s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721203000,"log":"[2026-02-17T13:03:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q29ftwf3lc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721204000,"log":"[2026-02-17T12:48:54Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1om8q31aa9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721205000,"log":"[2026-02-17T15:21:34Z TRACE cache::redis] cache delete key=stream:schema:1555","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y0zqcjlqxy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721206000,"log":"[2026-02-17T14:59:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m6vluapiot","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721207000,"log":"[2026-02-17T13:26:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/121351768362865100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vvycdv9dc2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721208000,"log":"[2026-02-17T13:18:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6067819179872418000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gymz2sbtz8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721209000,"log":"[2026-02-17T14:54:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f825nztm1j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721210000,"log":"[2026-02-17T14:22:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 947 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vhtg86hg02","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721211000,"log":"[2026-02-17T15:00:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gna47xev10","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721212000,"log":"[2026-02-17T13:04:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mbytt51ils","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721213000,"log":"[2026-02-17T14:46:46Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=izoea6f0","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-peayoib1zs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721214000,"log":"[2026-02-17T12:54:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zk9l2pa996","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721215000,"log":"[2026-02-17T13:15:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2914849808245925000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2wvjzn9tf1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721216000,"log":"[2026-02-17T13:45:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 400 456ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-dovq1f01wh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721217000,"log":"[2026-02-17T12:44:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eilwtt12wa","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721218000,"log":"[2026-02-17T13:50:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 419ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-upnqwe1s7t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721219000,"log":"[2026-02-17T13:46:01Z INFO actix_web::middleware::logger] GET /api/_bulk 404 432ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7n1rlblmgq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721220000,"log":"[2026-02-17T13:52:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 439ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-t1suhu58x9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721221000,"log":"[2026-02-17T14:08:15Z DEBUG sqlx::query] DELETE FROM alerts returned 704 rows in 68ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9jclyf9ses","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721222000,"log":"[2026-02-17T14:43:20Z INFO tonic::transport::server] grpc compactor response_sent client_id=7mk6naj3","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h9cbrx819n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721223000,"log":"[2026-02-17T14:42:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=8qns6n4r","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-h823q6f88e","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721224000,"log":"[2026-02-17T12:39:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wpjzkzt1ko","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721225000,"log":"[2026-02-17T14:38:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=qiiwuti8","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5fzr3yw23e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721226000,"log":"[2026-02-17T14:31:17Z DEBUG sqlx::query] SELECT FROM streams returned 760 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5ibt15n4ht","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721227000,"log":"[2026-02-17T14:52:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-joe39cw4po","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721228000,"log":"[2026-02-17T13:09:33Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4658206385987061000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5980vfy1pm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721229000,"log":"[2026-02-17T14:05:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 277ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2kdp85f134","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721230000,"log":"[2026-02-17T14:54:38Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e0ouuyuh2j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721231000,"log":"[2026-02-17T13:41:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2003600810811423500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-luyxs3wqjd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721232000,"log":"[2026-02-17T13:57:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 404 405ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4hli9aao58","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721233000,"log":"[2026-02-17T15:20:32Z TRACE cache::redis] cache get key=user:session:920","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-rjq41rz5m3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721234000,"log":"[2026-02-17T14:43:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=i3njzubt","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l8nz3znvb1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721235000,"log":"[2026-02-17T12:59:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-twuhkfliah","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721236000,"log":"[2026-02-17T14:08:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 76 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pusszgslbj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721237000,"log":"[2026-02-17T14:51:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-toj0u6skdt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721238000,"log":"[2026-02-17T13:44:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 168ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pou22dzfmu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721239000,"log":"[2026-02-17T14:55:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j2ucupoqvo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721240000,"log":"[2026-02-17T12:39:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-f516jqiqim","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721241000,"log":"[2026-02-17T13:18:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7707763100522602000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pbnlev6r4j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721242000,"log":"[2026-02-17T13:08:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-adw3if0jws","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721243000,"log":"[2026-02-17T12:57:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5407kcgimv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721244000,"log":"[2026-02-17T14:30:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 27 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jdd2qbictv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721245000,"log":"[2026-02-17T13:07:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-s2ntbe1p7y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721246000,"log":"[2026-02-17T14:32:01Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=mi5f8wpb","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-u9q6lu2p8m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721247000,"log":"[2026-02-17T14:44:02Z INFO tonic::transport::server] grpc ingester connected client_id=h2fgmf9b","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y50nrbewrk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721248000,"log":"[2026-02-17T13:15:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5150642879671541000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3qrssdndz7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721249000,"log":"[2026-02-17T12:44:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r7pjkxw6jq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721250000,"log":"[2026-02-17T14:55:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ts2c0z6hny","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721251000,"log":"[2026-02-17T12:35:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ivn0dlng3x","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721252000,"log":"[2026-02-17T13:30:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8639208985854982000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8qdplxnvy5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721253000,"log":"[2026-02-17T13:04:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eda3xd2p05","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721254000,"log":"[2026-02-17T13:30:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1534537350987952400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9hcjc3is31","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721255000,"log":"[2026-02-17T14:43:08Z INFO tonic::transport::server] grpc querier disconnected client_id=y8lz27xu","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sb7unfh94f","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721256000,"log":"[2026-02-17T13:19:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3621116067590844000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ckfkt65r4c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721257000,"log":"[2026-02-17T14:08:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 338 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rykp5g7gh3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721258000,"log":"[2026-02-17T14:24:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 415 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e9vucygn4d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721259000,"log":"[2026-02-17T14:31:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 744 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hw5a120pq4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721260000,"log":"[2026-02-17T12:41:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zifk08iccj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721261000,"log":"[2026-02-17T15:18:06Z TRACE cache::redis] cache hit key=auth:token:8618","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mq87sbtrq9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721262000,"log":"[2026-02-17T13:44:05Z INFO actix_web::middleware::logger] GET /api/_search 500 86ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gjptpgmtku","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721263000,"log":"[2026-02-17T15:06:51Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5661","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hx1859ljho","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721264000,"log":"[2026-02-17T13:16:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2326485637055673000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-83z6oa34gb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721265000,"log":"[2026-02-17T12:53:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y35dl3f3m4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721266000,"log":"[2026-02-17T14:35:15Z INFO tonic::transport::server] grpc compactor response_sent client_id=w6u4m7b6","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9cfjiqp4if","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721267000,"log":"[2026-02-17T14:33:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=58farmmu","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nkcuwwif2u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721268000,"log":"[2026-02-17T13:22:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6324174747444991000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-bkch1wst31","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721269000,"log":"[2026-02-17T13:43:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 374ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2zph7awe46","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721270000,"log":"[2026-02-17T13:35:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8598080181206890000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yzrp0qkly4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721271000,"log":"[2026-02-17T13:02:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h2c18ibpqv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721272000,"log":"[2026-02-17T13:54:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 401 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f2f2bxzwbb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721273000,"log":"[2026-02-17T13:38:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9028133061099946000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rz2jwp7rej","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721274000,"log":"[2026-02-17T13:39:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3149074876082489300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rdlbty7mqi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721275000,"log":"[2026-02-17T13:38:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8150689384581800000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-iqck6dvbel","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721276000,"log":"[2026-02-17T14:10:48Z DEBUG sqlx::query] INSERT FROM users returned 665 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e6nsgotsj0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721277000,"log":"[2026-02-17T14:29:07Z DEBUG sqlx::query] SELECT FROM streams returned 950 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4feclg71jd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721278000,"log":"[2026-02-17T13:48:15Z INFO actix_web::middleware::logger] DELETE /api/_search 403 58ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-buw2ahwt0a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721279000,"log":"[2026-02-17T14:55:19Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-iy5j7kusnb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721280000,"log":"[2026-02-17T14:22:43Z DEBUG sqlx::query] INSERT FROM users returned 161 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-82u3djecvk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721281000,"log":"[2026-02-17T15:02:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u7zd5iu5na","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721282000,"log":"[2026-02-17T14:47:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=8w2sx6hv","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hh6zlbr2qo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721283000,"log":"[2026-02-17T15:04:39Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pd89fh4otk","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721284000,"log":"[2026-02-17T13:44:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 200 356ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s454shx9vb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721285000,"log":"[2026-02-17T12:36:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h3xyz9uq5b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721286000,"log":"[2026-02-17T13:37:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4935120883449821000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8g1m7m9lk1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721287000,"log":"[2026-02-17T13:45:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 404 12ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ay6hkbcsqc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721288000,"log":"[2026-02-17T13:07:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ekpnrfeceh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721289000,"log":"[2026-02-17T13:41:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6915549948675906000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zlxnna81za","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721290000,"log":"[2026-02-17T14:03:40Z INFO actix_web::middleware::logger] POST /api/_search 400 372ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gysj31gnc1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721291000,"log":"[2026-02-17T14:23:27Z DEBUG sqlx::query] SELECT FROM streams returned 504 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5s9sxferun","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721292000,"log":"[2026-02-17T13:17:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3165451609203415600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dt66vg2kj3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721293000,"log":"[2026-02-17T14:12:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 776 rows in 72ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8n6txo0e0s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721294000,"log":"[2026-02-17T13:41:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4923231091501034000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8vnnk9sjhg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721295000,"log":"[2026-02-17T14:55:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5c5n89egin","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721296000,"log":"[2026-02-17T14:11:47Z DEBUG sqlx::query] SELECT FROM streams returned 784 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-g6tejbpdvk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721297000,"log":"[2026-02-17T14:58:45Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c9ucru466k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721298000,"log":"[2026-02-17T14:20:05Z DEBUG sqlx::query] DELETE FROM alerts returned 685 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qtprkqsqao","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721299000,"log":"[2026-02-17T15:14:34Z TRACE cache::redis] cache delete key=stream:schema:3877","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lkata9vgq1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721300000,"log":"[2026-02-17T15:21:06Z TRACE cache::redis] cache hit key=auth:token:3190","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ptzsvswihe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721301000,"log":"[2026-02-17T13:08:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qe26j6qb7k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721302000,"log":"[2026-02-17T14:13:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 850 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dkwaifp47j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721303000,"log":"[2026-02-17T14:32:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=yn6iv6y0","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xzrqyjn78i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721304000,"log":"[2026-02-17T15:18:30Z TRACE cache::redis] cache hit key=query:result:705","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5x4g271u9s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721305000,"log":"[2026-02-17T13:39:03Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7025600222965272000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-776g5ivu6i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721306000,"log":"[2026-02-17T12:42:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-o7tz9w9lil","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721307000,"log":"[2026-02-17T13:55:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 500 388ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7xtpfzy8q9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721308000,"log":"[2026-02-17T12:49:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-45atqu195g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721309000,"log":"[2026-02-17T14:55:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sub24huugk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721310000,"log":"[2026-02-17T14:01:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 400 55ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-06duvmf3gi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721311000,"log":"[2026-02-17T13:32:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3779552821167453000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ze6oj2qxhk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721312000,"log":"[2026-02-17T12:41:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s2fcip0b8d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721313000,"log":"[2026-02-17T14:26:58Z DEBUG sqlx::query] INSERT FROM users returned 433 rows in 67ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w9bk5vi19l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721314000,"log":"[2026-02-17T14:01:10Z INFO actix_web::middleware::logger] PUT /api/_search 404 186ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-st611dayxw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721315000,"log":"[2026-02-17T14:30:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 578 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l8asnqowd3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721316000,"log":"[2026-02-17T14:13:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 749 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ks3yv3mghq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721317000,"log":"[2026-02-17T13:40:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4137925911245710000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-z2d4ptruv7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721318000,"log":"[2026-02-17T12:54:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mrwytnmmko","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721319000,"log":"[2026-02-17T14:38:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=liblw1q8","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-waklcxzdu8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721320000,"log":"[2026-02-17T15:09:04Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8511","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qyhcyts2ah","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721321000,"log":"[2026-02-17T14:32:00Z INFO tonic::transport::server] grpc compactor response_sent client_id=fjjhdo5g","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x8mz74x8d6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721322000,"log":"[2026-02-17T13:37:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8535670955630037000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-h9mye2xbnh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721323000,"log":"[2026-02-17T14:56:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4picl4hswy","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721324000,"log":"[2026-02-17T12:53:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ogl9zszkpe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721325000,"log":"[2026-02-17T13:41:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6413837283851461000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tmqm80t3bo","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721326000,"log":"[2026-02-17T13:53:45Z INFO actix_web::middleware::logger] GET /api/_search 204 446ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-38vnkxjd0k","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721327000,"log":"[2026-02-17T13:19:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1274198018935608300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aekkcjtr3z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721328000,"log":"[2026-02-17T14:59:37Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-isygkwtact","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721329000,"log":"[2026-02-17T15:12:45Z DEBUG metrics::reporter] recorded metric active_connections value=6422","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0v7h5eflw2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721330000,"log":"[2026-02-17T14:29:05Z DEBUG sqlx::query] DELETE FROM alerts returned 629 rows in 43ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q8kal7efh5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721331000,"log":"[2026-02-17T13:27:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5964489383755955000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cj05frclvi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721332000,"log":"[2026-02-17T14:02:29Z INFO actix_web::middleware::logger] GET /api/v1/traces 401 163ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-55mq688zsr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721333000,"log":"[2026-02-17T15:09:03Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6829","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-j5udqql0g3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721334000,"log":"[2026-02-17T12:42:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-clkzodpa9h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721335000,"log":"[2026-02-17T14:40:08Z INFO tonic::transport::server] grpc querier disconnected client_id=vi4ckubh","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2gpj83zu46","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721336000,"log":"[2026-02-17T15:05:49Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=991","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9s3otdh651","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721337000,"log":"[2026-02-17T13:24:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/951108168597887200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d5jdn9qwtv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721338000,"log":"[2026-02-17T15:02:26Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gm6tf2nxj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721339000,"log":"[2026-02-17T13:56:58Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 404 373ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-50boa1z3e1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721340000,"log":"[2026-02-17T13:10:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3719057408109292000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xrjnnciwgu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721341000,"log":"[2026-02-17T14:47:59Z INFO tonic::transport::server] grpc router request_received client_id=rjicwz1i","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8cbgid96jb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721342000,"log":"[2026-02-17T14:20:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 486 rows in 60ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jk1y81kqld","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721343000,"log":"[2026-02-17T13:49:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 204 106ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mt4x8blqv7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721344000,"log":"[2026-02-17T13:22:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3298348761975618000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uw0mhqct9c","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721345000,"log":"[2026-02-17T14:33:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=5om6rj3e","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-q45iqbh31w","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721346000,"log":"[2026-02-17T13:14:11Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5927284905737311000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-pmzgjhzz15","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721347000,"log":"[2026-02-17T13:59:31Z INFO actix_web::middleware::logger] DELETE /api/_bulk 404 215ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ji9eg8dh3v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721348000,"log":"[2026-02-17T13:51:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 503 342ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-48cziwikdl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721349000,"log":"[2026-02-17T13:41:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/650138631709023000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i72u7k3pzd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721350000,"log":"[2026-02-17T12:59:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8b71bgui4r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721351000,"log":"[2026-02-17T15:12:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8583","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rmdqjahoqq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721352000,"log":"[2026-02-17T14:13:33Z DEBUG sqlx::query] INSERT FROM users returned 268 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ou505ftjmc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721353000,"log":"[2026-02-17T14:20:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 35 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-8olbhx4xqo","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721354000,"log":"[2026-02-17T14:57:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nrovt4dyvm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721355000,"log":"[2026-02-17T12:38:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p1s7kxnejx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721356000,"log":"[2026-02-17T14:23:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 266 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ispxykrw61","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721357000,"log":"[2026-02-17T13:03:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dflm37dqey","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721358000,"log":"[2026-02-17T14:38:19Z INFO tonic::transport::server] grpc router request_received client_id=cm1uh7ah","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-duoer6vr3c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721359000,"log":"[2026-02-17T14:58:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fx561kmp4r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721360000,"log":"[2026-02-17T13:25:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2773873471937114000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-y0ap7o8tel","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721361000,"log":"[2026-02-17T14:26:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 449 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i99c39wdig","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721362000,"log":"[2026-02-17T13:09:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9247285460493883000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-84bddh062y","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721363000,"log":"[2026-02-17T14:20:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 271 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9hzjvjzc0j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721364000,"log":"[2026-02-17T12:58:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rxi1cae5b6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721365000,"log":"[2026-02-17T14:16:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 43 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-cgid0eohs9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721366000,"log":"[2026-02-17T13:47:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 348ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-phugvure8i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721367000,"log":"[2026-02-17T14:34:47Z INFO tonic::transport::server] grpc ingester connected client_id=1uh4y6li","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-94vh84qt2u","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721368000,"log":"[2026-02-17T13:34:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1606329083430353400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bc7scj5udp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721369000,"log":"[2026-02-17T12:37:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5a23h52zzx","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721370000,"log":"[2026-02-17T13:23:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4246833869765183000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nhbehrbs66","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721371000,"log":"[2026-02-17T14:38:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=pcy3et4g","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-eyj7c3q2fp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721372000,"log":"[2026-02-17T15:15:47Z TRACE cache::redis] cache expire key=user:session:6048","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9f9438pd9e","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721373000,"log":"[2026-02-17T13:57:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 60ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-h0uf9mdd4u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721374000,"log":"[2026-02-17T12:50:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6w410tq2r9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721375000,"log":"[2026-02-17T14:16:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 52 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-147zvqe9rd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721376000,"log":"[2026-02-17T14:59:47Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a5s6oq86q8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721377000,"log":"[2026-02-17T14:43:14Z INFO tonic::transport::server] grpc router request_received client_id=rvtd4qql","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-80dl4f91dh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721378000,"log":"[2026-02-17T15:02:57Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6u891j6lf1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721379000,"log":"[2026-02-17T14:35:38Z INFO tonic::transport::server] grpc querier disconnected client_id=lx7rsarx","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5lxzhqymhz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721380000,"log":"[2026-02-17T14:39:09Z INFO tonic::transport::server] grpc router request_received client_id=kazqikcr","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zgoxj1e22a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721381000,"log":"[2026-02-17T13:50:20Z INFO actix_web::middleware::logger] POST /api/_search 401 293ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rycfnzmohp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721382000,"log":"[2026-02-17T14:39:38Z INFO tonic::transport::server] grpc querier disconnected client_id=hq3i9usg","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ridhlx4q1l","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721383000,"log":"[2026-02-17T14:45:23Z INFO tonic::transport::server] grpc querier disconnected client_id=sh37g489","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5ctnhep48b","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721384000,"log":"[2026-02-17T14:16:57Z DEBUG sqlx::query] SELECT FROM streams returned 904 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w37571bt6a","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721385000,"log":"[2026-02-17T14:44:18Z INFO tonic::transport::server] grpc querier disconnected client_id=twpugjh0","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-q9r71at53d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721386000,"log":"[2026-02-17T13:50:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 86ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-us73kxu5jv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721387000,"log":"[2026-02-17T14:16:44Z DEBUG sqlx::query] UPDATE FROM organizations returned 330 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-x95d0otdnx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721388000,"log":"[2026-02-17T12:42:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w6pplvp1lb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721389000,"log":"[2026-02-17T12:50:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-21d9vh1ss5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721390000,"log":"[2026-02-17T13:53:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 401 432ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l1u5n0hq09","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721391000,"log":"[2026-02-17T15:21:23Z TRACE cache::redis] cache expire key=org:config:6148","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nc77w4dksc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721392000,"log":"[2026-02-17T13:46:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 209ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-26bpb1zpm6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721393000,"log":"[2026-02-17T12:48:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8pg1x0gafl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721394000,"log":"[2026-02-17T13:09:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6262646467408910000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bhzazno32h","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721395000,"log":"[2026-02-17T13:35:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9769618525146548000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pyp3wldsmd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721396000,"log":"[2026-02-17T13:36:18Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5172204546816548000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-83dks7kqlk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721397000,"log":"[2026-02-17T13:34:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3540211796252486000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kl2fsfvdjt","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721398000,"log":"[2026-02-17T14:51:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6t5mxje3xc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721399000,"log":"[2026-02-17T14:26:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 72 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8ynombqq1i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721400000,"log":"[2026-02-17T13:47:20Z INFO actix_web::middleware::logger] POST /api/_search 401 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-w7rt6gdt34","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721401000,"log":"[2026-02-17T14:24:10Z DEBUG sqlx::query] DELETE FROM alerts returned 28 rows in 87ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-204fsj48us","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721402000,"log":"[2026-02-17T14:20:20Z DEBUG sqlx::query] DELETE FROM alerts returned 541 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n5wyndydqt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721403000,"log":"[2026-02-17T13:50:55Z INFO actix_web::middleware::logger] DELETE /api/_search 400 323ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nrpdr3ziwp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721404000,"log":"[2026-02-17T12:46:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w5talv76id","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721405000,"log":"[2026-02-17T14:35:33Z INFO tonic::transport::server] grpc querier disconnected client_id=pjtjeubx","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nilizsh8px","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721406000,"log":"[2026-02-17T14:53:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cyrp4okemh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721407000,"log":"[2026-02-17T13:03:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ritdgvedsp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721408000,"log":"[2026-02-17T14:15:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 479 rows in 37ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-egk3kc5ad1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721409000,"log":"[2026-02-17T14:26:34Z DEBUG sqlx::query] UPDATE FROM organizations returned 36 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0k3mgeyl6h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721410000,"log":"[2026-02-17T15:10:23Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=5086","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1bl6g7qkzm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721411000,"log":"[2026-02-17T13:44:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 363ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vvcwhrzu6z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721412000,"log":"[2026-02-17T14:02:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 403 15ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mne8wvsujn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721413000,"log":"[2026-02-17T14:15:20Z DEBUG sqlx::query] DELETE FROM alerts returned 439 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ih13irrvro","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721414000,"log":"[2026-02-17T15:12:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6105","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u8dbsu932e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721415000,"log":"[2026-02-17T13:02:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vldtqjtqtu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721416000,"log":"[2026-02-17T14:29:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 449 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ajheriqfj7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721417000,"log":"[2026-02-17T14:16:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 559 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-utx0cb0eyr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721418000,"log":"[2026-02-17T13:26:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/4443619843013302300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n6gf26rzuv","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721419000,"log":"[2026-02-17T14:55:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-sv732ocg2q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721420000,"log":"[2026-02-17T13:40:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/2656765810154851000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zaj0nisnrw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721421000,"log":"[2026-02-17T14:58:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fe0g23zpcd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721422000,"log":"[2026-02-17T14:24:25Z DEBUG sqlx::query] DELETE FROM alerts returned 79 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-40f48wz10m","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721423000,"log":"[2026-02-17T12:50:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i0x07xl5rg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721424000,"log":"[2026-02-17T14:15:25Z DEBUG sqlx::query] DELETE FROM alerts returned 126 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gihj8hrdia","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721425000,"log":"[2026-02-17T14:10:37Z DEBUG sqlx::query] SELECT FROM streams returned 511 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f5i9ihoah6","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721426000,"log":"[2026-02-17T13:57:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 108ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-21wvqg821a","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721427000,"log":"[2026-02-17T15:14:53Z TRACE cache::redis] cache expire key=org:config:2610","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n6zpu1pisg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721428000,"log":"[2026-02-17T15:08:43Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=2034","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-545y3nhugl","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721429000,"log":"[2026-02-17T15:16:13Z TRACE cache::redis] cache miss key=org:config:7112","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-inrqh9zjdv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721430000,"log":"[2026-02-17T13:05:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5thauwjn95","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721431000,"log":"[2026-02-17T14:56:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p63nsvs2qb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721432000,"log":"[2026-02-17T14:10:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 187 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w7k54uzjnh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721433000,"log":"[2026-02-17T14:42:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=43fq1bq0","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sm3c9zir7n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721434000,"log":"[2026-02-17T15:15:18Z TRACE cache::redis] cache hit key=org:config:7055","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-07b3jrcwze","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721435000,"log":"[2026-02-17T13:10:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1577351817580998700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dkuxizb6p9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721436000,"log":"[2026-02-17T14:09:24Z DEBUG sqlx::query] UPDATE FROM organizations returned 869 rows in 50ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vmprxzxeg5","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721437000,"log":"[2026-02-17T15:13:02Z DEBUG metrics::reporter] recorded metric http_requests_total value=581","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1lvv4fkqr1","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721438000,"log":"[2026-02-17T14:49:23Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-lni0b14lg4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721439000,"log":"[2026-02-17T14:35:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=2zyh7u94","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-p2pvzch6cz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721440000,"log":"[2026-02-17T13:42:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 400 104ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ynz6yexqg3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721441000,"log":"[2026-02-17T12:59:26Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5lotv44di0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721442000,"log":"[2026-02-17T14:54:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-alr9pwf1ve","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721443000,"log":"[2026-02-17T13:32:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/784201518145275500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nqw5h5kd5b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721444000,"log":"[2026-02-17T13:08:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1451635633143113500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-37r0050eoi","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721445000,"log":"[2026-02-17T13:00:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w5zmfbdm73","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721446000,"log":"[2026-02-17T12:40:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s95wj14u42","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721447000,"log":"[2026-02-17T13:26:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1826733502159891700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mw1wrmngc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721448000,"log":"[2026-02-17T14:22:57Z DEBUG sqlx::query] SELECT FROM streams returned 668 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ndt4xyqdjz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721449000,"log":"[2026-02-17T14:58:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jjypkw5bw1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721450000,"log":"[2026-02-17T14:25:18Z DEBUG sqlx::query] INSERT FROM users returned 587 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5fvtz2sict","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721451000,"log":"[2026-02-17T13:06:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bff03y2dg0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721452000,"log":"[2026-02-17T14:12:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 779 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-atn435ov4t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721453000,"log":"[2026-02-17T13:17:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5699754780603920000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ikjmfa55ah","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721454000,"log":"[2026-02-17T14:01:50Z INFO actix_web::middleware::logger] PUT /api/_search 201 48ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r4g930k3ty","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721455000,"log":"[2026-02-17T14:43:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ti4hnsde","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ixtovmfgpu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721456000,"log":"[2026-02-17T13:53:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 492ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7fy4ros1cb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721457000,"log":"[2026-02-17T13:06:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j780xisabx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721458000,"log":"[2026-02-17T15:03:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uw18dc32x2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721459000,"log":"[2026-02-17T15:03:35Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xka66aiiix","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721460000,"log":"[2026-02-17T13:22:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2352972018776225000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-7ylhf9fkd4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721461000,"log":"[2026-02-17T14:07:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 152 rows in 77ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2h0ep935hr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721462000,"log":"[2026-02-17T12:53:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ttk8re5qje","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721463000,"log":"[2026-02-17T13:23:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5178413348127660000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z4ks9zx4ai","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721464000,"log":"[2026-02-17T14:53:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5v89oca0gg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721465000,"log":"[2026-02-17T15:19:08Z TRACE cache::redis] cache get key=org:config:9806","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-czt491j0s9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721466000,"log":"[2026-02-17T13:52:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 7ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cf9uww6ydl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721467000,"log":"[2026-02-17T13:13:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5467992848202203000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hsldavlhiv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721468000,"log":"[2026-02-17T15:08:39Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=7258","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dj0wp2r7m3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721469000,"log":"[2026-02-17T14:19:20Z DEBUG sqlx::query] DELETE FROM alerts returned 269 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fy6ddsu0qi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721470000,"log":"[2026-02-17T15:14:02Z TRACE cache::redis] cache get key=user:session:1358","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d4n4szxm34","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721471000,"log":"[2026-02-17T12:39:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u4q76tfory","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721472000,"log":"[2026-02-17T12:39:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n3fvhmecp5","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721473000,"log":"[2026-02-17T13:32:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6629818792909192000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7bkhmaz2t7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721474000,"log":"[2026-02-17T14:32:49Z INFO tonic::transport::server] grpc router request_received client_id=vm025jvi","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ktwxnh7wb8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721475000,"log":"[2026-02-17T13:15:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9040072768921090000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hugg4vk2jx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721476000,"log":"[2026-02-17T15:01:44Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-055qnoxswm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721477000,"log":"[2026-02-17T13:11:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5684297165080163000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w2ieck2ct2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721478000,"log":"[2026-02-17T13:51:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zzcruiznx7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721479000,"log":"[2026-02-17T15:00:42Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5ua5blrar2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721480000,"log":"[2026-02-17T14:29:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 709 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9eb6v0uos6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721481000,"log":"[2026-02-17T14:23:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 728 rows in 1ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-1ajwzivz2j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721482000,"log":"[2026-02-17T12:59:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n6bfr2jwov","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721483000,"log":"[2026-02-17T14:14:05Z DEBUG sqlx::query] DELETE FROM alerts returned 586 rows in 29ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nn345ccfap","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721484000,"log":"[2026-02-17T15:19:26Z TRACE cache::redis] cache get key=auth:token:8161","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5u3xlwyvwe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721485000,"log":"[2026-02-17T13:48:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 348ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-em49ubqkwz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721486000,"log":"[2026-02-17T13:51:09Z INFO actix_web::middleware::logger] GET /api/v1/traces 503 197ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uoirg78c05","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721487000,"log":"[2026-02-17T13:47:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 372ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h2f09ty3r2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721488000,"log":"[2026-02-17T13:50:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 120ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fwfh3obopo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721489000,"log":"[2026-02-17T14:16:28Z DEBUG sqlx::query] INSERT FROM users returned 848 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e55lt4frmz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721490000,"log":"[2026-02-17T13:55:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 455ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zequvc9805","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721491000,"log":"[2026-02-17T12:39:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-etxwwr22lz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721492000,"log":"[2026-02-17T12:41:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-le3peiju23","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721493000,"log":"[2026-02-17T14:57:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fhb9li195h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721494000,"log":"[2026-02-17T12:56:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-bs4cnjy6ot","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721495000,"log":"[2026-02-17T15:03:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jjcef1cw8t","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721496000,"log":"[2026-02-17T15:18:46Z TRACE cache::redis] cache delete key=auth:token:1820","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8ys60f6e4f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721497000,"log":"[2026-02-17T15:18:32Z TRACE cache::redis] cache get key=user:session:8983","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yhpyjh99tj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721498000,"log":"[2026-02-17T14:19:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 445 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a4drzf4v8h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721499000,"log":"[2026-02-17T14:42:02Z INFO tonic::transport::server] grpc ingester connected client_id=cnsz4g9g","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9o5zfjzt8h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721500000,"log":"[2026-02-17T13:19:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3336518616792711700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u6n8qcsvxf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721501000,"log":"[2026-02-17T14:37:29Z INFO tonic::transport::server] grpc router request_received client_id=k2a6z9ae","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-i68r6hb6l9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721502000,"log":"[2026-02-17T14:02:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 135ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3ny2ql4u7n","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721503000,"log":"[2026-02-17T13:56:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 401 355ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gttmdwmuhx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721504000,"log":"[2026-02-17T12:40:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c6qanyl6mk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721505000,"log":"[2026-02-17T13:09:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6294642429427373000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-moer2o8c3r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721506000,"log":"[2026-02-17T13:23:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5621123741160006000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8pvxbgofiu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721507000,"log":"[2026-02-17T15:13:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4196","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-59fl7suu4n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721508000,"log":"[2026-02-17T13:42:35Z INFO actix_web::middleware::logger] DELETE /api/_search 500 260ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nalplghhb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721509000,"log":"[2026-02-17T12:51:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-bg0gq4v5qi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721510000,"log":"[2026-02-17T15:05:36Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=9368","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n5s1583ieo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721511000,"log":"[2026-02-17T15:08:40Z DEBUG metrics::reporter] recorded metric active_connections value=5006","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g9zf2bovez","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721512000,"log":"[2026-02-17T14:47:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=vd2fza5v","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-gv6mbjmxye","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721513000,"log":"[2026-02-17T13:10:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7827415236786401000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xpal1xd7hw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721514000,"log":"[2026-02-17T15:16:32Z TRACE cache::redis] cache get key=user:session:8249","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z1ller9dfn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721515000,"log":"[2026-02-17T13:32:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3226107346408513000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vfdlc43qhg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721516000,"log":"[2026-02-17T14:24:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 828 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3twijajrc8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721517000,"log":"[2026-02-17T13:43:00Z INFO actix_web::middleware::logger] POST /api/_search 403 446ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ngi101n5h6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721518000,"log":"[2026-02-17T14:02:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 66ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i0rrewnelp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721519000,"log":"[2026-02-17T13:01:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-m470iugxmq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721520000,"log":"[2026-02-17T15:08:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=4494","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k31ct2fbfn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721521000,"log":"[2026-02-17T13:04:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-41ed836un6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721522000,"log":"[2026-02-17T14:58:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ufyilia0ve","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721523000,"log":"[2026-02-17T12:39:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0ewou6r9qq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721524000,"log":"[2026-02-17T13:17:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6952370820953466000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jveliqczgw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721525000,"log":"[2026-02-17T12:54:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q314v04m9u","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721526000,"log":"[2026-02-17T14:51:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-303goffrsp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721527000,"log":"[2026-02-17T13:31:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5412318662714544000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4t15bll0lw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721528000,"log":"[2026-02-17T12:46:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j8g6w2pd1v","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721529000,"log":"[2026-02-17T14:42:42Z INFO tonic::transport::server] grpc ingester connected client_id=s9xirrz7","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c15qfpn8sj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721530000,"log":"[2026-02-17T13:11:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4757938975451101000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-t9fc1l8crb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721531000,"log":"[2026-02-17T13:29:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1369073699307754500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2ab1oadjlk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721532000,"log":"[2026-02-17T12:48:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xrwgc1x3vv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721533000,"log":"[2026-02-17T14:08:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 3 rows in 69ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-os7mz0fudw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721534000,"log":"[2026-02-17T13:28:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7812222360779899000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tr1v1k0m9z","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721535000,"log":"[2026-02-17T13:57:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 314ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2pg7l5749m","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721536000,"log":"[2026-02-17T14:57:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dtesc48u7q","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721537000,"log":"[2026-02-17T13:52:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 7ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mqvs3n1455","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721538000,"log":"[2026-02-17T14:18:48Z DEBUG sqlx::query] INSERT FROM users returned 43 rows in 91ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8o5ygpge","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721539000,"log":"[2026-02-17T13:06:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-yyd56wj10t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721540000,"log":"[2026-02-17T15:11:35Z DEBUG metrics::reporter] recorded metric active_connections value=4219","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-p7krwuel61","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721541000,"log":"[2026-02-17T13:08:32Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2698988086636840000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5mm3ca95nj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721542000,"log":"[2026-02-17T13:17:08Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9979112537126154000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xhvt3pz4pc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721543000,"log":"[2026-02-17T14:11:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 949 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-16yfzdsqg4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721544000,"log":"[2026-02-17T13:12:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/3362934597318834000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-h4ybph8xe4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721545000,"log":"[2026-02-17T14:08:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 70 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gpi78fwkj2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721546000,"log":"[2026-02-17T14:22:19Z DEBUG sqlx::query] UPDATE FROM organizations returned 781 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-snyvsg1zep","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721547000,"log":"[2026-02-17T12:42:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ozha7kuw8d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721548000,"log":"[2026-02-17T14:07:12Z DEBUG sqlx::query] SELECT FROM streams returned 220 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-78h8syrxnx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721549000,"log":"[2026-02-17T13:18:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5676190226195522000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ntnzlp7mpf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721550000,"log":"[2026-02-17T13:53:40Z INFO actix_web::middleware::logger] POST /api/_search 404 484ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wornky8sdx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721551000,"log":"[2026-02-17T12:55:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-l5ryb5bgfn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721552000,"log":"[2026-02-17T13:00:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7cfy8qu3bk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721553000,"log":"[2026-02-17T14:58:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pk59ivfphp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721554000,"log":"[2026-02-17T14:10:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 779 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jaqx3eg5tz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721555000,"log":"[2026-02-17T14:38:33Z INFO tonic::transport::server] grpc querier disconnected client_id=qvex7ih9","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pijnx63xoe","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721556000,"log":"[2026-02-17T12:39:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ng6qy5ulnp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721557000,"log":"[2026-02-17T12:39:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ivv0387kai","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721558000,"log":"[2026-02-17T12:43:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-45gq009vca","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721559000,"log":"[2026-02-17T14:49:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ngh85t4n40","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721560000,"log":"[2026-02-17T13:37:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1291031260667623400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-u88r7r513u","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721561000,"log":"[2026-02-17T15:13:09Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=5285","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ylo6h54x08","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721562000,"log":"[2026-02-17T14:56:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-32rjpxehe3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721563000,"log":"[2026-02-17T12:44:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z7j97xblf8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721564000,"log":"[2026-02-17T14:28:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 131 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w10r8sgyy1","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721565000,"log":"[2026-02-17T13:23:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1597799597932008200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-43221eb3bz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721566000,"log":"[2026-02-17T13:18:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3135287113897433600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-53kjl467mv","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721567000,"log":"[2026-02-17T14:51:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ooe8wpmwg2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721568000,"log":"[2026-02-17T14:29:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 444 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hlryav8y82","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721569000,"log":"[2026-02-17T14:05:45Z INFO actix_web::middleware::logger] GET /api/_search 204 139ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vr1e5s9oek","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721570000,"log":"[2026-02-17T13:46:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 349ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-x1j9bzqz8d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721571000,"log":"[2026-02-17T13:17:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/8868329760113702000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2o0m4lx5jo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721572000,"log":"[2026-02-17T13:52:46Z INFO actix_web::middleware::logger] PUT /api/_bulk 404 481ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2d4ldnnmv7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721573000,"log":"[2026-02-17T14:14:37Z DEBUG sqlx::query] SELECT FROM streams returned 412 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n00e1tjmqj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721574000,"log":"[2026-02-17T14:39:19Z INFO tonic::transport::server] grpc router request_received client_id=wiem0b62","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ljyjy73v2h","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721575000,"log":"[2026-02-17T13:19:10Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6987452211720957000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-udc3t42vo9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721576000,"log":"[2026-02-17T13:44:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 503 482ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0g3arb2g79","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721577000,"log":"[2026-02-17T13:44:23Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 500 42ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-666654u8f1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721578000,"log":"[2026-02-17T13:29:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2274703345723436800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-o03emrqp02","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721579000,"log":"[2026-02-17T15:14:08Z TRACE cache::redis] cache get key=org:config:8786","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3u36803eic","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721580000,"log":"[2026-02-17T14:57:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xbwxgac39o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721581000,"log":"[2026-02-17T12:56:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vslk4vhcmi","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721582000,"log":"[2026-02-17T14:50:17Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3xvfsmq3cp","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721583000,"log":"[2026-02-17T13:42:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 309ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z1gw387olo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721584000,"log":"[2026-02-17T13:01:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eqx0tcymbf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721585000,"log":"[2026-02-17T14:08:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 296 rows in 7ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-98r1lgsmv3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721586000,"log":"[2026-02-17T15:09:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=109","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rjl9et1ehp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721587000,"log":"[2026-02-17T15:18:38Z TRACE cache::redis] cache get key=org:config:4018","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-kogmlgzp6s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721588000,"log":"[2026-02-17T15:11:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=1350","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wibi65qepz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721589000,"log":"[2026-02-17T14:32:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=pd6x0kdx","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5axjctf9jk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721590000,"log":"[2026-02-17T15:00:54Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-plw9d0185x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721591000,"log":"[2026-02-17T12:40:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3j1rniarbc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721592000,"log":"[2026-02-17T14:58:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-z577aee7wg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721593000,"log":"[2026-02-17T13:29:39Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2225456142669528600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-hmhkdelgsx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721594000,"log":"[2026-02-17T13:50:16Z INFO actix_web::middleware::logger] POST /api/_bulk 200 27ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gslqhpwxin","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721595000,"log":"[2026-02-17T14:37:43Z INFO tonic::transport::server] grpc querier disconnected client_id=wo5ygnq2","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a6caqx4qus","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721596000,"log":"[2026-02-17T13:29:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3846812626364349000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-45l8y41n09","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721597000,"log":"[2026-02-17T13:51:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 455ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gs3qne6t86","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721598000,"log":"[2026-02-17T13:39:05Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4094014539941302000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-c5bjznup6o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721599000,"log":"[2026-02-17T13:22:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6055364931821774000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lj3mtvpx8c","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721600000,"log":"[2026-02-17T14:27:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 825 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-azemw0tkrz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721601000,"log":"[2026-02-17T13:02:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kvqbsnm6xy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721602000,"log":"[2026-02-17T12:35:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gq4c3wnwhj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721603000,"log":"[2026-02-17T14:38:38Z INFO tonic::transport::server] grpc querier disconnected client_id=kcnv58ww","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gyxulwov5p","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721604000,"log":"[2026-02-17T14:55:29Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-kphjhajwwd","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721605000,"log":"[2026-02-17T12:55:52Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2u1ncar0sj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721606000,"log":"[2026-02-17T14:02:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 400ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-d5p7xjxqlj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721607000,"log":"[2026-02-17T12:37:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w6ej255lc4","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721608000,"log":"[2026-02-17T13:04:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2u5ds7te4x","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721609000,"log":"[2026-02-17T12:38:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i03g1jqog3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721610000,"log":"[2026-02-17T12:36:03Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k2u5aj7j6i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721611000,"log":"[2026-02-17T14:04:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 500 132ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o5l29e0ha8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721612000,"log":"[2026-02-17T14:02:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 400 369ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bfyac6e2cd","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721613000,"log":"[2026-02-17T13:54:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 403 478ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5arpsrmyw0","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721614000,"log":"[2026-02-17T15:01:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qi5ngu93cu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721615000,"log":"[2026-02-17T14:47:17Z INFO tonic::transport::server] grpc ingester connected client_id=d6hgp948","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8ejbuthum8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721616000,"log":"[2026-02-17T14:35:58Z INFO tonic::transport::server] grpc querier disconnected client_id=bixkmi2h","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5weooo98av","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721617000,"log":"[2026-02-17T14:06:48Z INFO actix_web::middleware::logger] POST /api/v1/metrics 204 37ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-005ybkd32g","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721618000,"log":"[2026-02-17T14:03:20Z INFO actix_web::middleware::logger] POST /api/_search 201 53ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-23h78tsp75","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721619000,"log":"[2026-02-17T13:57:00Z INFO actix_web::middleware::logger] POST /api/_search 503 193ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8hl78eh2cl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721620000,"log":"[2026-02-17T13:26:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7640844652023840000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xokeh89e4t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721621000,"log":"[2026-02-17T13:15:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9024068829543404000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r0m56p4mdp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721622000,"log":"[2026-02-17T13:55:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 195ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k0qsxn2ooq","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721623000,"log":"[2026-02-17T13:39:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6348549287891247000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ishb8w5sr7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721624000,"log":"[2026-02-17T13:04:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-focjmmg5p9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721625000,"log":"[2026-02-17T14:33:59Z INFO tonic::transport::server] grpc router request_received client_id=8ay99zr3","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-82nk9jggwa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721626000,"log":"[2026-02-17T12:52:27Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m7dzrmimyk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721627000,"log":"[2026-02-17T13:06:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aih5ef8215","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721628000,"log":"[2026-02-17T14:03:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 500 266ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d77ff6k4r5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721629000,"log":"[2026-02-17T12:43:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n5h0i8at26","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721630000,"log":"[2026-02-17T15:11:17Z DEBUG metrics::reporter] recorded metric http_requests_total value=7229","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5qixc6rqpd","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721631000,"log":"[2026-02-17T13:13:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/251066874546139740.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gv01g7gxfn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721632000,"log":"[2026-02-17T12:37:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fpz9xnaq0t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721633000,"log":"[2026-02-17T13:10:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4296113818764309000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xcl2m454ej","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721634000,"log":"[2026-02-17T14:13:27Z DEBUG sqlx::query] SELECT FROM streams returned 552 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-brhq72oiin","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721635000,"log":"[2026-02-17T13:46:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 402ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fyi6nf29k6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721636000,"log":"[2026-02-17T14:46:37Z INFO tonic::transport::server] grpc ingester connected client_id=8ja5jox8","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jdqe08zq17","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721637000,"log":"[2026-02-17T14:17:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 866 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rnwn72ewvf","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721638000,"log":"[2026-02-17T13:41:01Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6480574856321357000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5meb6r71od","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721639000,"log":"[2026-02-17T14:10:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 936 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-79fgkld5aj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721640000,"log":"[2026-02-17T12:47:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5nd8j6zref","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721641000,"log":"[2026-02-17T12:41:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0nj3mwotny","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721642000,"log":"[2026-02-17T14:15:48Z DEBUG sqlx::query] INSERT FROM users returned 4 rows in 86ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-w1vt3wd140","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721643000,"log":"[2026-02-17T14:46:12Z INFO tonic::transport::server] grpc ingester connected client_id=wlaa46h5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-w0751854y3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721644000,"log":"[2026-02-17T13:43:51Z INFO actix_web::middleware::logger] DELETE /api/_bulk 204 153ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4508wexytd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721645000,"log":"[2026-02-17T13:21:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/8405413607131442000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rilbcrnu06","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721646000,"log":"[2026-02-17T14:49:58Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d9osn9ik21","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721647000,"log":"[2026-02-17T13:03:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-71ooszsywo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721648000,"log":"[2026-02-17T13:02:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ouc5ax4lol","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721649000,"log":"[2026-02-17T13:10:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3052933991393336300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-aiwmgaj54v","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721650000,"log":"[2026-02-17T15:09:33Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=1157","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nbosjytooo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721651000,"log":"[2026-02-17T13:36:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5363675687125642000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r99erab6zg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721652000,"log":"[2026-02-17T12:42:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-amb8ubv0d1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721653000,"log":"[2026-02-17T13:05:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qz9e0aw1o","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721654000,"log":"[2026-02-17T13:40:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/1471139030153110300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tczfv550jx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721655000,"log":"[2026-02-17T14:23:07Z DEBUG sqlx::query] SELECT FROM streams returned 295 rows in 31ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yq1pmbevzb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721656000,"log":"[2026-02-17T14:06:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 204 97ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mdv21s7cq5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721657000,"log":"[2026-02-17T13:36:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3920554137387610600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-19u1kg56v7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721658000,"log":"[2026-02-17T15:01:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-k0boccscxt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721659000,"log":"[2026-02-17T14:16:17Z DEBUG sqlx::query] SELECT FROM streams returned 887 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0blo72lydy","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721660000,"log":"[2026-02-17T14:27:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 768 rows in 83ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qfhrym0ujg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721661000,"log":"[2026-02-17T12:39:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tcnw6tx3kn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721662000,"log":"[2026-02-17T12:55:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vjugmnq43w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721663000,"log":"[2026-02-17T14:43:22Z INFO tonic::transport::server] grpc ingester connected client_id=horf7f03","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-c4yig7ik0i","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721664000,"log":"[2026-02-17T14:09:27Z DEBUG sqlx::query] SELECT FROM streams returned 130 rows in 54ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ka5f4k96ts","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721665000,"log":"[2026-02-17T14:50:18Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uhgwqyng05","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721666000,"log":"[2026-02-17T13:21:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9404819085357994000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tfaf4yznre","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721667000,"log":"[2026-02-17T12:53:32Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-24ku923rtm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721668000,"log":"[2026-02-17T14:40:22Z INFO tonic::transport::server] grpc ingester connected client_id=rzsrtkao","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-es5qeq95n5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721669000,"log":"[2026-02-17T13:23:19Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8902921004393893000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-y26if0levl","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721670000,"log":"[2026-02-17T15:07:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=2457","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ryyicvpdrl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721671000,"log":"[2026-02-17T14:33:53Z INFO tonic::transport::server] grpc querier disconnected client_id=4pozo3za","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-84ipexmnj7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721672000,"log":"[2026-02-17T14:52:09Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j0bln7nk0t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721673000,"log":"[2026-02-17T13:36:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8841521213899053000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-had78termb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721674000,"log":"[2026-02-17T12:40:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-c351nc18jm","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721675000,"log":"[2026-02-17T14:59:08Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nl72xdki51","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721676000,"log":"[2026-02-17T14:07:40Z DEBUG sqlx::query] DELETE FROM alerts returned 860 rows in 56ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ly2y1jfwb8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721677000,"log":"[2026-02-17T13:12:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2365712102075716600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tnxc4ccc5r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721678000,"log":"[2026-02-17T13:54:05Z INFO actix_web::middleware::logger] GET /api/_search 401 333ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-i7l58w30bi","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721679000,"log":"[2026-02-17T14:27:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 534 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-j4tkt5ndy4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721680000,"log":"[2026-02-17T14:07:47Z DEBUG sqlx::query] SELECT FROM streams returned 113 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kaqg4n6t8v","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721681000,"log":"[2026-02-17T14:21:17Z DEBUG sqlx::query] SELECT FROM streams returned 282 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-a7f1j6c1o8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721682000,"log":"[2026-02-17T12:52:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-nzp930nr8l","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721683000,"log":"[2026-02-17T13:48:05Z INFO actix_web::middleware::logger] GET /api/_search 401 257ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8fu3g0zgwz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721684000,"log":"[2026-02-17T12:35:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-t3t47gtmj1","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721685000,"log":"[2026-02-17T12:46:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ik9hdn5f77","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721686000,"log":"[2026-02-17T14:59:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-uhmyad7az7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721687000,"log":"[2026-02-17T14:43:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=bcues7cy","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7c0jzohdbc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721688000,"log":"[2026-02-17T14:21:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 493 rows in 10ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zfu6g6jpgp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721689000,"log":"[2026-02-17T13:45:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 204 16ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fpp6ego5sq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721690000,"log":"[2026-02-17T13:06:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8hgghdk6tt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721691000,"log":"[2026-02-17T13:11:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6888543244916677000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tqn8kxwj62","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721692000,"log":"[2026-02-17T14:17:03Z DEBUG sqlx::query] INSERT FROM users returned 833 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iwj94dw2lz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721693000,"log":"[2026-02-17T15:16:34Z TRACE cache::redis] cache delete key=stream:schema:8326","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9kozgcvgtn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721694000,"log":"[2026-02-17T14:32:04Z INFO tonic::transport::server] grpc router request_received client_id=v55gsj5k","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-zwf4qr2lyq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721695000,"log":"[2026-02-17T12:38:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m8lzdno6bm","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721696000,"log":"[2026-02-17T13:00:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vscfqrj5xx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721697000,"log":"[2026-02-17T13:20:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/244953322978533900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rgitrofgph","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721698000,"log":"[2026-02-17T13:05:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jo5iwaxcq6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721699000,"log":"[2026-02-17T14:50:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ye5414quu7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721700000,"log":"[2026-02-17T14:37:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=04is9tas","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ttjhmid4ua","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721701000,"log":"[2026-02-17T14:14:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 63 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-afetl8yz3z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721702000,"log":"[2026-02-17T13:11:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3522346505135778300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1699y2oioz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721703000,"log":"[2026-02-17T15:00:53Z ERROR zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-n4wr6ny7kf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721704000,"log":"[2026-02-17T14:08:13Z DEBUG sqlx::query] INSERT FROM users returned 219 rows in 49ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-i1ha4fawv3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721705000,"log":"[2026-02-17T12:41:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-atvhfe8gs4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721706000,"log":"[2026-02-17T14:49:49Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rcygy7gqon","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721707000,"log":"[2026-02-17T14:12:06Z DEBUG sqlx::query] MERGE FROM dashboards returned 539 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6ingpfy9ao","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721708000,"log":"[2026-02-17T14:40:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=ff8ccojt","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ndc37sm1v2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721709000,"log":"[2026-02-17T13:24:12Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/5453751589016373000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-841xuddf3q","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721710000,"log":"[2026-02-17T13:55:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 167ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zbk85i61fn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721711000,"log":"[2026-02-17T14:19:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 317 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cxklb7xcat","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721712000,"log":"[2026-02-17T12:53:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-n0pod40g70","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721713000,"log":"[2026-02-17T12:44:28Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ouoreohn4i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721714000,"log":"[2026-02-17T12:56:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xik1espchd","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721715000,"log":"[2026-02-17T14:13:55Z DEBUG sqlx::query] DELETE FROM alerts returned 582 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g8canrqsxg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721716000,"log":"[2026-02-17T14:20:09Z DEBUG sqlx::query] UPDATE FROM organizations returned 588 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6dw9dik6kg","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721717000,"log":"[2026-02-17T14:56:14Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nu926d3p1l","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721718000,"log":"[2026-02-17T13:04:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-uz10zp7tjb","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721719000,"log":"[2026-02-17T14:43:57Z INFO tonic::transport::server] grpc ingester connected client_id=wbxcmdg5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-u5vqg789uh","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721720000,"log":"[2026-02-17T14:46:29Z INFO tonic::transport::server] grpc router request_received client_id=tzenug0n","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n6xbyfjwdm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721721000,"log":"[2026-02-17T14:18:59Z DEBUG sqlx::query] UPDATE FROM organizations returned 930 rows in 81ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ngwtxpue8a","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721722000,"log":"[2026-02-17T13:38:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/6489270664525663000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-982rxaa4sz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721723000,"log":"[2026-02-17T12:48:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kaimiccncx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721724000,"log":"[2026-02-17T14:08:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 119 rows in 70ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wdqjg4v3wp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721725000,"log":"[2026-02-17T13:19:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3520015693023259600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8yvolc9533","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721726000,"log":"[2026-02-17T13:06:45Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4kfqzak4ju","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721727000,"log":"[2026-02-17T12:38:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-rqgadkz4qg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721728000,"log":"[2026-02-17T13:16:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5405511938114835000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6o3eyy0q6s","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721729000,"log":"[2026-02-17T14:09:03Z DEBUG sqlx::query] INSERT FROM users returned 770 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7rah0hox71","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721730000,"log":"[2026-02-17T13:45:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 500 341ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-59ncowf9wa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721731000,"log":"[2026-02-17T12:52:23Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e1u8wzabur","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721732000,"log":"[2026-02-17T13:37:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/5131423763855303000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-pa6a6zkrp","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721733000,"log":"[2026-02-17T14:19:37Z DEBUG sqlx::query] SELECT FROM streams returned 977 rows in 59ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pujd5xuw6r","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721734000,"log":"[2026-02-17T14:50:25Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tb8p9esank","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721735000,"log":"[2026-02-17T13:09:27Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3108310067272613000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-689y2gc4bu","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721736000,"log":"[2026-02-17T13:13:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/684999054481374500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hg0f7i0hyb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721737000,"log":"[2026-02-17T14:04:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 503 241ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-h8cjwf9mjb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721738000,"log":"[2026-02-17T13:10:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6211235336669994000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yajjz5h7w8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721739000,"log":"[2026-02-17T13:35:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1546601211715608000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5skxi2zjx0","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721740000,"log":"[2026-02-17T12:51:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vuo2yvgjw2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721741000,"log":"[2026-02-17T14:44:07Z INFO tonic::transport::server] grpc ingester connected client_id=c8ge0pb5","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-far0il8sdb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721742000,"log":"[2026-02-17T14:18:51Z DEBUG sqlx::query] MERGE FROM dashboards returned 301 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-rquxp4a5mi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721743000,"log":"[2026-02-17T13:20:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4141614421758062000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ri7u0jncwe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721744000,"log":"[2026-02-17T15:16:28Z TRACE cache::redis] cache delete key=org:config:9401","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-o12vqq8q2j","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721745000,"log":"[2026-02-17T12:40:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gcs1y9ptki","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721746000,"log":"[2026-02-17T13:47:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 470ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xh1fokpdmz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721747000,"log":"[2026-02-17T13:22:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1004187004109509100.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7bjhuc4cwf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721748000,"log":"[2026-02-17T13:42:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 404 262ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-t5n437oy7x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721749000,"log":"[2026-02-17T12:45:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-u696yb8gpu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721750000,"log":"[2026-02-17T13:24:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7634640035350093000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ilnqlivuja","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721751000,"log":"[2026-02-17T12:36:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x0gi67un9f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721752000,"log":"[2026-02-17T15:06:15Z DEBUG metrics::reporter] recorded metric active_connections value=8916","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yle1zmkmjz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721753000,"log":"[2026-02-17T13:34:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9175009206493767000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mp3f972iru","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721754000,"log":"[2026-02-17T13:13:36Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2605701429077684700.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dt1plwzrgr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721755000,"log":"[2026-02-17T13:12:13Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4203361746725176000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fjifr5drfo","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721756000,"log":"[2026-02-17T12:57:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wp7kn54jhg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721757000,"log":"[2026-02-17T15:08:18Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6018","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-it6hm5d019","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721758000,"log":"[2026-02-17T13:26:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9480557881267444000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qwcotupvaj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721759000,"log":"[2026-02-17T12:56:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ildn49it4i","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721760000,"log":"[2026-02-17T15:20:51Z TRACE cache::redis] cache set key=auth:token:8166","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wkau8ua0oh","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721761000,"log":"[2026-02-17T13:27:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5279300097425112000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-g31b6ytdrg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721762000,"log":"[2026-02-17T14:35:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=x7p3tzqk","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6m4pxzfe02","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721763000,"log":"[2026-02-17T14:40:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=13gqdkm3","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xmfx9rqwtt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721764000,"log":"[2026-02-17T12:50:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-700u29rkc9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721765000,"log":"[2026-02-17T15:07:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=9296","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-p8h1y8izr2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721766000,"log":"[2026-02-17T12:53:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fajykzdeo6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721767000,"log":"[2026-02-17T15:05:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=2593","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7uypaex649","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721768000,"log":"[2026-02-17T14:12:55Z DEBUG sqlx::query] DELETE FROM alerts returned 631 rows in 46ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qjca4vqjod","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721769000,"log":"[2026-02-17T12:52:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1yj6phak1z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721770000,"log":"[2026-02-17T15:14:16Z TRACE cache::redis] cache delete key=auth:token:627","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tms4j7glbz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721771000,"log":"[2026-02-17T14:39:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=plaobznl","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sqwtrj6k42","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721772000,"log":"[2026-02-17T13:49:13Z INFO actix_web::middleware::logger] GET /api/v1/metrics 200 436ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-7f51asrlxs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721773000,"log":"[2026-02-17T13:47:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 404 201ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-7tfth8omt7","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721774000,"log":"[2026-02-17T13:20:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6979407707445276000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-db22ofqyul","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721775000,"log":"[2026-02-17T13:38:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/719626240122861400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3kj4jljtk3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721776000,"log":"[2026-02-17T15:09:40Z DEBUG metrics::reporter] recorded metric active_connections value=1536","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lwigxk6l0f","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721777000,"log":"[2026-02-17T13:56:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 404 71ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3sggeceayc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721778000,"log":"[2026-02-17T13:05:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6qnn46yvdb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721779000,"log":"[2026-02-17T13:28:42Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7194786332777117000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-taqtpa9ogk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721780000,"log":"[2026-02-17T14:50:20Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8dq58uuao9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721781000,"log":"[2026-02-17T14:45:49Z INFO tonic::transport::server] grpc router request_received client_id=19csttro","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-33h3eqci12","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721782000,"log":"[2026-02-17T12:58:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-89nq288dpx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721783000,"log":"[2026-02-17T13:12:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6027762617099570000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v54eawywjb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721784000,"log":"[2026-02-17T12:40:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-52ng4l62ho","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721785000,"log":"[2026-02-17T13:06:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xq1fdkbjia","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721786000,"log":"[2026-02-17T13:28:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6326774213831661000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q3sf4gj6dr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721787000,"log":"[2026-02-17T14:35:44Z INFO tonic::transport::server] grpc router request_received client_id=hfz0r4l5","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rh4han2nfq","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721788000,"log":"[2026-02-17T14:20:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 996 rows in 18ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ock16j20vj","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721789000,"log":"[2026-02-17T13:38:02Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6716041896527285000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9hihqbycfk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721790000,"log":"[2026-02-17T12:36:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-s84yyamem4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721791000,"log":"[2026-02-17T13:43:15Z INFO actix_web::middleware::logger] DELETE /api/_search 204 286ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j06x7osamp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721792000,"log":"[2026-02-17T15:18:14Z TRACE cache::redis] cache get key=stream:schema:6097","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-23kk9ztql2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721793000,"log":"[2026-02-17T14:21:08Z DEBUG sqlx::query] INSERT FROM users returned 550 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-8plj3ck9gv","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721794000,"log":"[2026-02-17T15:18:59Z TRACE cache::redis] cache expire key=stream:schema:9054","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j30j2f6e2x","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721795000,"log":"[2026-02-17T12:38:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-r1x9brtjk7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721796000,"log":"[2026-02-17T13:44:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 204 242ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hd6u58t0vc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721797000,"log":"[2026-02-17T13:27:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7997389705592011000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fvpzrf6b6e","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721798000,"log":"[2026-02-17T14:38:36Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=xdre1b5e","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-93550s9kyk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721799000,"log":"[2026-02-17T14:41:04Z INFO tonic::transport::server] grpc router request_received client_id=ko7bpohn","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-qun83x726b","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721800000,"log":"[2026-02-17T13:58:18Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 403 248ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gvekmrzwx5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721801000,"log":"[2026-02-17T14:12:25Z DEBUG sqlx::query] DELETE FROM alerts returned 523 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lnvzk9wqfs","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721802000,"log":"[2026-02-17T12:42:18Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-al8d9081fg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721803000,"log":"[2026-02-17T13:33:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/2490366175740672500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-m0z0m917p4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721804000,"log":"[2026-02-17T14:18:26Z DEBUG sqlx::query] MERGE FROM dashboards returned 905 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v00zlbc8xp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721805000,"log":"[2026-02-17T12:58:56Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-e8p8thmpap","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721806000,"log":"[2026-02-17T14:56:21Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bs9hwe05ql","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721807000,"log":"[2026-02-17T13:26:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5566097831150647000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2i44q8h3l2","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721808000,"log":"[2026-02-17T14:13:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 476 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-nayrmuh0lm","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721809000,"log":"[2026-02-17T13:48:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 201 278ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-iu87k3o8sz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721810000,"log":"[2026-02-17T14:44:19Z INFO tonic::transport::server] grpc router request_received client_id=n57boboc","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-e9wvp55zq2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721811000,"log":"[2026-02-17T12:53:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-uy3mzp7gjc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721812000,"log":"[2026-02-17T13:59:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 200 162ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-00y7ghbyfj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721813000,"log":"[2026-02-17T14:13:54Z DEBUG sqlx::query] UPDATE FROM organizations returned 49 rows in 48ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-5vouia7nnu","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721814000,"log":"[2026-02-17T14:37:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=6i2xifqg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-buqg5g04sr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721815000,"log":"[2026-02-17T12:58:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qg1z06gt1w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721816000,"log":"[2026-02-17T13:41:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/7394957779319655000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-6gmdkyy3a9","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721817000,"log":"[2026-02-17T12:35:24Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lu11sbdkld","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721818000,"log":"[2026-02-17T15:05:11Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-2t7s7tzdqr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721819000,"log":"[2026-02-17T13:47:12Z INFO actix_web::middleware::logger] POST /api/v1/logs 403 466ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-80my867f92","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721820000,"log":"[2026-02-17T14:15:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 310 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gcbskmy15x","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721821000,"log":"[2026-02-17T12:48:58Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-vpbwizhnrp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721822000,"log":"[2026-02-17T14:34:30Z INFO tonic::transport::server] grpc compactor response_sent client_id=pkqgvu39","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nwvk4tsge7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721823000,"log":"[2026-02-17T13:36:47Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/389548187952168960.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-wlcemyfxba","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721824000,"log":"[2026-02-17T14:55:06Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3arf79hbe3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721825000,"log":"[2026-02-17T13:26:52Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3154052804852943000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-lk3gbgjhxq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721826000,"log":"[2026-02-17T12:56:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-5c0o0obl0n","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721827000,"log":"[2026-02-17T13:26:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1543234064892749000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-72o9embkt2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721828000,"log":"[2026-02-17T13:29:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5621884533559172000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-m92sh0uhgw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721829000,"log":"[2026-02-17T12:38:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2xpohmwhe1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721830000,"log":"[2026-02-17T13:32:50Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/243864362167010020.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r9qvtn0fs5","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721831000,"log":"[2026-02-17T14:21:32Z DEBUG sqlx::query] SELECT FROM streams returned 64 rows in 22ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ispwb2zs32","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721832000,"log":"[2026-02-17T15:17:03Z TRACE cache::redis] cache set key=org:config:2103","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-otgd76skyz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721833000,"log":"[2026-02-17T13:11:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5998243514397593000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-zvx7fnnbgp","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721834000,"log":"[2026-02-17T14:08:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 431 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-a5p9ijr86f","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721835000,"log":"[2026-02-17T14:08:27Z DEBUG sqlx::query] SELECT FROM streams returned 394 rows in 76ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x4xw0ns0jf","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721836000,"log":"[2026-02-17T12:42:59Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-g2qb2ihfc6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721837000,"log":"[2026-02-17T15:02:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0vdu39m99x","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721838000,"log":"[2026-02-17T12:54:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oy2g0m0oou","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721839000,"log":"[2026-02-17T12:42:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-z0gl6qh3sn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721840000,"log":"[2026-02-17T12:38:04Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-np8htdd2nf","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721841000,"log":"[2026-02-17T13:06:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3rqieocvon","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721842000,"log":"[2026-02-17T14:58:40Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-mpqy3qzq2c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721843000,"log":"[2026-02-17T14:06:02Z INFO actix_web::middleware::logger] PUT /api/v1/logs 201 340ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6lw2hcvsd2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721844000,"log":"[2026-02-17T13:43:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 236ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6w1jom422p","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721845000,"log":"[2026-02-17T12:43:09Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9154cjo7ym","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721846000,"log":"[2026-02-17T14:45:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=ndu2gy9i","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3cdbnpfura","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721847000,"log":"[2026-02-17T13:20:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4161655148495804400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1iom5yv37g","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721848000,"log":"[2026-02-17T14:17:50Z DEBUG sqlx::query] DELETE FROM alerts returned 174 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-vzi3r5qc6r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721849000,"log":"[2026-02-17T13:23:40Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/462817292428565600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-d70eugbsnw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721850000,"log":"[2026-02-17T13:14:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6542924953865890000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3ghrf7hzj0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721851000,"log":"[2026-02-17T12:42:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-evi8q3xv8s","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721852000,"log":"[2026-02-17T13:16:21Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/6279192422700079000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vu5t52o89b","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721853000,"log":"[2026-02-17T13:59:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 500 321ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xzffkfv4us","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721854000,"log":"[2026-02-17T14:04:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 404 456ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mtlpmw0otk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721855000,"log":"[2026-02-17T12:36:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-397ktejax3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721856000,"log":"[2026-02-17T12:55:30Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5grnz4n5xb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721857000,"log":"[2026-02-17T13:02:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-j8lh349g0j","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721858000,"log":"[2026-02-17T14:32:33Z INFO tonic::transport::server] grpc querier disconnected client_id=1h84ikkt","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nzcbvwin17","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721859000,"log":"[2026-02-17T13:44:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 446ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jxki6awc58","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721860000,"log":"[2026-02-17T13:44:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 201 254ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-flp1vks51z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721861000,"log":"[2026-02-17T14:54:28Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nqq8fpkewz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721862000,"log":"[2026-02-17T12:44:34Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nhhi0vjre6","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721863000,"log":"[2026-02-17T15:13:50Z TRACE cache::redis] cache get key=query:result:4282","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f83n3qrhmx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721864000,"log":"[2026-02-17T14:58:41Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dgv9oag359","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721865000,"log":"[2026-02-17T13:10:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5013164752273538000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-pymyk37ua9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721866000,"log":"[2026-02-17T13:36:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/3706767684427163600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-skqfk4h0ec","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721867000,"log":"[2026-02-17T14:49:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-x86vplstea","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721868000,"log":"[2026-02-17T13:06:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-n5nteq5hvb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721869000,"log":"[2026-02-17T14:24:18Z DEBUG sqlx::query] INSERT FROM users returned 661 rows in 96ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5hqgxjhcp6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721870000,"log":"[2026-02-17T13:34:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/7126129967053823000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-6mq4wbznlo","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721871000,"log":"[2026-02-17T14:57:24Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-q6yji95trr","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721872000,"log":"[2026-02-17T12:40:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-us3d4q5vsq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721873000,"log":"[2026-02-17T15:14:06Z TRACE cache::redis] cache hit key=auth:token:1669","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-cnk7cxsnos","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721874000,"log":"[2026-02-17T14:33:58Z INFO tonic::transport::server] grpc querier disconnected client_id=9qlkwofz","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-1lqgpigge1","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721875000,"log":"[2026-02-17T13:46:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 400 242ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mvwzhqz58y","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721876000,"log":"[2026-02-17T12:50:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n1yzlun4rt","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721877000,"log":"[2026-02-17T12:52:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ffb18ltlgx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721878000,"log":"[2026-02-17T14:11:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 219 rows in 97ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ikrcejsk4y","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721879000,"log":"[2026-02-17T13:17:29Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2959806649316589600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d5yaea9q22","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721880000,"log":"[2026-02-17T14:07:08Z DEBUG sqlx::query] INSERT FROM users returned 952 rows in 89ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-df5b3c71rr","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721881000,"log":"[2026-02-17T15:13:08Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=7842","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-a0i3z2tpbs","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721882000,"log":"[2026-02-17T12:55:16Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-yfwzwrsb1t","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721883000,"log":"[2026-02-17T13:25:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7655574374660920000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-knzafc95gh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721884000,"log":"[2026-02-17T14:58:07Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-75ksr256h4","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721885000,"log":"[2026-02-17T14:06:40Z INFO actix_web::middleware::logger] POST /api/_search 400 260ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yin8ti9qaz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721886000,"log":"[2026-02-17T12:50:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-d1oo0wpo46","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721887000,"log":"[2026-02-17T14:06:55Z DEBUG sqlx::query] DELETE FROM alerts returned 961 rows in 19ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-2qngk4llmd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721888000,"log":"[2026-02-17T14:31:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 121 rows in 33ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ds9kn52gxl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721889000,"log":"[2026-02-17T14:40:41Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=qlmqsq0z","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1y7te07xba","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721890000,"log":"[2026-02-17T14:20:46Z DEBUG sqlx::query] MERGE FROM dashboards returned 978 rows in 45ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mgvxsfw4sq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721891000,"log":"[2026-02-17T13:45:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 403 373ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-lxljex74mb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721892000,"log":"[2026-02-17T12:47:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kgimuf45ot","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721893000,"log":"[2026-02-17T14:59:51Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c3ygynvqox","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721894000,"log":"[2026-02-17T14:04:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 165ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-30af7oji3a","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721895000,"log":"[2026-02-17T14:54:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jq19u07d3v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721896000,"log":"[2026-02-17T14:02:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 500 152ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-0c7go7xuan","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721897000,"log":"[2026-02-17T12:36:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-xfav1li0wy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721898000,"log":"[2026-02-17T14:57:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3g98odkcv5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721899000,"log":"[2026-02-17T15:17:48Z TRACE cache::redis] cache hit key=org:config:3823","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-mst4o8nfnu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721900000,"log":"[2026-02-17T15:07:24Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=4670","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-l5ohe82tbt","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721901000,"log":"[2026-02-17T14:57:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-djcv2mh8uc","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721902000,"log":"[2026-02-17T13:42:20Z INFO actix_web::middleware::logger] POST /api/_search 201 274ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sskdud9kkx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721903000,"log":"[2026-02-17T13:38:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/8693043856882616000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rgbsbhmwr3","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721904000,"log":"[2026-02-17T14:22:41Z DEBUG sqlx::query] MERGE FROM dashboards returned 826 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q7wf18tg6k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721905000,"log":"[2026-02-17T15:04:01Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aj9cxc99vi","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721906000,"log":"[2026-02-17T14:35:05Z INFO tonic::transport::server] grpc compactor response_sent client_id=kd8rzq4l","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-a328hpasrg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721907000,"log":"[2026-02-17T15:01:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4f148lwc46","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721908000,"log":"[2026-02-17T14:26:22Z DEBUG sqlx::query] SELECT FROM streams returned 537 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-fnn5gxayq5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721909000,"log":"[2026-02-17T12:37:35Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0ig17vqgiu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721910000,"log":"[2026-02-17T13:39:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9256440395811668000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wwdsfd95yg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721911000,"log":"[2026-02-17T12:47:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gvtry4q7ak","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721912000,"log":"[2026-02-17T14:17:36Z DEBUG sqlx::query] MERGE FROM dashboards returned 431 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vs4ulbff9m","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721913000,"log":"[2026-02-17T15:11:19Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=574","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-w95wujrw4s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721914000,"log":"[2026-02-17T12:51:21Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8tyu2m6nhl","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721915000,"log":"[2026-02-17T13:43:32Z INFO actix_web::middleware::logger] POST /api/v1/logs 201 394ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-tj0l5kn1bt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721916000,"log":"[2026-02-17T13:14:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/1829188949920448300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xz2n05lmxf","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721917000,"log":"[2026-02-17T14:51:10Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wsc1k4spzp","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721918000,"log":"[2026-02-17T13:10:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7134957195134117000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-dqi8qhaqob","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721919000,"log":"[2026-02-17T15:02:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-41p7o7bmf7","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721920000,"log":"[2026-02-17T13:39:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/8284181336069695000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ptk7ytct5d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721921000,"log":"[2026-02-17T13:35:51Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4483507334482499600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y91ytwc3r2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721922000,"log":"[2026-02-17T14:52:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-poyjkdbvf8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721923000,"log":"[2026-02-17T13:37:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3875983540258703000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9h57pl756r","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721924000,"log":"[2026-02-17T13:29:16Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/649540217684363400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-d5ara7kyms","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721925000,"log":"[2026-02-17T14:44:27Z INFO tonic::transport::server] grpc ingester connected client_id=t7tddhxo","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ugsqk6sg5d","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721926000,"log":"[2026-02-17T14:06:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 296ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jifhj3o0ft","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721927000,"log":"[2026-02-17T13:09:54Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2543161135775833000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mt63kjdnbn","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721928000,"log":"[2026-02-17T12:50:37Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-f7dokxlvxn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721929000,"log":"[2026-02-17T13:09:24Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7363114105724222000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-d4sqmejleb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721930000,"log":"[2026-02-17T13:57:30Z INFO actix_web::middleware::logger] PUT /api/_search 204 71ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7vui30g5vb","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721931000,"log":"[2026-02-17T14:11:11Z DEBUG sqlx::query] MERGE FROM dashboards returned 551 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8m947yzk0h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721932000,"log":"[2026-02-17T13:52:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 289ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3d368e6uyw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721933000,"log":"[2026-02-17T14:22:32Z DEBUG sqlx::query] SELECT FROM streams returned 915 rows in 44ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-1aywda4zd7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721934000,"log":"[2026-02-17T14:48:09Z INFO tonic::transport::server] grpc router request_received client_id=s8tfrjrv","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-efl2vstyav","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721935000,"log":"[2026-02-17T13:09:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/4716071084428259000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3ogpr8sbda","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721936000,"log":"[2026-02-17T14:39:14Z INFO tonic::transport::server] grpc router request_received client_id=bnjbojjv","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-lzmxvo66ym","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721937000,"log":"[2026-02-17T14:25:28Z DEBUG sqlx::query] INSERT FROM users returned 809 rows in 9ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-32n0vusx1o","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721938000,"log":"[2026-02-17T15:18:57Z TRACE cache::redis] cache set key=user:session:4255","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pxanpu7dmk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721939000,"log":"[2026-02-17T13:20:00Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/9250419549044700000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-k1lbor2wei","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721940000,"log":"[2026-02-17T13:16:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/5290230545956687000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ow4se9jo2k","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721941000,"log":"[2026-02-17T13:12:41Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5108916906643459000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i49vrde05d","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721942000,"log":"[2026-02-17T13:33:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/5053350686213161000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hzmw52vojr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721943000,"log":"[2026-02-17T12:39:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xyrd8fsrip","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721944000,"log":"[2026-02-17T13:33:59Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/1464947181432065800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-mgivtmzpsz","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721945000,"log":"[2026-02-17T14:06:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 383 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-xqf1gedm5h","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721946000,"log":"[2026-02-17T15:19:06Z TRACE cache::redis] cache hit key=auth:token:7754","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gtdhjrhdrr","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721947000,"log":"[2026-02-17T15:07:11Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=7568","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-14n761gtl9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721948000,"log":"[2026-02-17T13:02:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8icauxgck6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721949000,"log":"[2026-02-17T14:21:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 105 rows in 66ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9ggvo74i4f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721950000,"log":"[2026-02-17T12:39:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c4x2cm8au9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721951000,"log":"[2026-02-17T13:28:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/8162650321988192000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-vang0htbbq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721952000,"log":"[2026-02-17T15:03:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nvdkew114r","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721953000,"log":"[2026-02-17T13:58:26Z INFO actix_web::middleware::logger] PUT /api/_bulk 401 410ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1eku85chk6","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721954000,"log":"[2026-02-17T14:59:50Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l35yxfy2mx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721955000,"log":"[2026-02-17T13:51:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 400 205ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-qpi3gsi836","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721956000,"log":"[2026-02-17T15:06:28Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=3300","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ooax1yqm4e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721957000,"log":"[2026-02-17T14:27:21Z DEBUG sqlx::query] MERGE FROM dashboards returned 526 rows in 17ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-n3sduu2k4k","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721958000,"log":"[2026-02-17T14:18:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 921 rows in 100ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-r9nnc0g6kn","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721959000,"log":"[2026-02-17T13:24:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3573691111127379500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fc9oa2a0w8","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721960000,"log":"[2026-02-17T14:04:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 204 129ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p2fowxdjmm","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721961000,"log":"[2026-02-17T13:39:25Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/2402095823604919300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-3ovo26bqfx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721962000,"log":"[2026-02-17T14:42:11Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=tciblkpf","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-vz273txkm9","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721963000,"log":"[2026-02-17T13:01:44Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fd6s7wuzlb","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721964000,"log":"[2026-02-17T14:10:40Z DEBUG sqlx::query] DELETE FROM alerts returned 538 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-18dajssph2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721965000,"log":"[2026-02-17T13:46:04Z INFO actix_web::middleware::logger] POST /api/v1/traces 200 164ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-eboo8iyk0f","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721966000,"log":"[2026-02-17T13:15:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6833643692388545000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3w8pdoq95k","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721967000,"log":"[2026-02-17T12:38:15Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4j17aprx6t","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721968000,"log":"[2026-02-17T12:45:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l1cgyxcrlt","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721969000,"log":"[2026-02-17T13:37:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/9315930191407434000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-vjpumsos69","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721970000,"log":"[2026-02-17T13:47:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 201 311ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tidapdjawh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721971000,"log":"[2026-02-17T13:57:01Z INFO actix_web::middleware::logger] GET /api/_bulk 200 106ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-eue67ot435","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721972000,"log":"[2026-02-17T13:06:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6w1mudlfat","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721973000,"log":"[2026-02-17T12:44:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-c79fpqisrs","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721974000,"log":"[2026-02-17T14:25:43Z DEBUG sqlx::query] INSERT FROM users returned 590 rows in 98ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nqo3nbozob","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721975000,"log":"[2026-02-17T14:21:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 163 rows in 26ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0bslx8ji52","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721976000,"log":"[2026-02-17T15:03:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5iwtykw4yg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721977000,"log":"[2026-02-17T13:30:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2176708606802193400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-4ypjluk0iy","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721978000,"log":"[2026-02-17T14:06:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 500 485ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jdh6nuk4m9","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721979000,"log":"[2026-02-17T14:42:28Z INFO tonic::transport::server] grpc querier disconnected client_id=392nosrv","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-66h3xex3ce","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721980000,"log":"[2026-02-17T12:53:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gaokp4dsui","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721981000,"log":"[2026-02-17T13:20:30Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/949257401706995500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-xg4uu9wx2d","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721982000,"log":"[2026-02-17T13:38:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5316142540628888000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-wh0jpv5xe6","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721983000,"log":"[2026-02-17T14:13:23Z DEBUG sqlx::query] INSERT FROM users returned 720 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ol4omt0l0w","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721984000,"log":"[2026-02-17T14:19:50Z DEBUG sqlx::query] DELETE FROM alerts returned 943 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tlkipfeue8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721985000,"log":"[2026-02-17T14:33:32Z INFO tonic::transport::server] grpc ingester connected client_id=o535egvj","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-emfv741xe9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721986000,"log":"[2026-02-17T14:47:14Z INFO tonic::transport::server] grpc router request_received client_id=d6fluixr","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-tda8rcr00q","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721987000,"log":"[2026-02-17T14:30:50Z DEBUG sqlx::query] DELETE FROM alerts returned 135 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v1wehdn5i8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721988000,"log":"[2026-02-17T14:03:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 201 492ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uy7f8qy6fx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721989000,"log":"[2026-02-17T15:00:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7jnskockq8","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721990000,"log":"[2026-02-17T15:18:51Z TRACE cache::redis] cache set key=auth:token:4589","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bielo9sfa7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721991000,"log":"[2026-02-17T14:52:46Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-obaqqfyodb","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331721992000,"log":"[2026-02-17T14:05:35Z INFO actix_web::middleware::logger] DELETE /api/_search 201 479ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-708kddelgc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721993000,"log":"[2026-02-17T13:49:07Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 400 495ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-kh206b5ige","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721994000,"log":"[2026-02-17T13:53:38Z INFO actix_web::middleware::logger] PUT /api/v1/metrics 401 333ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-zdtag1od5i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721995000,"log":"[2026-02-17T13:46:59Z INFO actix_web::middleware::logger] DELETE /api/v1/traces 201 40ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0ektk51bfx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721996000,"log":"[2026-02-17T14:02:01Z INFO actix_web::middleware::logger] GET /api/_bulk 400 492ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-urkw90vgdg","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721997000,"log":"[2026-02-17T14:45:56Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=czch47nb","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-7g92pgcmnx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331721998000,"log":"[2026-02-17T13:04:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cjhrjc0inp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331721999000,"log":"[2026-02-17T13:31:15Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/720999166816822500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-14vx2val71","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722000000,"log":"[2026-02-17T13:05:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-np6ics28fe","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722001000,"log":"[2026-02-17T14:33:26Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=gteb3mnu","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-qlr9r444ok","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722002000,"log":"[2026-02-17T12:48:43Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-erpfs2w7vu","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722003000,"log":"[2026-02-17T13:11:06Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4872240444505332000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8hhb5bhyed","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722004000,"log":"[2026-02-17T14:17:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 72 rows in 75ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-sslbw46kyx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722005000,"log":"[2026-02-17T12:50:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-k7cslluwrc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722006000,"log":"[2026-02-17T13:37:37Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/7862134468972366000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-loukme86lk","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722007000,"log":"[2026-02-17T14:34:21Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=udn3sspq","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-ismp6x8cu2","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722008000,"log":"[2026-02-17T13:09:55Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6137796035306267000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-lzedqlb85c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722009000,"log":"[2026-02-17T15:11:56Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=5980","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-3fthc05pul","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722010000,"log":"[2026-02-17T15:05:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=3068","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0qjmwfy71r","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722011000,"log":"[2026-02-17T13:58:53Z INFO actix_web::middleware::logger] GET /api/v1/metrics 401 293ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ldb58urxvj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722012000,"log":"[2026-02-17T15:12:07Z DEBUG metrics::reporter] recorded metric http_requests_total value=9387","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-x4juk45dqx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722013000,"log":"[2026-02-17T13:12:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/4415953698592218600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-msdrsg8xoq","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722014000,"log":"[2026-02-17T12:40:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pwgbn643v5","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722015000,"log":"[2026-02-17T13:34:45Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/5978526850830841000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-sg1nyij95n","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722016000,"log":"[2026-02-17T13:34:58Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3058863094664246000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-4xpw9egd88","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722017000,"log":"[2026-02-17T14:23:37Z DEBUG sqlx::query] SELECT FROM streams returned 850 rows in 36ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ha8cq7qqgx","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722018000,"log":"[2026-02-17T15:15:41Z TRACE cache::redis] cache expire key=auth:token:3033","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i8xxkgshuq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722019000,"log":"[2026-02-17T14:29:30Z DEBUG sqlx::query] DELETE FROM alerts returned 825 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4j4zojzlfr","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722020000,"log":"[2026-02-17T14:48:06Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=l3x2n36y","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mb16k9vmka","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722021000,"log":"[2026-02-17T12:41:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dw6x60ehm7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722022000,"log":"[2026-02-17T14:14:47Z DEBUG sqlx::query] SELECT FROM streams returned 455 rows in 65ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3fmdv3ocvn","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722023000,"log":"[2026-02-17T13:01:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-bxws4tcqty","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722024000,"log":"[2026-02-17T13:56:52Z INFO actix_web::middleware::logger] POST /api/v1/logs 200 346ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jxoe2bo415","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722025000,"log":"[2026-02-17T13:05:12Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-gfoijt25yo","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722026000,"log":"[2026-02-17T12:50:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-s4fhxjke17","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722027000,"log":"[2026-02-17T13:26:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1739060643008003600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-s9wii4sa1d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722028000,"log":"[2026-02-17T15:16:12Z TRACE cache::redis] cache hit key=user:session:6901","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ljhr2ii1h2","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722029000,"log":"[2026-02-17T13:48:20Z INFO actix_web::middleware::logger] POST /api/_search 201 143ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-nv4ljtxqxv","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722030000,"log":"[2026-02-17T13:28:14Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/6314764290716959000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-uldofxfgg9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722031000,"log":"[2026-02-17T13:26:44Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2401450031375350000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0m8fxp8f3t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722032000,"log":"[2026-02-17T14:04:44Z INFO actix_web::middleware::logger] POST /api/v1/traces 401 103ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6c35gry44","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722033000,"log":"[2026-02-17T14:40:31Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=is7zuvic","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-fq5uwldsii","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722034000,"log":"[2026-02-17T12:56:38Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tqaa2e47l4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722035000,"log":"[2026-02-17T14:45:18Z INFO tonic::transport::server] grpc querier disconnected client_id=zyp4wdyf","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hc7fedcgbj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722036000,"log":"[2026-02-17T14:28:25Z DEBUG sqlx::query] DELETE FROM alerts returned 499 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-8uks7i1qum","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722037000,"log":"[2026-02-17T12:48:07Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-8p9faf6nqa","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722038000,"log":"[2026-02-17T13:40:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4919789971767776000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3rsbzqytqp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722039000,"log":"[2026-02-17T14:16:14Z DEBUG sqlx::query] UPDATE FROM organizations returned 617 rows in 90ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fxiwe9c7ct","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722040000,"log":"[2026-02-17T14:42:47Z INFO tonic::transport::server] grpc ingester connected client_id=duxceylb","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9hmxx33lw3","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722041000,"log":"[2026-02-17T14:59:59Z ERROR zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0snr2tgsb2","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722042000,"log":"[2026-02-17T14:33:19Z INFO tonic::transport::server] grpc router request_received client_id=d1wf9ha7","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-gw0pjpqxfj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722043000,"log":"[2026-02-17T14:48:56Z ERROR zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-8u0lgm0nxe","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722044000,"log":"[2026-02-17T14:03:49Z INFO actix_web::middleware::logger] GET /api/v1/traces 400 91ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-ntj9vq53xw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722045000,"log":"[2026-02-17T12:36:47Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iu2pueiaa8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722046000,"log":"[2026-02-17T14:46:50Z INFO tonic::transport::server] grpc compactor response_sent client_id=o0xxpytg","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yqcmkmyxms","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722047000,"log":"[2026-02-17T12:48:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dv7hk1vk6s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722048000,"log":"[2026-02-17T14:31:12Z DEBUG sqlx::query] SELECT FROM streams returned 2 rows in 93ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-uujbjnbcs7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722049000,"log":"[2026-02-17T13:55:47Z INFO actix_web::middleware::logger] DELETE /api/v1/logs 500 184ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-eeimvml50h","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722050000,"log":"[2026-02-17T13:25:38Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/4522427587897215500.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-fbwoz8bhzy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722051000,"log":"[2026-02-17T14:13:32Z DEBUG sqlx::query] SELECT FROM streams returned 965 rows in 80ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-ccoha9wbej","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722052000,"log":"[2026-02-17T13:43:54Z INFO actix_web::middleware::logger] PUT /api/v1/traces 403 444ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-b6wnpsoivs","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722053000,"log":"[2026-02-17T13:00:33Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-l69zpn2r9z","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722054000,"log":"[2026-02-17T14:45:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=yl3782kh","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-pmgcczc19o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722055000,"log":"[2026-02-17T15:10:14Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=543","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-fxfkoeebaj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722056000,"log":"[2026-02-17T14:05:50Z INFO actix_web::middleware::logger] PUT /api/_search 500 57ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-qmxv6ulg0o","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722057000,"log":"[2026-02-17T13:09:53Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/7080078903701687000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-usywybo29d","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722058000,"log":"[2026-02-17T13:24:43Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/525488033140126850.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-da8jpwleb6","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722059000,"log":"[2026-02-17T13:03:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-3mtbb23hur","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722060000,"log":"[2026-02-17T13:09:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3419574793177382400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-tgxynctp0p","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722061000,"log":"[2026-02-17T12:45:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-osaokhmog9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722062000,"log":"[2026-02-17T14:00:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 281ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ookiyr82hx","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722063000,"log":"[2026-02-17T13:47:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 149ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-y63ydq50fa","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722064000,"log":"[2026-02-17T15:03:05Z ERROR zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dtjdmbk1x3","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722065000,"log":"[2026-02-17T14:04:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 503 432ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-tc67inzerh","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722066000,"log":"[2026-02-17T13:09:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/6202845298673958000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9ktfkji4ew","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722067000,"log":"[2026-02-17T13:36:28Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/1089079544393802900.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-qj2e5forf4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722068000,"log":"[2026-02-17T13:05:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-4o8rklpmww","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722069000,"log":"[2026-02-17T12:56:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-198hcduqc0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722070000,"log":"[2026-02-17T12:36:41Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1ktbhf45pw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722071000,"log":"[2026-02-17T12:41:50Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-41fk9tqife","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722072000,"log":"[2026-02-17T13:22:17Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3395666779211555000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9be564ldrm","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722073000,"log":"[2026-02-17T12:52:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-0xc4wh1n7z","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722074000,"log":"[2026-02-17T13:06:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bhon8wn2ql","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722075000,"log":"[2026-02-17T12:37:40Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l83zacosqu","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722076000,"log":"[2026-02-17T15:21:27Z TRACE cache::redis] cache set key=user:session:2678","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xrfjk75wih","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722077000,"log":"[2026-02-17T15:06:06Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=4949","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-461glh5duq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722078000,"log":"[2026-02-17T13:50:33Z INFO actix_web::middleware::logger] GET /api/v1/metrics 503 221ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-sdswirkkgy","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722079000,"log":"[2026-02-17T14:02:15Z INFO actix_web::middleware::logger] DELETE /api/_search 503 12ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-ch8vbd4cxl","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722080000,"log":"[2026-02-17T14:10:27Z DEBUG sqlx::query] SELECT FROM streams returned 923 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-9kfnxpktco","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722081000,"log":"[2026-02-17T14:34:19Z INFO tonic::transport::server] grpc router request_received client_id=qtu8ixat","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-35vvp4gcqx","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722082000,"log":"[2026-02-17T14:18:43Z DEBUG sqlx::query] INSERT FROM users returned 385 rows in 25ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-5ptsk026df","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722083000,"log":"[2026-02-17T13:50:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 503 268ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mf1qwzgqhc","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722084000,"log":"[2026-02-17T13:09:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/4842918677221379000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aqfgx36hbu","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722085000,"log":"[2026-02-17T14:49:43Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-6r34siza87","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722086000,"log":"[2026-02-17T13:55:06Z INFO actix_web::middleware::logger] PUT /api/_bulk 204 394ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-wn0oirghvz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722087000,"log":"[2026-02-17T15:13:43Z TRACE cache::redis] cache miss key=org:config:6692","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-22nw4p5wbp","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722088000,"log":"[2026-02-17T14:49:15Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-j7rxikdo4a","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722089000,"log":"[2026-02-17T13:05:01Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-y56xuiwsc4","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722090000,"log":"[2026-02-17T14:57:34Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-35h2z531zj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722091000,"log":"[2026-02-17T12:38:31Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bahrpbvqtz","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722092000,"log":"[2026-02-17T14:49:52Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-p81pejftzl","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722093000,"log":"[2026-02-17T14:19:28Z DEBUG sqlx::query] INSERT FROM users returned 501 rows in 34ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yea4uurgec","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722094000,"log":"[2026-02-17T13:35:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3562486223950818300.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yjqvxyrn0k","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722095000,"log":"[2026-02-17T14:21:58Z DEBUG sqlx::query] INSERT FROM users returned 501 rows in 14ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ek9lae8b6q","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722096000,"log":"[2026-02-17T13:51:56Z INFO actix_web::middleware::logger] POST /api/_bulk 201 420ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dugyf5qjpw","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722097000,"log":"[2026-02-17T14:04:36Z INFO actix_web::middleware::logger] POST /api/_bulk 403 23ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dkes7i59hk","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722098000,"log":"[2026-02-17T15:06:26Z DEBUG metrics::reporter] recorded metric memory_usage_bytes value=6224","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-q94r4hkxnx","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722099000,"log":"[2026-02-17T15:06:57Z DEBUG metrics::reporter] recorded metric http_requests_total value=7180","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-l942k53rj9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722100000,"log":"[2026-02-17T14:40:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=7oy4c5wu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-9abto6ywr2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722101000,"log":"[2026-02-17T14:45:45Z INFO tonic::transport::server] grpc compactor response_sent client_id=ev3w5wk2","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-1rm6yhoicg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722102000,"log":"[2026-02-17T13:39:07Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9518642412180537000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-haals6cjnt","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722103000,"log":"[2026-02-17T13:25:23Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/9470839819093604000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eirrua5yj3","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722104000,"log":"[2026-02-17T14:44:16Z INFO tonic::transport::server] grpc alertmanager stream_opened client_id=8lzctvy1","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-dcahsmfmty","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722105000,"log":"[2026-02-17T13:40:46Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/2377847568361187000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-gbwcelp31s","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722106000,"log":"[2026-02-17T14:03:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 402ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-e3w6nrhtf4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722107000,"log":"[2026-02-17T14:35:43Z INFO tonic::transport::server] grpc querier disconnected client_id=p3xxzsew","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xcp0qamruw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722108000,"log":"[2026-02-17T12:45:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-pxjilatlu7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722109000,"log":"[2026-02-17T14:47:25Z INFO tonic::transport::server] grpc compactor response_sent client_id=661a91nh","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-b3ym2hrxs4","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722110000,"log":"[2026-02-17T12:51:53Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wl3w0gck65","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722111000,"log":"[2026-02-17T13:46:11Z INFO actix_web::middleware::logger] DELETE /api/_bulk 500 386ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-swwe4o11k8","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722112000,"log":"[2026-02-17T14:13:15Z DEBUG sqlx::query] DELETE FROM alerts returned 654 rows in 24ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-41leq4qpl0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722113000,"log":"[2026-02-17T12:58:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jug9dcc536","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722114000,"log":"[2026-02-17T14:53:33Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-jgmro54u74","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722115000,"log":"[2026-02-17T12:46:06Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-mz4rx4turq","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722116000,"log":"[2026-02-17T15:08:53Z DEBUG metrics::reporter] recorded metric query_duration_seconds value=6777","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-nsdnqz7vd7","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722117000,"log":"[2026-02-17T14:24:57Z DEBUG sqlx::query] SELECT FROM streams returned 463 rows in 15ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-q3lct9zdts","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722118000,"log":"[2026-02-17T12:44:17Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-dakj0hu83g","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722119000,"log":"[2026-02-17T15:00:00Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-074d07aph2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722120000,"log":"[2026-02-17T15:06:59Z DEBUG metrics::reporter] recorded metric ingestion_bytes_total value=8714","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-r72z2lblv9","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722121000,"log":"[2026-02-17T13:33:57Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/3422803234750615600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-icge387ulb","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722122000,"log":"[2026-02-17T13:31:34Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/7432370223018547000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jymu4zv6pa","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722123000,"log":"[2026-02-17T14:12:10Z DEBUG sqlx::query] DELETE FROM alerts returned 356 rows in 71ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kcei9ttqc","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722124000,"log":"[2026-02-17T12:41:14Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6xphw278wj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722125000,"log":"[2026-02-17T14:14:01Z DEBUG sqlx::query] MERGE FROM dashboards returned 845 rows in 32ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-smwyl9hkb0","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722126000,"log":"[2026-02-17T14:48:32Z ERROR zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v82pxqy1gj","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722127000,"log":"[2026-02-17T14:36:29Z INFO tonic::transport::server] grpc router request_received client_id=103tzmlp","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-9e3elnv7t","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722128000,"log":"[2026-02-17T14:27:27Z DEBUG sqlx::query] SELECT FROM streams returned 27 rows in 16ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-4sc1wzocxz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722129000,"log":"[2026-02-17T15:17:02Z TRACE cache::redis] cache get key=user:session:9061","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-jl9s697ql9","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722130000,"log":"[2026-02-17T14:15:39Z DEBUG sqlx::query] UPDATE FROM organizations returned 632 rows in 6ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2x64h2oji1","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722131000,"log":"[2026-02-17T13:03:25Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-dd91xk7agk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722132000,"log":"[2026-02-17T13:14:22Z INFO zinc_enl::service::search::datafusion] search file: Bhargav_organization_29/logs/default/2022/12/26/20/9544356221983009000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-s58fwhl8qr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722133000,"log":"[2026-02-17T12:49:11Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-v8tjvdh45g","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722134000,"log":"[2026-02-17T14:02:22Z INFO actix_web::middleware::logger] PUT /api/v1/logs 404 299ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-svaf6d1j7w","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722135000,"log":"[2026-02-17T13:04:49Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-0g906m9dsj","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722136000,"log":"[2026-02-17T14:28:53Z DEBUG sqlx::query] INSERT FROM users returned 142 rows in 94ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-cy2hkmiuut","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722137000,"log":"[2026-02-17T12:42:48Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-xa190h1n6i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722138000,"log":"[2026-02-17T13:52:42Z INFO actix_web::middleware::logger] PUT /api/v1/logs 204 72ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-corthfab3t","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722139000,"log":"[2026-02-17T12:56:22Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-th7kpvi4jr","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722140000,"log":"[2026-02-17T15:21:05Z TRACE cache::redis] cache expire key=query:result:5650","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-jwssbe9wvz","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722141000,"log":"[2026-02-17T12:52:10Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-7s3eg55y5c","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722142000,"log":"[2026-02-17T13:06:51Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-84dtkwms8i","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722143000,"log":"[2026-02-17T13:17:35Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/6761109533461642000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-f6hlpg3wue","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722144000,"log":"[2026-02-17T15:16:57Z TRACE cache::redis] cache set key=user:session:264","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-3wtdb4p07f","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722145000,"log":"[2026-02-17T14:24:13Z DEBUG sqlx::query] INSERT FROM users returned 359 rows in 42ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-eaprn21clk","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722146000,"log":"[2026-02-17T12:37:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-5d4hh1bums","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722147000,"log":"[2026-02-17T13:08:00Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-oali91ppwg","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722148000,"log":"[2026-02-17T14:20:30Z DEBUG sqlx::query] DELETE FROM alerts returned 150 rows in 82ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-m7kuzitew0","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722149000,"log":"[2026-02-17T12:50:02Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-37ocz4q1x8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722150000,"log":"[2026-02-17T14:26:50Z DEBUG sqlx::query] DELETE FROM alerts returned 959 rows in 63ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yc0ugfszed","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722151000,"log":"[2026-02-17T13:53:57Z INFO actix_web::middleware::logger] GET /api/v1/logs 403 141ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-2x47uuax59","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722152000,"log":"[2026-02-17T14:55:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wykbukll9i","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722153000,"log":"[2026-02-17T13:43:55Z INFO actix_web::middleware::logger] DELETE /api/_search 404 40ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-rt6ssfj911","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722154000,"log":"[2026-02-17T13:16:20Z INFO zinc_enl::service::search::datafusion] search file: staging/logs/logs/2022/12/26/20/3173025739488915000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-kbu86jpq74","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722155000,"log":"[2026-02-17T13:55:00Z INFO actix_web::middleware::logger] POST /api/_search 403 89ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-r72yer08j2","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722156000,"log":"[2026-02-17T14:47:42Z INFO tonic::transport::server] grpc ingester connected client_id=qdfoyn53","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-yh5vycrh90","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722157000,"log":"[2026-02-17T14:47:03Z INFO tonic::transport::server] grpc querier disconnected client_id=yo7ult0j","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-0js5ablony","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722158000,"log":"[2026-02-17T13:16:48Z INFO zinc_enl::service::search::datafusion] search file: default_org/logs/metrics/2022/12/26/20/3549698360537616400.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-hgg2g69wuj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722159000,"log":"[2026-02-17T14:30:03Z DEBUG sqlx::query] INSERT FROM users returned 595 rows in 78ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2uch9di6jc","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722160000,"log":"[2026-02-17T14:01:08Z INFO actix_web::middleware::logger] POST /api/v1/metrics 401 317ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g5c87o0umw","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722161000,"log":"[2026-02-17T15:01:30Z WARN zinc_enl::handler::http] AuthenticationError: authentication token expired","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-zow97mqgdj","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722162000,"log":"[2026-02-17T13:44:34Z INFO actix_web::middleware::logger] PUT /api/v1/traces 200 158ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-jwane1sv14","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722163000,"log":"[2026-02-17T13:04:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-se8widpqta","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722164000,"log":"[2026-02-17T12:47:42Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-iwrvy1zbcz","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722165000,"log":"[2026-02-17T14:12:08Z DEBUG sqlx::query] INSERT FROM users returned 524 rows in 99ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-obc9h0rws4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722166000,"log":"[2026-02-17T12:57:05Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4famvjoeut","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722167000,"log":"[2026-02-17T13:28:49Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/2727154625291193000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-42ol1r6xjw","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722168000,"log":"[2026-02-17T14:18:32Z DEBUG sqlx::query] SELECT FROM streams returned 207 rows in 95ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-v6u1apv1q8","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722169000,"log":"[2026-02-17T12:48:08Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ig9e7exwjz","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722170000,"log":"[2026-02-17T14:42:23Z INFO tonic::transport::server] grpc querier disconnected client_id=ktiv1hx5","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-2m0fcts0gf","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722171000,"log":"[2026-02-17T15:09:45Z DEBUG metrics::reporter] recorded metric active_connections value=7898","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-4av8wpou9l","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722172000,"log":"[2026-02-17T14:28:18Z DEBUG sqlx::query] INSERT FROM users returned 492 rows in 8ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-wo8bmx827u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722173000,"log":"[2026-02-17T14:39:35Z INFO tonic::transport::server] grpc compactor response_sent client_id=gavwktsb","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-5505wa49jk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722174000,"log":"[2026-02-17T15:02:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-is9cq07usg","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722175000,"log":"[2026-02-17T13:31:04Z INFO zinc_enl::service::search::datafusion] search file: production/logs/traces/2022/12/26/20/999993993537524600.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-m4t40mjceh","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722176000,"log":"[2026-02-17T13:26:56Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1751859928210903800.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-i0b74teh65","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722177000,"log":"[2026-02-17T13:53:37Z INFO actix_web::middleware::logger] GET /api/v1/logs 400 30ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6tg1rdou82","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722178000,"log":"[2026-02-17T14:51:04Z WARN zinc_enl::handler::http] ValidationError: invalid request payload","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-2u3bvera2s","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722179000,"log":"[2026-02-17T14:25:31Z DEBUG sqlx::query] MERGE FROM dashboards returned 332 rows in 51ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-81a7mzqvx3","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722180000,"log":"[2026-02-17T13:54:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 469ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-nwd9h2ctub","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722181000,"log":"[2026-02-17T12:55:36Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-jznw0duc9p","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722182000,"log":"[2026-02-17T15:03:16Z WARN zinc_enl::handler::http] RateLimitError: rate limit exceeded for client","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-idaq5dcxo7","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722183000,"log":"[2026-02-17T12:39:19Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-rcdisfe85g","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722184000,"log":"[2026-02-17T14:41:40Z INFO tonic::transport::server] grpc compactor response_sent client_id=ivqub7zu","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-78mzo8g672","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722185000,"log":"[2026-02-17T13:53:36Z INFO actix_web::middleware::logger] POST /api/_bulk 204 173ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-tjb0g5lt7j","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722186000,"log":"[2026-02-17T13:48:50Z INFO actix_web::middleware::logger] PUT /api/_search 401 204ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-fxjb413qos","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722187000,"log":"[2026-02-17T14:54:03Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-agpefcyrmn","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722188000,"log":"[2026-02-17T14:11:49Z DEBUG sqlx::query] UPDATE FROM organizations returned 138 rows in 47ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-p71vfzsi94","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722189000,"log":"[2026-02-17T14:14:58Z DEBUG sqlx::query] INSERT FROM users returned 623 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-g2gxucfibj","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722190000,"log":"[2026-02-17T14:11:17Z DEBUG sqlx::query] SELECT FROM streams returned 484 rows in 2ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-6zxkppdk4j","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722191000,"log":"[2026-02-17T15:09:30Z DEBUG metrics::reporter] recorded metric active_connections value=4569","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-l8jj7434jw","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722192000,"log":"[2026-02-17T14:03:24Z INFO actix_web::middleware::logger] POST /api/v1/traces 403 188ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-6x207s7tav","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722193000,"log":"[2026-02-17T12:59:57Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oj7tg8j3n0","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722194000,"log":"[2026-02-17T14:50:27Z WARN zinc_enl::handler::http] ConnectionError: failed to connect to upstream","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-xlz6j8fhku","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722195000,"log":"[2026-02-17T14:10:16Z DEBUG sqlx::query] MERGE FROM dashboards returned 292 rows in 27ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-aevdevpd48","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722196000,"log":"[2026-02-17T14:24:42Z DEBUG sqlx::query] SELECT FROM streams returned 892 rows in 55ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-mge7n4gvi7","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722197000,"log":"[2026-02-17T12:40:39Z INFO aws_config::default_provider::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-3es5j97zg8","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722198000,"log":"[2026-02-17T14:24:29Z DEBUG sqlx::query] UPDATE FROM organizations returned 182 rows in 62ms","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-01cbaudoto","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722199000,"log":"[2026-02-17T12:41:13Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-ew58hr6zub","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722200000,"log":"[2026-02-17T14:05:17Z INFO actix_web::middleware::logger] GET /api/v1/logs 201 120ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-oqxhvlcdiw","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722201000,"log":"[2026-02-17T13:43:21Z INFO actix_web::middleware::logger] GET /api/_bulk 503 345ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-1lssthl47v","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722202000,"log":"[2026-02-17T13:03:29Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"default","kubernetes_pod_name":"prometheus-k8s-wjfrqc9zoi","kubernetes_container_name":"nginx","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722203000,"log":"[2026-02-17T13:24:31Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/1438894204623615200.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-bqd5p5030w","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722204000,"log":"[2026-02-17T13:51:43Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 404 155ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k2zot3oc2u","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722205000,"log":"[2026-02-17T14:59:48Z WARN zinc_enl::handler::http] TimeoutError: request timeout after 30s","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-yg9mfpl4x4","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722206000,"log":"[2026-02-17T14:31:13Z DEBUG sqlx::query] INSERT FROM users returned 17 rows in 3ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-k0e5bx4cf5","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722207000,"log":"[2026-02-17T14:02:03Z INFO actix_web::middleware::logger] DELETE /api/v1/metrics 403 27ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-gi5khpvfpg","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722208000,"log":"[2026-02-17T12:40:46Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-0l6wm0w17z","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722209000,"log":"[2026-02-17T13:27:26Z INFO zinc_enl::service::search::datafusion] search file: test_org/logs/events/2022/12/26/20/9429301338423500000.parquet, need add columns: [\"from\"]","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-w66o7xijiv","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722210000,"log":"[2026-02-17T14:25:27Z DEBUG sqlx::query] SELECT FROM streams returned 626 rows in 12ms","stream":"stderr","kubernetes_namespace_name":"ziox","kubernetes_pod_name":"ziox-querier-z2trri9s2s","kubernetes_container_name":"ziox","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722211000,"log":"[2026-02-17T14:06:16Z INFO actix_web::middleware::logger] POST /api/_bulk 404 414ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-hfpfnxjhzy","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722212000,"log":"[2026-02-17T14:23:20Z DEBUG sqlx::query] DELETE FROM alerts returned 289 rows in 35ms","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-n2fj0e52dk","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722213000,"log":"[2026-02-17T14:03:28Z INFO actix_web::middleware::logger] POST /api/v1/metrics 200 296ms","stream":"stderr","kubernetes_namespace_name":"monitoring","kubernetes_pod_name":"ziox-ingester-oor9dgpo9e","kubernetes_container_name":"prometheus","kubernetes_host":"ip-10-2-15-197.us-east-2.compute.internal"},{"_timestamp":1771331722214000,"log":"[2026-02-17T13:08:20Z INFO aws_sdk_s3::client] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-yn09oquzbh","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-56-159.us-east-2.compute.internal"},{"_timestamp":1771331722215000,"log":"[2026-02-17T14:23:56Z DEBUG sqlx::query] MERGE FROM dashboards returned 492 rows in 4ms","stream":"stderr","kubernetes_namespace_name":"ingress-nginx","kubernetes_pod_name":"coredns-khyjvjte5e","kubernetes_container_name":"sidecar","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"},{"_timestamp":1771331722216000,"log":"[2026-02-17T12:49:55Z INFO aws_config::meta::credentials] provide_credentials; provider=default_chain","stream":"stderr","kubernetes_namespace_name":"kube-system","kubernetes_pod_name":"nginx-ingress-v3qutvgwmd","kubernetes_container_name":"coredns","kubernetes_host":"ip-10-2-50-35.us-east-2.compute.internal"}] |