jianmu
/
openobserve
mirror of https://github.com/openobserve/openobserve
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: upgrade rust to 2025-05-20 (#7316) 

### **PR Type**
Enhancement


___

### **Description**
- Use `if let ... && let ...` patterns

- Inline named args in `format!` calls

- Simplify visitor `visit` calls with `_ =`

- Replace string interpolation in logging macros


___



### **Changes walkthrough** 📝
<table><thead><tr><th></th><th align="left">Relevant
files</th></tr></thead><tbody><tr><td><strong>Enhancement</strong></td><td><details><summary>5
files</summary><table>
<tr>
<td><strong>sql.rs</strong><dd><code>Refactor SQL visitor and limit
logic</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-d9ccf393c53794b0ef2816eef12742c26e23d2788cde27780af7570662aea446">+249/-255</a></td>

</tr>

<tr>
<td><strong>mysql.rs</strong><dd><code>Refactor time_range early-return
patterns</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-06d02c738c6a4dc22ff022c16befef8d46aece6cfaa88bb4a5d3d5f92c2e50a1">+51/-51</a>&nbsp;
</td>

</tr>

<tr>
<td><strong>sql.rs</strong><dd><code>Expand UDF list and simplify
visitors</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-a0a09c66a20d878ec7683ae222c919fbe092b8001871ade7f455b89573ce0911">+43/-39</a>&nbsp;
</td>

</tr>

<tr>
<td><strong>mod.rs</strong><dd><code>Refactor error mapping with
`std::io::Error::other`</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
</dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-8ce79cea5617c3a661e5b64058ad3846abf8652bb44fbb88dfb9117ffcc577fd">+20/-36</a>&nbsp;
</td>

</tr>

<tr>
<td><strong>engine.rs</strong><dd><code>Simplify unary ops and format
messages</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-a98ac5b55a4c9440c09c4e2dfa3b6f7f24f9e58fcb6bf8b10577a0b4f123ed41">+49/-63</a>&nbsp;
</td>

</tr>

</table></details></td></tr><tr><td><strong>Formatting</strong></td><td><details><summary>4
files</summary><table>
<tr>
<td><strong>mysql.rs</strong><dd><code>Inline format! and logging
braces</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-412dc407af348f4b54b7d98de6a928876b0333ea501792498a0568850f83d7b7">+59/-76</a>&nbsp;
</td>

</tr>

<tr>
<td><strong>nats.rs</strong><dd><code>Simplify error mapping and
formatting</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2d6809ea34ea04477afb9a6f8287c93cef488b45ef8f0ebeec2241571b93994c">+49/-54</a>&nbsp;
</td>

</tr>

<tr>
<td><strong>sqlite.rs</strong><dd><code>Use brace shorthand in SQL
strings</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-85f17b22460902614e5c28df8966ea3e7073325bd8dd43fa396ed9268b6b4b06">+45/-56</a>&nbsp;
</td>

</tr>

<tr>
<td><strong>action.rs</strong><dd><code>Use brace shorthand in Display
impls</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; </dd></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-fc87c75a7df9539d7e7ee6184dfab02d4a8932323e81f95d57c05363aa44752d">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>
</table></details></td></tr><tr><td><strong>Additional
files</strong></td><td><details><summary>101 files</summary><table>
<tr>
  <td><strong>Cargo.toml</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2e9d962a08321605940b5a657135052fbcef87b5e360662bb527c96d9a615542">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>clippy.toml</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f3dbfb2563c3490394f44c26b51837018e79a79e75fd31e89b19e943a38317ea">+1/-0</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>rust-toolchain.toml</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2b1bde2cf3a858b7bf7424cb8bcbf01f35b94dc80b925d9432cbab3319ca9b4e">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>cli.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-cb1eb3e74723d891353ccf1e2e5e0d246d2e78a5e69dbe4c73006ea8412d65e0">+8/-8</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>http.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-790b7780a51e8eecd4754b623af8d15ad87baaaf29455ba646e2e410f83f5498">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>load.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-bfc7e9b23a8fa100bcad8490d697a3090f139440fcd2ce02fb539c1791100d50">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>export.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-b9ca11eac019064e747b0c75157c280c8e8fa6d75edb83ca9ae46777e9668cc0">+2/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>import.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-e4415a147f0fe6e557c9fee2a5cad9058a46c30cfb79fe131d13802c0c8bb804">+2/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>etcd.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-37c2c2803d1c12ebb4ef9299bc79c9e6740fc5f19f7f4d84e7cfbb610b9561a9">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-d15434a27e2b8406811f5a707b605203b603dad4a262c0642326be860bd2b568">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>nats.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-901773980aa255942e7865b977fe63811f31511ac40bc512d6bbb14dbb54bece">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>alert_folders.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-6ed0a38075313ced0916343ee6e6b86ab639f8ed59e4968df395aa30d72bd300">+3/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-1b8694fad31c8249afeb419d92a9facbd170289db2c89100552883df876bb275">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>wal.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-4928ae81597b28facdc3644f786ac14c60b0a406e1c7b8eac7232fbafbcbb6e3">+5/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>loki.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-121c166af5eab8b9cb83716afa7b65fa7856c006c809b3d910ced184c52ec587">+8/-8</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>telemetry.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-3ff026b46d9431e0657828a3bde4ab118fec006b353c2abc80be4b74633c98b4">+2/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>auth.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-9bc2b1841504a435f66ad094f72ddbe7b7fa75e5e1557b8adbfe36f7f0592e1b">+39/-43</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>auth_tests.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2688cbdc0b009b3f0dd829c69ab86c6a7d0778ab3e65e916a410a76aa6ae6480">+1/-0</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>functions.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f4df68dbcf485d6b98c0e673c12a19be2edab416e5fc22c7d08396d375cd9126">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>http.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-3b50a69c1ec2f63f8eebb5f9880e17826463a586768aed3e3e15242833609d6b">+2/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>jwt.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-ad8a7b809caff90b4c49885a9b5d49ae03ca3fe00bef37a72d61b08b59df449c">+5/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>redirect_response.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f0415572330f30fceb16f230517e07f2c186e834cb8ca634795e5f92c1c57717">+3/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>stream.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2a60b2ae71e0a5c9dcceb588f262738f99df65fc2cb5b7002f2c37f87639e3b6">+2/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>websocket.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-6ff41099eded8eca4a0e579142d7af4429efb83350917780e0d3ae7dac13847a">+16/-17</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>ider.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-6ea42b8eab8ba0708f93f9a8b44056d9a9e6ac42517ffba12f81420d6b0bd2b0">+1/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>alert.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-45c7eaf1a764ab52f3e014fe0555c03dc59d77c5ce974684cdee4ed90463a234">+2/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>destinations.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-e0ba33d95ef7f0eac13bdac7e750462907a0308f01c027ea206d058e282c977b">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>inverted_index.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-8bf9d9403f05f53f0c6fd5e6a6dbf0204ea80465b233567552cb7395d08d1c3c">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>components.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-e2d647c422ae05ac91abc683c7b11a3fb7da61ee30968ca6e84cdba40221bbb9">+2/-0</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>promql.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-0893731b3f0345378627894e3ea4a9f8e8735b38622b2b96ab55e81760d188db">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>ratelimit.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-fa7dc03fb526c09d5eb58021872d821fcd72eeb4d6a670224f5286012485bc3d">+1/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>search.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-373e83a31e5bbc82fba05d7ae274b0dbd624ed748272232cf7eb8b8c3b2f3afc">+22/-22</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>sql.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-845dcf22be9d370a278c081df83a72bb8dc4b729f95c201839787088c82b57e7">+39/-39</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>stream.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2438d04c3fe97bc5452e5b74a303163b8723d93455cda32c95f0525435ad46bd">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>timed_annotations.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f94b4d6d12d68e7054ad33601e666b08e4594586013dc34384fa21b1df5fe6e8">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>async_file.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-8275f369cab85b5e9a97ed11d236de3364850b6989698936f19e70a0144e6a0d">+19/-23</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>download_utils.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-990aff76afde0d775c850302bf6399998aee73da05c76e719b5fbad6e7ed4b27">+3/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>file.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-405c4fb06d186c2423f78d42f3d195b12961877b091feff032f7df500450a72a">+6/-9</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>flatten.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-d13aed536f8b484d0223d7158e45e3a51ea181846965f73be42dedae975afcc7">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>md5.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f4d317c045b3df075e418dd6a7cd51f275befd1f2d15622afcb41c20024a8ee5">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>prom_json_encoder.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-5c6b4c5bb9f3859855269fe2b06a05167469ebc2ac04b84fb45d8d92aba6b825">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>query_select_utils.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2396baca05d001b681c6cc302c2acc901d4dc1e7acf32dd1be6fa9a759578a46">+30/-30</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>record_batch_ext.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f84aa2842a164d6e85aa05a718159e87a325cac139c59e561f6f59313c6d7c5d">+1/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>schema.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2431bc157b9e51be2f16836a995b5ed52f41dded921a404bf2eebd506b49fdfb">+1/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>cgroup.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-167dbe688715e2a299b86aa662fd5170b68e54ec0f6a04e0ebfa0492b3ec6b00">+11/-9</a>&nbsp;
&nbsp; </td>

</tr>

<tr>
  <td><strong>net.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f95051aabe608a9348aecbcc7f6866576f683f0e72cb4c84f7ed3e06c7e78b67">+7/-8</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>time.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-7b7e5972c321c99c5147af56a71afc1136a7c75844b8442e207de60bc86b7ddf">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-9a4dddd253d61f35b8d61d372477e24a4523f419aad2a63b226b19a584b8624f">+2/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>event.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-689769712cbd6bd02beba078c1e1dfc188e2be39ab19a0e232b8bd12bc7f1e78">+11/-11</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>ingest.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-27ffe683f384ff836c6aac746240d755bdf17f3c6e0a65d8ec1a1ff82aaf7d5c">+7/-12</a>&nbsp;
&nbsp; </td>

</tr>

<tr>
  <td><strong>jwt.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-232cab74f4cbf135af424744272e17732fbda8ac005d33c0acb4597e65479d02">+61/-62</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>validator.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f4d4b51574b378a8c372345ea48875834a4eadacfac33fa4d0ceca1e0ef96b35">+17/-18</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>dashboards.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-250eeebeec9d4ac0ac271466ed83cd33fea3f121080491b340dc95d9af287259">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>action.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-616df3014c553f763ea57e925f1bff7bb47f9467c632940d93a45b7b1e4c3a07">+5/-6</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>deprecated.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-3a5d80eac45bff881716e2cf099f90130659576eca017ad238a61781d3341b7e">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-e67f49865e1cd36775282c55ba0c0579a52ce7542f9d8dde450033f0c66a9f4c">+2/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>ingest.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-24f250254ecfdbe004af9dc0ea23f8a2c0105ae0898690ed3e565089e5d8eef9">+2/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>loki.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-34691872632b619be8aadf255104f0e004fcf701dba98f88e231de5966eed099">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>settings.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-1e807254f2faf8fd09b802f71b23412f203eed289033f96bfc62b15dd997db8c">+5/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>around.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-93b3688de26f8f7c9dc0389763e9999a254bcded15b6ae8c8349f18aec83bd4f">+9/-9</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-96630a0dbb6dd7b2b0248fa325c7c7041ce31340c32826675a4a70772e9d29f6">+19/-20</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>multi_streams.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-c3233cdf1e68cb2455f91457719b9c2f477e735ee16546fdfe111e76768bd054">+17/-18</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>search_inspector.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f818db4ca10b411c3559cc2a2659c49a59a45ecde0e4a22f9ef023da0bf4a009">+27/-29</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>search_job.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-d848b311eae318d198558c29a007c8973afb5f8867a31ec056f4e037dcd1b854">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-095d70f65b6e69bf7de0180fcbcddcd7c89c4cf00095d7b06f039a07e73916bb">+2/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-c10dd7b4ea20ffe5840c93a6ae4298aec0c32c8ba1b114192fda6c2bac1c41d1">+7/-7</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-ca09c9c9b020049bdbc3382c466bfcca80daac92e98760d9a9a76172abf34049">+10/-12</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-619eafba0f0509d905b021f4a16659010aca23c62cffd9cd3e6c2bbe204ce200">+14/-18</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-64694c886c809b6efe449e6eba9fbb33d31ce5a3c49cea1a30006fc1d9522665">+4/-7</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-56897e42f9436a640bc70fddc25686485ba8ae1223635b32d252a0333309f748">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>session.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-6a1a862158668be26ad1575d7050eb9139bca3e469db56b32641b4c46d0f8144">+6/-7</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>encoder.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-1825b00f221539b371b9fe8aa89f9cbc1cfb94f3b731f053e64c51fb5c5934cd">+1/-2</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-02556e3f515441b5d2b6bbea7829df488beac03c4cc41309a6638ae3c0e0aec6">+2/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-66207bd699343342402f88cdd331f4c549783754e4921d0c8d43e026e2d8e575">+12/-12</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f8e74805561fa288a434e6e5ca8fb2c9068fb7f54a5a21f0a9bbba5f73249a5f">+21/-23</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>etcd.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-33bb9257c1a1b9162c8679191b8d8b75c0b7395f72291ce73d77247ff924a66e">+20/-22</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>postgres.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-1554e69f6d6c29812af88f0dcf093240b24ff85f9e292322c71cb19b0dda808d">+30/-45</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-90c725d0e77a6e7d4bcc130cf5783848a5cf0efdb304b351cd63cacf8775fc36">+1/-0</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-3df833bc826eb12e0819620d1fe316ef1ac352d293c11e57cc3df0da3c031e93">+10/-10</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>postgres.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-bb24a30c9c3744eb31fb49e4271a9651d1f9a94a87455ac0074364b879af9651">+27/-24</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>sqlite.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-85ff00530831fdaa4e13c05cb649f2e20479cce04a4b65ef78d131cdb89b6c71">+36/-35</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-bff7752b2aae0e9cef3ece1b9ddf17b9aa2f52f81fa777e6249039580aefbd69">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>nats.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-b05729b03dff9a5e29eaf536d0f90e94a0c27a27d1a3a41b52f25dc6dc380138">+8/-23</a>&nbsp;
&nbsp; </td>

</tr>

<tr>
  <td><strong>mysql.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-a275fb17137e6e7cd3dbc6432164136b20d1c2d4757eadaadd0b4a983473865a">+14/-35</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>postgres.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-a281dfd3124defa76bf1100c7e4993def0cb3049cf1ad0b1c513ea0afe522d08">+2/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>sqlite.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-07751d2c8c454417335fa9dd06507099d8b1eaea9b04af99857db6d9c619e69f">+2/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-386fd4fad1e48ab0a90516cab1947b1888b270b2fe912f30d2255cc38e6c519b">+15/-12</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>accounts.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-7ff63b9d4046282262c347d3458395eae5b6853cb004e7349a37ec2274465fe0">+4/-4</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>mod.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-71f24f3312c19e01c4eedcd6a720893e01e64d176d2afbe1e7c6171a23059716">+4/-6</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>remote.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-fea1d32f99821f1913607362236bd9e5b65cb2a5f8ce2c11a9a4956ac646b3d1">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>dashboards.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-edd90c6ec720f676311aa31a635f954d559e63582f0b4c39cb9206bfb095737b">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>

<td><strong>m20250109_092400_recreate_tables_with_ksuids.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-c16140f94722b17bff8753f09fa80ee0bbabc2b2f2f24b8800b459ced12fe0e4">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>

<td><strong>m20250113_144600_create_unique_folder_name_idx.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-1f6dc440bcd651434f5bd6a06a4ec7d3bd88760754edeeb8d38e861208a56cbd">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>m20250125_132500_populate_templates_table.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-fa1e504b43669656f6fd652e221e954f3f3da7fd2892d524a0e8772b6d738107">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>

<td><strong>m20250320_000001_remove_alert_name_unique_constraint.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-f4094b2945a877db5709045deafdd2883ff31fd671d0782da058af40e522d559">+10/-17</a>&nbsp;
</td>

</tr>

<tr>

<td><strong>m20250611_000003_populate_reports_scheduled_jobs.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-6567d3d0361c1a184ba7877795a8c3c4f73ea9807b3d4c13efe621449dd3f80f">+8/-12</a>&nbsp;
&nbsp; </td>

</tr>

<tr>
  <td><strong>search_jobs.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-b78a126c6e978d43e798fa2bcb56f7bb0132c107c716d4a9891c58b3c5a33ff9">+7/-8</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>timed_annotations.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-4bc910d02f5ddc598e394d06172debe456c7db655cc16fef53eab55005880378">+14/-21</a>&nbsp;
</td>

</tr>

<tr>
  <td><strong>partition.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-441e26dda30cadfa3cb1efbd9569aec96af8fdc779e4916814fb5f9477db3f6c">+1/-1</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>writer.rs</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-d1a9b1542cc2758afe4fa67abd454dd818718f4e1c7570303519af5a58d6301f">+5/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td><strong>Additional files not shown</strong></td>
<td><a
href="https://github.com/openobserve/openobserve/pull/7316/files#diff-2f328e4cd8dbe3ad193e49d92bcf045f47a6b72b1e9487d366f6b8288589b4ca"></a></td>

</tr>
</table></details></td></tr></tr></tbody></table>

___

> <details> <summary> Need help?</summary><li>Type <code>/help how to
...</code> in the comments thread for any questions about PR-Agent
usage.</li><li>Check out the <a
href="https://qodo-merge-docs.qodo.ai/usage-guide/">documentation</a>
for more information.</li></details>
This commit is contained in:
Huaijin 2025-07-01 21:25:45 +08:00 committed by GitHub
parent c8bdada9da
commit b141504cf8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
262 changed files with 5315 additions and 2786 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
Cargo.lock generated
View File

@ -4069,11 +4069,12 @@ dependencies = [
[[package]]
name = "flate2"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d"
dependencies = [
"crc32fast",
"libz-rs-sys",
"libz-sys",
"miniz_oxide",
]
@ -5425,9 +5426,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.172"
version = "0.2.174"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
[[package]]
name = "libflate"
@ -5487,6 +5488,15 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "libz-rs-sys"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221"
dependencies = [
"zlib-rs",
]
[[package]]
name = "libz-sys"
version = "1.1.20"
@ -5764,9 +5774,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.8.8"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
]
@ -6399,7 +6409,7 @@ dependencies = [
"vrl",
"wal",
"x509-parser",
"zip 2.5.0",
"zip 3.0.0",
"zstd",
]
@ -10391,9 +10401,9 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "unindent"
version = "0.2.3"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
[[package]]
name = "universal-hash"
@ -11512,16 +11522,15 @@ dependencies = [
[[package]]
name = "zip"
version = "2.5.0"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c03817464f64e23f6f37574b4fdc8cf65925b5bfd2b0f2aedf959791941f88"
checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308"
dependencies = [
"aes",
"arbitrary",
"bzip2 0.5.2",
"constant_time_eq",
"crc32fast",
"crossbeam-utils",
"deflate64",
"flate2",
"getrandom 0.3.2",
@ -11538,6 +11547,12 @@ dependencies = [
"zstd",
]
[[package]]
name = "zlib-rs"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a"
[[package]]
name = "zopfli"
version = "0.8.1"

2
Cargo.toml
View File

@ -390,4 +390,4 @@ utoipa-swagger-ui = { version = "4", features = ["actix-web"] }
vector-enrichment = { version = "0.1.0", package = "enrichment", git = "https://github.com/openobserve/vector", rev = "063cabbbf4bc6f75794fa0ccd3b0bd5c074f0e35" }
vrl = { version = "0.22", features = ["value", "compiler", "test"] }
zstd = "0.13"
zip = "2.5.0"
zip = "3.0.0"

1
clippy.toml Normal file
View File

@ -0,0 +1 @@
large-error-threshold = 200

2
rust-toolchain.toml
View File

@ -1,3 +1,3 @@
[toolchain]
channel = "nightly-2025-03-02"
channel = "nightly-2025-05-20"
components = ["rustfmt", "clippy", "llvm-tools"]

16
src/cli/basic/cli.rs
View File

@ -259,7 +259,7 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
match command.get_one::<String>("path") {
Some(path) => {
set_permission(path, 0o777)?;
println!("init dir {} successfully", path);
println!("init dir {path} successfully");
}
None => {
return Err(anyhow::anyhow!("please set data path"));
@ -372,7 +372,7 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
Some(to) => to.to_string(),
None => "".to_string(),
};
println!("Running migration file_list from {} to {}", from, to);
println!("Running migration file_list from {from} to {to}");
migration::file_list::run(&from, &to).await?;
}
"migrate-meta" => {
@ -384,7 +384,7 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
Some(to) => to.to_string(),
None => "".to_string(),
};
println!("Running migration metadata from {} to {}", from, to);
println!("Running migration metadata from {from} to {to}");
migration::meta::run(&from, &to).await?
}
"migrate-dashboards" => {
@ -404,10 +404,10 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
let file = command.get_one::<String>("file").unwrap();
match file_list::delete_parquet_file(&account, file, true).await {
Ok(_) => {
println!("delete parquet file {} successfully", file);
println!("delete parquet file {file} successfully");
}
Err(e) => {
println!("delete parquet file {} failed, error: {}", file, e);
println!("delete parquet file {file} failed, error: {e}");
}
}
}
@ -511,13 +511,13 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
}
"parse-id" => {
let id = command.get_one::<String>("id").unwrap();
println!("id: {}", id);
println!("id: {id}");
let id = id.parse::<i64>().unwrap();
let ts = config::ider::to_timestamp_millis(id);
println!("timestamp: {}", ts);
println!("timestamp: {ts}");
let t = chrono::Utc.timestamp_nanos(ts * 1_000_000);
let td = t.format("%Y-%m-%dT%H:%M:%SZ").to_string();
println!("datetimes: {}", td);
println!("datetimes: {td}");
}
"consistent-hash" => {
let files = command

2
src/cli/basic/http.rs
View File

@ -39,7 +39,7 @@ pub async fn query(
query,
..Default::default()
};
let url = format!("/api/{}/_search", org);
let url = format!("/api/{org}/_search");
let body = serde_json::to_vec(&search_req)?;
let response = request(&url, Some(body), reqwest::Method::POST).await?;
let Some(body) = response else {

8
src/cli/basic/load.rs
View File

@ -26,8 +26,8 @@ pub async fn load_file_list_from_s3(
"prefix is required, eg: files/default/logs/default/2025/"
));
}
println!("account: {}", account);
println!("prefix: {}", prefix);
println!("account: {account}");
println!("prefix: {prefix}");
println!("Listing files...");
let files = infra::storage::list(account, prefix).await?;
@ -35,13 +35,13 @@ pub async fn load_file_list_from_s3(
println!("Processing files...");
for (i, file) in files.iter().enumerate() {
println!("{} {}", i, file);
println!("{i} {file}");
let (stream_key, date, file_name) = parse_file_key_columns(file)?;
let (org, stream) = stream_key.split_once('/').unwrap();
let file_meta = infra::storage::get_file_meta(account, file).await?;
if insert {
if let Err(e) = infra::file_list::add(account, file, &file_meta).await {
println!("insert to db with file {} error: {}", file, e);
println!("insert to db with file {file} error: {e}");
}
} else {
println!(

7
src/cli/data/export.rs
View File

@ -48,10 +48,7 @@ impl Context for Export {
.as_ref()
.and_then(|event_type| get_search_event_context_from_request(event_type, &query_map));
let query = search::Query {
sql: format!(
"select * from {} ORDER BY {} ASC",
table, TIMESTAMP_COL_NAME
),
sql: format!("select * from {table} ORDER BY {TIMESTAMP_COL_NAME} ASC"),
from: 0,
size: cfg.limit.query_default_limit,
quick_mode: false,
@ -90,7 +87,7 @@ impl Context for Export {
Ok(true)
}
Err(e) => {
eprintln!("search error: {:?}", e);
eprintln!("search error: {e:?}");
Ok(false)
}
}

4
src/cli/data/import.rs
View File

@ -48,10 +48,10 @@ async fn read_files_in_directory(c: Cli, dir_path: &str) -> Result<bool, anyhow:
metadata: None,
};
if let Err(e) = crate::service::ingestion::ingestion_service::ingest(req).await {
eprintln!("insert data fail {:?}: {:?}", path, e);
eprintln!("insert data fail {path:?}: {e:?}");
return Ok(false);
} else {
println!("insert data success: {:?}", path);
println!("insert data success: {path:?}");
}
} else if path.is_dir()
&& !read_files_in_directory(c.clone(), &path.to_string_lossy()).await?

8
src/common/infra/cluster/etcd.rs
View File

@ -194,7 +194,7 @@ async fn register() -> Result<()> {
let opt = PutOptions::new().with_lease(id);
if let Err(e) = client.put(key, val, Some(opt)).await {
dist_lock::unlock(&locker).await?;
return Err(Error::Message(format!("register node error: {}", e)));
return Err(Error::Message(format!("register node error: {e}")));
}
// 7. register ok, release lock
@ -273,7 +273,7 @@ pub(crate) async fn set_status(status: NodeStatus, new_lease_id: bool) -> Result
let opt = PutOptions::new().with_lease(unsafe { LOCAL_NODE_KEY_LEASE_ID });
let mut client = etcd::get_etcd_client().await.clone();
if let Err(e) = client.put(key, val, Some(opt)).await {
return Err(Error::Message(format!("online node error: {}", e)));
return Err(Error::Message(format!("online node error: {e}")));
}
Ok(())
@ -284,7 +284,7 @@ pub(crate) async fn leave() -> Result<()> {
let key = format!("{}nodes/{}", get_config().etcd.prefix, LOCAL_NODE.uuid);
let mut client = etcd::get_etcd_client().await.clone();
if let Err(e) = client.delete(key, None).await {
return Err(Error::Message(format!("leave node error: {}", e)));
return Err(Error::Message(format!("leave node error: {e}")));
}
Ok(())
@ -296,7 +296,7 @@ pub(crate) async fn update_local_node(node: &Node) -> Result<()> {
let val = json::to_string(&node).unwrap();
let mut client = etcd::get_etcd_client().await.clone();
if let Err(e) = client.put(key, val, Some(opt)).await {
return Err(Error::Message(format!("update node error: {}", e)));
return Err(Error::Message(format!("update node error: {e}")));
}
Ok(())
}

8
src/common/infra/cluster/mod.rs
View File

@ -374,10 +374,10 @@ async fn watch_node_list() -> Result<()> {
log::info!("[CLUSTER] join {:?}", item_value);
item_value.broadcasted = true;
// check if the same node is already in the cluster
if let Some(node) = get_cached_node_by_name(&item_value.name).await {
if node.uuid.ne(&item_value.uuid) {
NODES.write().await.remove(&node.uuid);
}
if let Some(node) = get_cached_node_by_name(&item_value.name).await
&& node.uuid.ne(&item_value.uuid)
{
NODES.write().await.remove(&node.uuid);
}
if item_value.is_interactive_querier() {
add_node_to_consistent_hash(

8
src/common/infra/cluster/nats.rs
View File

@ -200,7 +200,7 @@ async fn register() -> Result<()> {
let client = get_coordinator().await;
if let Err(e) = client.put(&key, val.into(), NEED_WATCH, None).await {
dist_lock::unlock(&locker).await?;
return Err(Error::Message(format!("register node error: {}", e)));
return Err(Error::Message(format!("register node error: {e}")));
}
// 7. register ok, release lock
@ -268,7 +268,7 @@ pub(crate) async fn set_status(status: NodeStatus) -> Result<()> {
let key = format!("/nodes/{}", LOCAL_NODE.uuid);
let client = get_coordinator().await;
if let Err(e) = client.put(&key, val.into(), NEED_WATCH, None).await {
return Err(Error::Message(format!("online node error: {}", e)));
return Err(Error::Message(format!("online node error: {e}")));
}
Ok(())
@ -279,7 +279,7 @@ pub(crate) async fn leave() -> Result<()> {
let key = format!("/nodes/{}", LOCAL_NODE.uuid);
let client = get_coordinator().await;
if let Err(e) = client.delete(&key, false, NEED_WATCH, None).await {
return Err(Error::Message(format!("leave node error: {}", e)));
return Err(Error::Message(format!("leave node error: {e}")));
}
Ok(())
@ -290,7 +290,7 @@ pub(crate) async fn update_local_node(node: &Node) -> Result<()> {
let val = json::to_vec(&node).unwrap();
let client = get_coordinator().await;
if let Err(e) = client.put(&key, val.into(), NEED_WATCH, None).await {
return Err(Error::Message(format!("update node error: {}", e)));
return Err(Error::Message(format!("update node error: {e}")));
}
Ok(())
}

6
src/common/infra/ofga/migrations/alert_folders.rs
View File

@ -53,7 +53,7 @@ pub async fn migrate_alert_folders<C: ConnectionTrait>(db: &C) -> Result<(), any
);
let org_id = folder.org;
let folder_id = folder.folder_id;
let obj_str = format!("{}:{}", alert_folders_ofga_type, folder_id);
let obj_str = format!("{alert_folders_ofga_type}:{folder_id}");
authorizer::authz::set_ownership(&org_id, &obj_str, "", "").await;
}
}
@ -132,7 +132,7 @@ pub async fn migrate_alert_folders<C: ConnectionTrait>(db: &C) -> Result<(), any
};
if alert_name.starts_with("_all_") {
let mut alert = alert.clone();
alert.object = format!("{}:{}", alert_folders_ofga_type, alert_name);
alert.object = format!("{alert_folders_ofga_type}:{alert_name}");
add_roles.push(alert);
continue;
}
@ -150,7 +150,7 @@ pub async fn migrate_alert_folders<C: ConnectionTrait>(db: &C) -> Result<(), any
for db_alert in db_alerts {
let mut alert = alert.clone();
alert.object = format!("{}:{}", alerts_ofga_type, db_alert.id);
alert.object = format!("{alerts_ofga_type}:{}", db_alert.id);
add_roles.push(alert);
}
}

8
src/common/infra/ofga/mod.rs
View File

@ -199,8 +199,8 @@ pub async fn init() -> Result<(), anyhow::Error> {
org_name,
&mut tuples,
OFGA_MODELS
.iter()
.map(|(_, fga_entity)| fga_entity.key)
.values()
.map(|fga_entity| fga_entity.key)
.collect(),
NON_OWNING_ORG.to_vec(),
)
@ -213,8 +213,8 @@ pub async fn init() -> Result<(), anyhow::Error> {
DEFAULT_ORG,
&mut tuples,
OFGA_MODELS
.iter()
.map(|(_, fga_entity)| fga_entity.key)
.values()
.map(|fga_entity| fga_entity.key)
.collect(),
NON_OWNING_ORG.to_vec(),
)

2
src/common/infra/wal.rs
View File

@ -250,7 +250,7 @@ impl RwFile {
dir_path = dir_path.replace(file_list_prefix, "/file_list/");
}
let id = ider::generate();
let file_name = format!("{thread_id}/{key}/{id}{}", FILE_EXT_JSON);
let file_name = format!("{thread_id}/{key}/{id}{FILE_EXT_JSON}");
let file_path = format!("{dir_path}{file_name}");
create_dir_all(Path::new(&file_path).parent().unwrap())
.await

16
src/common/meta/loki.rs
View File

@ -118,31 +118,31 @@ impl From<LokiError> for actix_web::HttpResponse {
// Client errors (400)
LokiError::InvalidTimestamp { message } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("invalid timestamp: {}", message)),
.body(format!("invalid timestamp: {message}")),
LokiError::InvalidLabels { message } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("invalid labels: {}", message)),
.body(format!("invalid labels: {message}")),
LokiError::EmptyStream => HttpResponse::BadRequest()
.content_type("text/plain")
.body("empty stream data"),
LokiError::UnsupportedContentType { content_type } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("unsupported content type: {}", content_type)),
.body(format!("unsupported content type: {content_type}")),
LokiError::UnsupportedContentEncoding { encoding } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("unsupported content encoding: {}", encoding)),
.body(format!("unsupported content encoding: {encoding}")),
LokiError::ProtobufDecode { source } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("failed to decode protobuf: {}", source)),
.body(format!("failed to decode protobuf: {source}")),
LokiError::SnappyDecompression { source } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("failed to decompress snappy: {}", source)),
.body(format!("failed to decompress snappy: {source}")),
LokiError::JsonParse { source } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("failed to parse JSON: {}", source)),
.body(format!("failed to parse JSON: {source}")),
LokiError::GzipDecompression { source } => HttpResponse::BadRequest()
.content_type("text/plain")
.body(format!("failed to decompress gzip: {}", source)),
.body(format!("failed to decompress gzip: {source}")),
// Server errors (500)
LokiError::Ingestion { .. } => HttpResponse::InternalServerError()

82
src/common/utils/auth.rs
View File

@ -211,12 +211,12 @@ impl FromRequest for UserEmail {
type Future = Ready<Result<Self, Error>>;
fn from_request(req: &HttpRequest, _: &mut Payload) -> Self::Future {
if let Some(auth_header) = req.headers().get("user_id") {
if let Ok(user_str) = auth_header.to_str() {
return ready(Ok(UserEmail {
user_id: user_str.to_lowercase(),
}));
}
if let Some(auth_header) = req.headers().get("user_id")
&& let Ok(user_str) = auth_header.to_str()
{
return ready(Ok(UserEmail {
user_id: user_str.to_lowercase(),
}));
}
ready(Err(actix_web::error::ErrorUnauthorized("No user found")))
}
@ -272,17 +272,17 @@ impl FromRequest for AuthExtractor {
// This is case for ingestion endpoints where we need to check
// permissions on the stream
if method.eq("POST") && INGESTION_EP.contains(&path_columns[url_len - 1]) {
if let Some(auth_header) = req.headers().get("Authorization") {
if let Ok(auth_str) = auth_header.to_str() {
return ready(Ok(AuthExtractor {
auth: auth_str.to_owned(),
method,
o2_type: format!("stream:{org_id}"),
org_id,
bypass_check: true,
parent_id: folder,
}));
}
if let Some(auth_header) = req.headers().get("Authorization")
&& let Ok(auth_str) = auth_header.to_str()
{
return ready(Ok(AuthExtractor {
auth: auth_str.to_owned(),
method,
o2_type: format!("stream:{org_id}"),
org_id,
bypass_check: true,
parent_id: folder,
}));
}
return ready(Err(actix_web::error::ErrorUnauthorized(
"Unauthorized Access",
@ -641,23 +641,22 @@ impl FromRequest for AuthExtractor {
};
// Check if the ws request is using internal grpc token
if method.eq("GET") && path.contains("/ws") {
if let Some(auth_header) = req.headers().get("Authorization") {
if auth_header
.to_str()
.unwrap()
.eq(&get_config().grpc.internal_grpc_token)
{
return ready(Ok(AuthExtractor {
auth: auth_header.to_str().unwrap().to_string(),
method,
o2_type: format!("stream:{org_id}"),
org_id,
bypass_check: true,
parent_id: folder,
}));
}
}
if method.eq("GET")
&& path.contains("/ws")
&& let Some(auth_header) = req.headers().get("Authorization")
&& auth_header
.to_str()
.unwrap()
.eq(&get_config().grpc.internal_grpc_token)
{
return ready(Ok(AuthExtractor {
auth: auth_header.to_str().unwrap().to_string(),
method,
o2_type: format!("stream:{org_id}"),
org_id,
bypass_check: true,
parent_id: folder,
}));
}
let auth_str = extract_auth_str(req);
@ -705,7 +704,7 @@ impl FromRequest for AuthExtractor {
.as_str(),
)
} else {
object_type.replace("stream:", format!("{}:", stream_type).as_str())
object_type.replace("stream:", format!("{stream_type}:").as_str())
}
}
None => object_type,
@ -795,7 +794,7 @@ impl FromRequest for AuthExtractor {
if access_token.starts_with("Basic") || access_token.starts_with("Bearer") {
access_token
} else {
format!("Bearer {}", access_token)
format!("Bearer {access_token}")
}
} else if let Some(auth_header) = req.headers().get("Authorization") {
if let Ok(auth_str) = auth_header.to_str() {
@ -856,7 +855,7 @@ pub fn extract_auth_str(req: &HttpRequest) -> String {
None => access_token,
}
} else {
format!("Bearer {}", access_token)
format!("Bearer {access_token}")
}
} else if let Some(cookie) = req.cookie("auth_ext") {
let val = config::utils::base64::decode_raw(cookie.value()).unwrap_or_default();
@ -897,13 +896,10 @@ pub fn generate_presigned_url(
let stage2 = get_hash(&format!("{}{}", &stage1, time), salt);
let stage3 = get_hash(&format!("{}{}", &stage2, exp_in), salt);
let user_pass = format!("{}:{}", username, stage3);
let user_pass = format!("{username}:{stage3}");
let auth = base64::engine::general_purpose::STANDARD.encode(user_pass);
format!(
"{}/auth/login?request_time={}&exp_in={}&auth={}",
base_url, time, exp_in, auth
)
format!("{base_url}/auth/login?request_time={time}&exp_in={exp_in}&auth={auth}")
}
#[cfg(not(feature = "enterprise"))]
@ -1002,7 +998,7 @@ pub async fn extract_auth_expiry_and_user_id(
}
};
let exp = decode(&stripped_bearer_token).await;
let bearer_full_token = format!("Bearer {}", stripped_bearer_token);
let bearer_full_token = format!("Bearer {stripped_bearer_token}");
let user_id = get_user_email_from_auth_str(&bearer_full_token).await;
return (exp, user_id);
}

3
src/common/utils/auth_tests.rs
View File

@ -3152,6 +3152,9 @@ mod tests {
file_list_dump_min_hour: Default::default(),
file_list_dump_debug_check: Default::default(),
use_stream_settings_for_partitions_enabled: Default::default(),
dashboard_placeholder: Default::default(),
search_inspector_enabled: bool::default(),
utf8_view_enabled: bool::default(),
},
limit: config::Limit {
cpu_num: usize::default(),

2
src/common/utils/functions.rs
View File

@ -26,7 +26,7 @@ use crate::common::{
};
pub async fn get_all_transform_keys(org_id: &str) -> Vec<String> {
let org_key = &format!("{}/", org_id);
let org_key = &format!("{org_id}/");
crate::common::infra::config::QUERY_FUNCTIONS
.clone()

5
src/common/utils/http.rs
View File

@ -14,7 +14,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
io::{Error, ErrorKind},
io::Error,
net::{AddrParseError, IpAddr, SocketAddr},
};
@ -52,8 +52,7 @@ pub(crate) fn get_search_type_from_request(
Some(s) => match SearchEventType::try_from(s.as_str()) {
Ok(search_type) => Some(search_type),
_ => {
return Err(Error::new(
ErrorKind::Other,
return Err(Error::other(
"'event_type' query param with value 'ui', 'dashboards', 'reports', 'alerts' , 'rum' or 'values' allowed",
));
}

10
src/common/utils/jwt.rs
View File

@ -75,11 +75,11 @@ pub(crate) async fn verify_decode_token(
)?;
let mut final_claims = HashMap::new();
let claims = decoded_token.clone().claims;
if let Some(federated_claims) = claims.get("federated_claims") {
if let Some(map) = federated_claims.as_object() {
for (key, value) in map.iter() {
final_claims.insert(key.to_string(), value.clone());
}
if let Some(federated_claims) = claims.get("federated_claims")
&& let Some(map) = federated_claims.as_object()
{
for (key, value) in map.iter() {
final_claims.insert(key.to_string(), value.clone());
}
};

6
src/common/utils/redirect_response.rs
View File

@ -41,11 +41,11 @@ impl RedirectResponse {
let query_string: String = self
.query_params
.iter()
.map(|(key, value)| format!("{}={}", key, value))
.map(|(key, value)| format!("{key}={value}"))
.collect::<Vec<String>>()
.join("&");
redirect_uri = format!("{}?{}", redirect_uri, query_string);
redirect_uri = format!("{redirect_uri}?{query_string}");
}
redirect_uri
@ -84,7 +84,7 @@ impl RedirectResponse {
impl fmt::Display for RedirectResponse {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let redirect_uri = self.build_full_redirect_uri();
write!(f, "Redirecting to {}", redirect_uri)
write!(f, "Redirecting to {redirect_uri}")
}
}

5
src/common/utils/stream.rs
View File

@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::io::{Error, ErrorKind};
use std::io::Error;
use actix_web::HttpResponse;
use arrow::array::{Int64Array, RecordBatch};
@ -29,8 +29,7 @@ use crate::service::users;
#[inline(always)]
pub fn stream_type_query_param_error() -> Result<HttpResponse, Error> {
Err(Error::new(
ErrorKind::Other,
Err(Error::other(
"only 'type' query param with value 'logs' or 'metrics' allowed",
))
}

33
src/common/utils/websocket.rs
View File

@ -84,22 +84,21 @@ impl VisitorMut for HistogramIntervalVisitorMut {
type Break = ();
fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow<Self::Break> {
if let Expr::Function(func) = expr {
if func.name.to_string().to_lowercase() == "histogram" {
if let FunctionArguments::List(list) = &mut func.args {
let mut args = list.args.iter();
// first is field
let _ = args.next();
// second is interval
if args.next().is_none() {
let interval_value = format!("{} seconds", self.interval);
list.args.push(sqlparser::ast::FunctionArg::Unnamed(
sqlparser::ast::FunctionArgExpr::Expr(Expr::Value(
sqlparser::ast::Value::SingleQuotedString(interval_value),
)),
));
}
}
if let Expr::Function(func) = expr
&& func.name.to_string().to_lowercase() == "histogram"
&& let FunctionArguments::List(list) = &mut func.args
{
let mut args = list.args.iter();
// first is field
let _ = args.next();
// second is interval
if args.next().is_none() {
let interval_value = format!("{} seconds", self.interval);
list.args.push(sqlparser::ast::FunctionArg::Unnamed(
sqlparser::ast::FunctionArgExpr::Expr(Expr::Value(
sqlparser::ast::Value::SingleQuotedString(interval_value),
)),
));
}
}
ControlFlow::Continue(())
@ -118,7 +117,7 @@ pub fn update_histogram_interval_in_query(
.unwrap();
let mut histogram_interval_visitor = HistogramIntervalVisitorMut::new(histogram_interval);
statement.visit(&mut histogram_interval_visitor);
let _ = statement.visit(&mut histogram_interval_visitor);
Ok(statement.to_string())
}

5
src/config/src/ider.rs
View File

@ -152,10 +152,7 @@ impl SnowflakeIdGenerator {
if now_millis == self.last_time_millis {
now_millis = biding_time_conditions(self.last_time_millis, self.epoch);
if now_millis == self.last_time_millis {
panic!(
"Clock is moving backwards. Rejecting requests until {}.",
now_millis
);
panic!("Clock is moving backwards. Rejecting requests until {now_millis}.");
}
}

8
src/config/src/meta/actions/action.rs
View File

@ -24,7 +24,7 @@ impl std::fmt::Display for ExecutionDetailsType {
ExecutionDetailsType::Repeat => "repeat",
ExecutionDetailsType::Service => "service",
};
write!(f, "{}", s)
write!(f, "{s}")
}
}
@ -79,7 +79,7 @@ impl Display for ActionStatus {
ActionStatus::Errored => "errored",
ActionStatus::Completed => "completed",
};
write!(f, "{}", s)
write!(f, "{s}")
}
}
@ -159,7 +159,7 @@ impl Display for ActionType {
ActionType::CronJob => "cronjob".to_string(),
ActionType::Service => "service".to_string(),
};
write!(f, "{}", str)
write!(f, "{str}")
}
}
impl TryFrom<&str> for ActionType {
@ -170,7 +170,7 @@ impl TryFrom<&str> for ActionType {
"job" => Ok(ActionType::Job),
"cronjob" => Ok(ActionType::CronJob),
"service" => Ok(ActionType::Service),
_ => Err(anyhow::anyhow!("Invalid action type: {}", s)),
_ => Err(anyhow::anyhow!("Invalid action type: {s}")),
}
}
}

2
src/config/src/meta/destinations.rs
View File

@ -52,7 +52,7 @@ impl std::fmt::Display for Module {
Module::Alert { .. } => "alert",
Module::Pipeline { .. } => "pipeline",
};
write!(f, "{}", s)
write!(f, "{s}")
}
}

2
src/config/src/meta/inverted_index.rs
View File

@ -72,7 +72,7 @@ impl std::fmt::Display for InvertedIndexOptimizeMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
InvertedIndexOptimizeMode::SimpleSelect(limit, ascend) => {
write!(f, "simple_select(limit: {}, ascend: {})", limit, ascend)
write!(f, "simple_select(limit: {limit}, ascend: {ascend})")
}
InvertedIndexOptimizeMode::SimpleCount => write!(f, "simple_count"),
InvertedIndexOptimizeMode::SimpleHistogram(min_value, bucket_width, num_buckets) => {

2
src/config/src/meta/pipeline/components.rs
View File

@ -26,6 +26,7 @@ use crate::meta::{
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "source_type")]
#[serde(rename_all = "snake_case")]
#[allow(clippy::large_enum_variant)]
pub enum PipelineSource {
Realtime(StreamParams),
Scheduled(DerivedStream),
@ -133,6 +134,7 @@ impl Edge {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "node_type")]
#[serde(rename_all = "snake_case")]
#[allow(clippy::large_enum_variant)]
pub enum NodeData {
RemoteStream(RemoteStreamParams),
Stream(StreamParams),

2
src/config/src/meta/promql.rs
View File

@ -241,7 +241,7 @@ impl From<&str> for Function {
"max" => Self::Max,
"last" => Self::Last,
"first" => Self::First,
_ => panic!("invalid downsampling function: {}", s),
_ => panic!("invalid downsampling function: {s}"),
}
}
}

5
src/config/src/meta/ratelimit.rs
View File

@ -153,8 +153,5 @@ pub fn get_resource_from_params(
api_group_operation: &str,
user_id: &str,
) -> String {
format!(
"{}:{}:{}:{}:{}:{}",
org, rule_type, user_role, api_group_name, api_group_operation, user_id
)
format!("{org}:{rule_type}:{user_role}:{api_group_name}:{api_group_operation}:{user_id}")
}

44
src/config/src/meta/search.rs
View File

@ -1358,32 +1358,32 @@ mod search_history_utils {
// Method to build the SQL query
pub fn build(self, search_stream_name: &str) -> String {
let mut query = format!("SELECT * FROM {} WHERE event='Search'", search_stream_name);
let mut query = format!("SELECT * FROM {search_stream_name} WHERE event='Search'");
if let Some(org_id) = self.org_id {
if !org_id.is_empty() {
query.push_str(&format!(" AND org_id = '{}'", org_id));
}
if let Some(org_id) = self.org_id
&& !org_id.is_empty()
{
query.push_str(&format!(" AND org_id = '{org_id}'"));
}
if let Some(stream_type) = self.stream_type {
if !stream_type.is_empty() {
query.push_str(&format!(" AND stream_type = '{}'", stream_type));
}
if let Some(stream_type) = self.stream_type
&& !stream_type.is_empty()
{
query.push_str(&format!(" AND stream_type = '{stream_type}'"));
}
if let Some(stream_name) = self.stream_name {
if !stream_name.is_empty() {
query.push_str(&format!(" AND stream_name = '{}'", stream_name));
}
if let Some(stream_name) = self.stream_name
&& !stream_name.is_empty()
{
query.push_str(&format!(" AND stream_name = '{stream_name}'"));
}
if let Some(user_email) = self.user_email {
if !user_email.is_empty() {
query.push_str(&format!(" AND user_email = '{}'", user_email));
}
if let Some(user_email) = self.user_email
&& !user_email.is_empty()
{
query.push_str(&format!(" AND user_email = '{user_email}'"));
}
if let Some(trace_id) = self.trace_id {
if !trace_id.is_empty() {
query.push_str(&format!(" AND trace_id = '{}'", trace_id));
}
if let Some(trace_id) = self.trace_id
&& !trace_id.is_empty()
{
query.push_str(&format!(" AND trace_id = '{trace_id}'"));
}
query
@ -1549,7 +1549,7 @@ impl StreamResponses {
pub fn to_chunks(&self) -> StreamResponseChunks {
// Helper function to format event data
let format_event = |event_type: &str, data: &str| -> BytesImpl {
let formatted = format!("event: {}\ndata: {}\n\n", event_type, data);
let formatted = format!("event: {event_type}\ndata: {data}\n\n");
BytesImpl::from(formatted.into_bytes())
};

78
src/config/src/meta/sql.rs
View File

@ -591,27 +591,27 @@ fn parse_expr_for_field(
}
}
SqlExpr::IsNull(expr) => {
if let SqlExpr::Identifier(ident) = expr.as_ref() {
if parse_expr_check_field_name(&ident.value, field) {
fields.push((
ident.value.to_string(),
SqlValue::String("".to_string()),
SqlOperator::Eq,
*expr_op,
));
}
if let SqlExpr::Identifier(ident) = expr.as_ref()
&& parse_expr_check_field_name(&ident.value, field)
{
fields.push((
ident.value.to_string(),
SqlValue::String("".to_string()),
SqlOperator::Eq,
*expr_op,
));
}
}
SqlExpr::IsNotNull(expr) => {
if let SqlExpr::Identifier(ident) = expr.as_ref() {
if parse_expr_check_field_name(&ident.value, field) {
fields.push((
ident.value.to_string(),
SqlValue::String("".to_string()),
SqlOperator::Eq,
*expr_op,
));
}
if let SqlExpr::Identifier(ident) = expr.as_ref()
&& parse_expr_check_field_name(&ident.value, field)
{
fields.push((
ident.value.to_string(),
SqlValue::String("".to_string()),
SqlOperator::Eq,
*expr_op,
));
}
}
_ => {}
@ -643,21 +643,21 @@ fn parse_expr_like(
field: &str,
fields: &mut Vec<(String, SqlValue, SqlOperator, SqlOperator)>,
) -> Result<(), anyhow::Error> {
if let SqlExpr::Identifier(ident) = expr {
if parse_expr_check_field_name(&ident.value, field) {
let val = get_value_from_expr(pattern);
if val.is_none() {
return Err(anyhow::anyhow!(
"SqlExpr::Like: We only support Identifier at the moment"
));
}
fields.push((
ident.value.to_string(),
val.unwrap(),
SqlOperator::Like,
*next_op,
if let SqlExpr::Identifier(ident) = expr
&& parse_expr_check_field_name(&ident.value, field)
{
let val = get_value_from_expr(pattern);
if val.is_none() {
return Err(anyhow::anyhow!(
"SqlExpr::Like: We only support Identifier at the moment"
));
}
fields.push((
ident.value.to_string(),
val.unwrap(),
SqlOperator::Like,
*next_op,
));
}
Ok(())
}
@ -979,10 +979,10 @@ fn get_field_name_from_expr(expr: &SqlExpr) -> Result<Option<Vec<String>>, anyho
fields.extend(v);
}
}
if let Some(expr) = else_result.as_ref() {
if let Some(v) = get_field_name_from_expr(expr)? {
fields.extend(v);
}
if let Some(expr) = else_result.as_ref()
&& let Some(v) = get_field_name_from_expr(expr)?
{
fields.extend(v);
}
Ok((!fields.is_empty()).then_some(fields))
}
@ -1013,10 +1013,10 @@ fn get_field_name_from_expr(expr: &SqlExpr) -> Result<Option<Vec<String>>, anyho
stride,
} => {
let mut func = |expr: &Option<SqlExpr>| -> Result<(), anyhow::Error> {
if let Some(expr) = expr {
if let Some(v) = get_field_name_from_expr(expr)? {
fields.extend(v);
}
if let Some(expr) = expr
&& let Some(v) = get_field_name_from_expr(expr)?
{
fields.extend(v);
}
Ok(())
};

2
src/config/src/meta/stream.rs
View File

@ -613,7 +613,7 @@ impl Display for TimeRange {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let time_range_start: DateTime<Utc> = Utc.timestamp_nanos(self.start * 1000);
let time_range_end: DateTime<Utc> = Utc.timestamp_nanos(self.end * 1000);
write!(f, "{} to {}", time_range_start, time_range_end)
write!(f, "{time_range_start} to {time_range_end}")
}
}
impl TimeRange {

8
src/config/src/meta/timed_annotations.rs
View File

@ -54,10 +54,10 @@ pub struct TimedAnnotation {
impl TimedAnnotation {
pub fn validate(&self) -> Result<(), String> {
if let Some(end_time) = self.end_time {
if end_time <= self.start_time {
return Err("end time must be greater than start time".to_string());
}
if let Some(end_time) = self.end_time
&& end_time <= self.start_time
{
return Err("end time must be greater than start time".to_string());
}
if self.title.is_empty() {

42
src/config/src/utils/async_file.rs
View File

@ -52,10 +52,7 @@ pub async fn get_file_contents(
if read != to_read {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
format!(
"Expected to read {} bytes, but read {} bytes",
to_read, read
),
format!("Expected to read {to_read} bytes, but read {read} bytes"),
));
}
buf
@ -80,7 +77,7 @@ pub async fn put_file_contents(
};
// Create a temporary file in the same directory
let temp_file = format!("{}.tmp", path);
let temp_file = format!("{path}.tmp");
// Write to temporary file first
let mut file_handle = File::create(&temp_file).await?;
@ -105,25 +102,25 @@ pub async fn clean_empty_dirs(
if entry.path().display().to_string() == dir {
continue;
}
if let Ok(f) = entry.file_type().await {
if f.is_dir() {
match last_updated {
None => {
if let Ok(f) = entry.file_type().await
&& f.is_dir()
{
match last_updated {
None => {
dirs.push(entry.path().to_str().unwrap().to_string());
}
Some(last_updated) => {
if let Ok(meta) = entry.metadata().await
&& meta.modified().unwrap() < last_updated
{
dirs.push(entry.path().to_str().unwrap().to_string());
}
Some(last_updated) => {
if let Ok(meta) = entry.metadata().await {
if meta.modified().unwrap() < last_updated {
dirs.push(entry.path().to_str().unwrap().to_string());
}
}
}
}
}
}
}
Some(Err(e)) => {
log::error!("clean_empty_dirs, err: {}", e);
log::error!("clean_empty_dirs, err: {e}");
break;
}
None => break,
@ -131,12 +128,11 @@ pub async fn clean_empty_dirs(
}
dirs.sort_by_key(|b| std::cmp::Reverse(b.len()));
for dir in dirs {
if let Ok(mut entries) = read_dir(&dir).await {
if let Ok(None) = entries.next_entry().await {
if let Err(e) = remove_dir(&dir).await {
log::error!("Failed to remove empty dir: {}, err: {}", dir, e);
}
}
if let Ok(mut entries) = read_dir(&dir).await
&& let Ok(None) = entries.next_entry().await
&& let Err(e) = remove_dir(&dir).await
{
log::error!("Failed to remove empty dir: {dir}, err: {e}");
}
}
Ok(())

6
src/config/src/utils/download_utils.rs
View File

@ -26,15 +26,15 @@ pub async fn download_file(client: &Client, url: &str, path: &str) -> Result<(),
.get(url)
.send()
.await
.or(Err(format!("Failed to GET from '{}'", &url)))?;
.or(Err(format!("Failed to GET from '{url}'")))?;
let total_size = res
.content_length()
.ok_or(format!("Failed to get content length from '{}'", &url))?;
.ok_or(format!("Failed to get content length from '{url}'"))?;
// download chunks
let mut file = File::create(path)
.await
.or(Err(format!("Failed to create file '{}'", path)))?;
.or(Err(format!("Failed to create file '{path}'")))?;
let mut downloaded: u64 = 0;
let mut stream = res.bytes_stream();

15
src/config/src/utils/file.rs
View File

@ -51,10 +51,7 @@ pub fn get_file_contents(
if read != to_read {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
format!(
"Expected to read {} bytes, but read {} bytes",
to_read, read
),
format!("Expected to read {to_read} bytes, but read {read} bytes"),
));
}
buf
@ -69,7 +66,7 @@ pub fn get_file_contents(
#[inline(always)]
pub fn put_file_contents(file: &str, contents: &[u8]) -> Result<(), std::io::Error> {
// Create a temporary file in the same directory
let temp_file = format!("{}.tmp", file);
let temp_file = format!("{file}.tmp");
// Write to temporary file first
let mut file_handle = File::create(&temp_file)?;
@ -146,9 +143,9 @@ pub async fn scan_files_with_channel(
if path_ext == ext {
files.push(path.to_str().unwrap().to_string());
if limit > 0 && files.len() >= limit {
tx.send(files.clone()).await.map_err(|e| {
std::io::Error::new(std::io::ErrorKind::Other, e.to_string())
})?;
tx.send(files.clone())
.await
.map_err(|e| std::io::Error::other(e.to_string()))?;
files.clear();
}
}
@ -157,7 +154,7 @@ pub async fn scan_files_with_channel(
if !files.is_empty() {
tx.send(files)
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
.map_err(|e| std::io::Error::other(e.to_string()))?;
}
Ok(())
}

2
src/config/src/utils/flatten.rs
View File

@ -105,7 +105,7 @@ fn flatten_object(
for (mut k, v) in current.into_iter() {
format_key(&mut k);
let parent_key = if depth > 0 {
format!("{}{}{}", parent_key, KEY_SEPARATOR, k)
format!("{parent_key}{KEY_SEPARATOR}{k}")
} else {
k
};

2
src/config/src/utils/md5.rs
View File

@ -16,7 +16,7 @@
// Get the md5 hash of a string
pub fn hash(input: &str) -> String {
let digest = md5::compute(input.as_bytes());
format!("{:x}", digest)
format!("{digest:x}")
}
// Get a short hash of a string

2
src/config/src/utils/prom_json_encoder.rs
View File

@ -63,7 +63,7 @@ impl JsonEncoder {
// Write `# TYPE` header.
let metric_type: MetricType = mf.get_field_type();
let lowercase_type = json!(format!("{:?}", metric_type).to_lowercase());
let lowercase_type = json!(format!("{metric_type:?}").to_lowercase());
mf_map.insert("__type__".to_string(), lowercase_type);
for m in mf.get_metric() {

60
src/config/src/utils/query_select_utils.rs
View File

@ -36,12 +36,12 @@ impl StringMatchReplacer {
/// Check if a value ends with "::_o2_custom" suffix
fn has_o2_custom_suffix(value: &Value) -> Option<String> {
if let Value::SingleQuotedString(s) = value {
if s.ends_with(O2_CUSTOM_SUFFIX) {
// Extract the prefix before "::_o2_custom"
let prefix = s.trim_end_matches(O2_CUSTOM_SUFFIX);
return Some(prefix.to_string());
}
if let Value::SingleQuotedString(s) = value
&& s.ends_with(O2_CUSTOM_SUFFIX)
{
// Extract the prefix before "::_o2_custom"
let prefix = s.trim_end_matches(O2_CUSTOM_SUFFIX);
return Some(prefix.to_string());
}
None
}
@ -91,14 +91,14 @@ impl StringMatchReplacer {
return None;
}
if let Expr::Value(value) = &list[0] {
if let Some(match_value) = Self::has_o2_custom_suffix(value) {
self.replacements_made += 1;
return Some(Self::create_str_match_function(
field_expr.clone(),
match_value,
));
}
if let Expr::Value(value) = &list[0]
&& let Some(match_value) = Self::has_o2_custom_suffix(value)
{
self.replacements_made += 1;
return Some(Self::create_str_match_function(
field_expr.clone(),
match_value,
));
}
None
@ -107,19 +107,19 @@ impl StringMatchReplacer {
/// Process equality expression and replace if it matches our pattern
fn process_equality_expression(&mut self, left: &Expr, right: &Expr) -> Option<Expr> {
// Check if right side is a value with _o2_custom suffix
if let Expr::Value(value) = right {
if let Some(match_value) = Self::has_o2_custom_suffix(value) {
self.replacements_made += 1;
return Some(Self::create_str_match_function(left.clone(), match_value));
}
if let Expr::Value(value) = right
&& let Some(match_value) = Self::has_o2_custom_suffix(value)
{
self.replacements_made += 1;
return Some(Self::create_str_match_function(left.clone(), match_value));
}
// Check if left side is a value with _o2_custom suffix
if let Expr::Value(value) = left {
if let Some(match_value) = Self::has_o2_custom_suffix(value) {
self.replacements_made += 1;
return Some(Self::create_str_match_function(right.clone(), match_value));
}
if let Expr::Value(value) = left
&& let Some(match_value) = Self::has_o2_custom_suffix(value)
{
self.replacements_made += 1;
return Some(Self::create_str_match_function(right.clone(), match_value));
}
None
@ -147,11 +147,11 @@ impl VisitorMut for StringMatchReplacer {
}
Expr::BinaryOp { left, right, op } => {
// Handle equality expressions
if matches!(op, sqlparser::ast::BinaryOperator::Eq) {
if let Some(replacement) = self.process_equality_expression(left, right) {
*expr = replacement;
return ControlFlow::Continue(());
}
if matches!(op, sqlparser::ast::BinaryOperator::Eq)
&& let Some(replacement) = self.process_equality_expression(left, right)
{
*expr = replacement;
return ControlFlow::Continue(());
}
// Continue with normal traversal
self.pre_visit_expr(left)?;
@ -299,7 +299,7 @@ pub fn replace_o2_custom_patterns(sql: &str) -> Result<String, String> {
let dialect = GenericDialect {};
let mut statements =
Parser::parse_sql(&dialect, sql).map_err(|e| format!("Parse error: {}", e))?;
Parser::parse_sql(&dialect, sql).map_err(|e| format!("Parse error: {e}"))?;
if statements.is_empty() {
return Err("No statements found".to_string());

3
src/config/src/utils/record_batch_ext.rs
View File

@ -92,8 +92,7 @@ pub fn convert_json_to_record_batch(
}
let (data_type, builder) = res.ok_or_else(|| {
ArrowError::SchemaError(format!(
"Cannot find key {} (value: {:?}) in schema {:?}",
k, v, schema
"Cannot find key {k} (value: {v:?}) in schema {schema:?}"
))
})?;
match data_type {

3
src/config/src/utils/schema.rs
View File

@ -188,8 +188,7 @@ fn convert_data_type(
}
_ => {
return Err(ArrowError::SchemaError(format!(
"Cannot infer schema from conflicting types: {:?} and {:?}",
f_type, data_type
"Cannot infer schema from conflicting types: {f_type:?} and {data_type:?}"
)));
}
}

82
src/config/src/utils/sql.rs
View File

@ -28,7 +28,7 @@ use sqlparser::{
use crate::TIMESTAMP_COL_NAME;
pub const AGGREGATE_UDF_LIST: [&str; 10] = [
pub const AGGREGATE_UDF_LIST: [&str; 16] = [
"min",
"max",
"avg",
@ -39,21 +39,26 @@ pub const AGGREGATE_UDF_LIST: [&str; 10] = [
"approx_percentile_cont",
"percentile_cont",
"summary_percentile",
"first_value",
"last_value",
"approx_distinct",
"approx_median",
"approx_percentile_cont",
"approx_percentile_cont_with_weight",
];
pub fn is_aggregate_query(query: &str) -> Result<bool, sqlparser::parser::ParserError> {
let ast = Parser::parse_sql(&GenericDialect {}, query)?;
for statement in ast.iter() {
if let Statement::Query(query) = statement {
if is_aggregate_in_select(query)
if let Statement::Query(query) = statement
&& (is_aggregate_in_select(query)
|| has_group_by(query)
|| has_having(query)
|| has_join(query)
|| has_subquery(statement)
|| has_union(query)
{
return Ok(true);
}
|| has_union(query))
{
return Ok(true);
}
}
Ok(false)
@ -67,14 +72,13 @@ pub fn is_simple_aggregate_query(query: &str) -> Result<bool, sqlparser::parser:
if has_subquery(statement) || has_window_functions(statement) {
return Ok(false);
}
if let Statement::Query(query) = statement {
if !is_aggregate_in_select(query)
if let Statement::Query(query) = statement
&& (!is_aggregate_in_select(query)
|| has_join(query)
|| has_union(query)
|| has_cte(query)
{
return Ok(false);
}
|| has_cte(query))
{
return Ok(false);
}
}
Ok(true)
@ -84,10 +88,11 @@ pub fn is_simple_aggregate_query(query: &str) -> Result<bool, sqlparser::parser:
pub fn is_simple_distinct_query(query: &str) -> Result<bool, sqlparser::parser::ParserError> {
let ast = Parser::parse_sql(&GenericDialect {}, query)?;
for statement in ast.iter() {
if let Statement::Query(query) = statement {
if has_distinct(query) && !has_group_by(query) {
return Ok(true);
}
if let Statement::Query(query) = statement
&& has_distinct(query)
&& !has_group_by(query)
{
return Ok(true);
}
}
Ok(false)
@ -113,10 +118,9 @@ fn is_aggregate_in_select(query: &Query) -> bool {
for select_item in &select.projection {
if let SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, alias: _ } =
select_item
&& is_aggregate_expression(expr)
{
if is_aggregate_expression(expr) {
return true;
}
return true;
}
}
}
@ -176,7 +180,7 @@ fn has_group_by(query: &Query) -> bool {
// Check if has distinct
fn has_distinct(query: &Query) -> bool {
let mut visitor = DistinctVisitor::new();
query.visit(&mut visitor);
let _ = query.visit(&mut visitor);
visitor.has_distinct
}
@ -197,11 +201,11 @@ impl Visitor for DistinctVisitor {
fn pre_visit_query(&mut self, query: &Query) -> ControlFlow<Self::Break> {
// Check for SELECT DISTINCT
if let SetExpr::Select(select) = query.body.as_ref() {
if select.distinct.is_some() {
self.has_distinct = true;
return ControlFlow::Break(());
}
if let SetExpr::Select(select) = query.body.as_ref()
&& select.distinct.is_some()
{
self.has_distinct = true;
return ControlFlow::Break(());
}
ControlFlow::Continue(())
}
@ -248,13 +252,13 @@ fn has_join(query: &Query) -> bool {
fn has_union(query: &Query) -> bool {
let mut visitor = UnionVisitor::new();
query.visit(&mut visitor);
let _ = query.visit(&mut visitor);
visitor.has_union
}
fn has_subquery(stat: &Statement) -> bool {
let mut visitor = SubqueryVisitor::new();
stat.visit(&mut visitor);
let _ = stat.visit(&mut visitor);
visitor.is_subquery
}
@ -320,7 +324,7 @@ impl Visitor for SubqueryVisitor {
fn has_timestamp(stat: &Statement) -> bool {
let mut visitor = TimestampVisitor::new();
stat.visit(&mut visitor);
let _ = stat.visit(&mut visitor);
visitor.timestamp_selected
}
@ -415,11 +419,11 @@ impl Visitor for TimestampVisitor {
// Handle alias chain: SELECT ts1 FROM (...) where ts1 is alias for
// _timestamp
if let Expr::Identifier(ident) = expr {
if self.timestamp_aliases.contains(&ident.value) {
self.timestamp_selected = true;
return ControlFlow::Break(());
}
if let Expr::Identifier(ident) = expr
&& self.timestamp_aliases.contains(&ident.value)
{
self.timestamp_selected = true;
return ControlFlow::Break(());
}
}
@ -436,10 +440,10 @@ impl Visitor for TimestampVisitor {
}
// If the expression is an alias we already know maps to timestamp
if let Expr::Identifier(ident) = expr {
if self.timestamp_aliases.contains(&ident.value) {
self.timestamp_aliases.insert(alias.value.clone());
}
if let Expr::Identifier(ident) = expr
&& self.timestamp_aliases.contains(&ident.value)
{
self.timestamp_aliases.insert(alias.value.clone());
}
}
@ -459,7 +463,7 @@ impl Visitor for TimestampVisitor {
fn has_window_functions(stat: &Statement) -> bool {
let mut visitor = WindowFunctionVisitor::new();
stat.visit(&mut visitor);
let _ = stat.visit(&mut visitor);
visitor.has_window_function
}

20
src/config/src/utils/sysinfo/cgroup.rs
View File

@ -49,20 +49,22 @@ pub fn get_memory_limit() -> usize {
}
fn read_cpu_cgroup_v1() -> usize {
if let Ok(val) = std::fs::read_to_string("/sys/fs/cgroup/cpu.max") {
if !val.is_empty() && !val.to_lowercase().starts_with("max") {
let columns = val.split(' ').collect::<Vec<&str>>();
return columns[0].parse::<usize>().unwrap_or_default();
}
if let Ok(val) = std::fs::read_to_string("/sys/fs/cgroup/cpu.max")
&& !val.is_empty()
&& !val.to_lowercase().starts_with("max")
{
let columns = val.split(' ').collect::<Vec<&str>>();
return columns[0].parse::<usize>().unwrap_or_default();
};
0
}
fn read_memory_cgroup_v1() -> usize {
if let Ok(val) = std::fs::read_to_string("/sys/fs/cgroup/memory.max") {
if !val.is_empty() && !val.to_lowercase().starts_with("max") {
return val.trim_end().parse::<usize>().unwrap_or_default();
}
if let Ok(val) = std::fs::read_to_string("/sys/fs/cgroup/memory.max")
&& !val.is_empty()
&& !val.to_lowercase().starts_with("max")
{
return val.trim_end().parse::<usize>().unwrap_or_default();
};
0
}

15
src/config/src/utils/sysinfo/net.rs
View File

@ -113,16 +113,15 @@ pub fn get_tcp_conn_resets() -> usize {
match std::fs::read_to_string("/proc/net/netstat") {
Ok(contents) => {
for line in contents.lines() {
if line.starts_with("TcpExt:") {
if let Some(next_line) = contents
if line.starts_with("TcpExt:")
&& let Some(next_line) = contents
.lines()
.nth(contents.lines().position(|l| l == line).unwrap() + 1)
{
let values: Vec<&str> = next_line.split_whitespace().collect();
// TCPAbortOnData is at index 19
if let Some(resets) = values.get(19) {
return resets.parse().unwrap_or(0);
}
{
let values: Vec<&str> = next_line.split_whitespace().collect();
// TCPAbortOnData is at index 19
if let Some(resets) = values.get(19) {
return resets.parse().unwrap_or(0);
}
}
}

8
src/config/src/utils/time.rs
View File

@ -283,16 +283,16 @@ pub fn format_duration(ms: u64) -> String {
let remaining_hours = hours % 24;
let mut parts = Vec::new();
if days > 0 {
parts.push(format!("{}d", days));
parts.push(format!("{days}d"));
}
if remaining_hours > 0 {
parts.push(format!("{}h", remaining_hours));
parts.push(format!("{remaining_hours}h"));
}
if remaining_minutes > 0 {
parts.push(format!("{}m", remaining_minutes));
parts.push(format!("{remaining_minutes}m"));
}
if remaining_seconds > 0 {
parts.push(format!("{}s", remaining_seconds));
parts.push(format!("{remaining_seconds}s"));
}
parts.join("")
}

7
src/handler/grpc/flight/mod.rs
View File

@ -158,12 +158,9 @@ impl FlightService for FlightServiceImpl {
.indent(false)
.to_string();
println!("+---------------------------+--------------------------+");
println!(
"follow physical plan, is_super_cluster_follower_leader: {}",
is_super_cluster
);
println!("follow physical plan, is_super_cluster_follower_leader: {is_super_cluster}");
println!("+---------------------------+--------------------------+");
println!("{}", plan);
println!("{plan}");
}
schema = add_scan_stats_to_schema(schema, scan_stats);

22
src/handler/grpc/request/event.rs
View File

@ -76,17 +76,17 @@ impl Event for Eventer {
}
// cache index for the parquet
if cfg.cache_latest_files.cache_index && item.meta.index_size > 0 {
if let Some(ttv_file) = convert_parquet_idx_file_name_to_tantivy_file(&item.key)
{
files_to_download.push((
item.id,
item.account.clone(),
ttv_file,
item.meta.index_size,
item.meta.max_ts,
));
}
if cfg.cache_latest_files.cache_index
&& item.meta.index_size > 0
&& let Some(ttv_file) = convert_parquet_idx_file_name_to_tantivy_file(&item.key)
{
files_to_download.push((
item.id,
item.account.clone(),
ttv_file,
item.meta.index_size,
item.meta.max_ts,
));
}
}

19
src/handler/grpc/request/ingest.rs
View File

@ -69,15 +69,14 @@ impl Ingest for Ingester {
.unwrap_or(IngestionType::Multi); // multi is just place holder
if log_ingestion_type != IngestionType::Json {
Err(Error::IngestionError(format!(
"Internal gPRC metric ingestion only supports json type data, got {:?}",
log_ingestion_type
"Internal gPRC metric ingestion only supports json type data, got {log_ingestion_type:?}"
)))
} else {
let data = bytes::Bytes::from(in_data.data);
crate::service::metrics::json::ingest(&org_id, data)
.await
.map(|_| ()) // we don't care about success response
.map_err(|e| Error::IngestionError(format!("error in ingesting metrics {}", e)))
.map_err(|e| Error::IngestionError(format!("error in ingesting metrics {e}")))
}
}
StreamType::Traces => {
@ -88,15 +87,14 @@ impl Ingest for Ingester {
.unwrap_or(IngestionType::Multi); // multi is just place holder
if log_ingestion_type != IngestionType::Json {
Err(Error::IngestionError(format!(
"Internal gRPC trace ingestion only supports json type data, got {:?}",
log_ingestion_type
"Internal gRPC trace ingestion only supports json type data, got {log_ingestion_type:?}"
)))
} else {
let data = bytes::Bytes::from(in_data.data);
crate::service::traces::ingest_json(&org_id, data, OtlpRequestType::Grpc, &stream_name)
.await
.map(|_| ()) // we don't care about success response
.map_err(|e| Error::IngestionError(format!("error in ingesting traces {}", e)))
.map_err(|e| Error::IngestionError(format!("error in ingesting traces {e}")))
}
}
StreamType::EnrichmentTables => {
@ -128,20 +126,17 @@ impl Ingest for Ingester {
.await
{
Err(e) => Err(Error::IngestionError(format!(
"Internal gPRC ingestion service errors saving enrichment data: {}",
e
"Internal gPRC ingestion service errors saving enrichment data: {e}"
))),
Ok(res) => {
if res.status() != StatusCode::OK {
let status: StatusCode = res.status();
log::error!(
"Internal gPRC ingestion service errors saving enrichment data: code: {}, body: {:?}",
status,
"Internal gPRC ingestion service errors saving enrichment data: code: {status}, body: {:?}",
res.into_body()
);
Err(Error::IngestionError(format!(
"Internal gPRC ingestion service errors saving enrichment data: http code {}",
status
"Internal gPRC ingestion service errors saving enrichment data: http code {status}"
)))
} else {
Ok(())

123
src/handler/http/auth/jwt.rs
View File

@ -137,64 +137,7 @@ pub async fn process_token(
// Check if the user exists in the database
let db_user = db::user::get_user_by_email(&user_email).await;
if db_user.is_none() {
log::info!("User does not exist in the database");
if openfga_cfg.enabled {
for (index, org) in source_orgs.iter().enumerate() {
// Assuming all the relevant tuples for this org exist
let mut tuples = vec![];
get_user_creation_tuples(
&org.name,
&user_email,
&org.role.to_string(),
&mut tuples,
);
// Create the org if it does not exist. `org.name` is the id of the org.
// Also it creates necessary ofga tuples for the newly created org
let _ = organization::check_and_create_org(&org.name).await;
if index == 0 {
// this is to allow user call organization api with org
tuples.push(get_user_org_tuple(&user_email, &user_email));
}
tuples_to_add.insert(org.name.to_owned(), tuples);
}
}
let updated_db_user = DBUser {
email: user_email.to_owned(),
first_name: name.to_owned(),
last_name: "".to_owned(),
password: "".to_owned(),
salt: "".to_owned(),
organizations: source_orgs,
is_external: true,
password_ext: Some("".to_owned()),
};
match users::create_new_user(updated_db_user).await {
Ok(_) => {
log::info!("User added to the database");
if openfga_cfg.enabled {
for (_, tuples) in tuples_to_add {
match update_tuples(tuples, vec![]).await {
Ok(_) => {
log::info!("User updated to the openfga");
}
Err(e) => {
log::error!("Error updating user to the openfga: {}", e);
}
}
}
}
}
Err(e) => {
log::error!("Error adding user to the database: {}", e);
}
}
} else {
if let Some(db_user) = db_user {
// check if user is service account and skip the role update ,
// assumption is always a service account irrespective of the orgs it belongs to
if res
@ -207,7 +150,7 @@ pub async fn process_token(
}
log::info!("User exists in the database perform check for role change");
let existing_db_user = db_user.unwrap();
let existing_db_user = db_user;
let existing_orgs = existing_db_user.organizations;
let mut orgs_removed = Vec::new();
let mut orgs_role_changed = HashMap::new();
@ -356,7 +299,64 @@ pub async fn process_token(
}
}
}
};
} else {
log::info!("User does not exist in the database");
if openfga_cfg.enabled {
for (index, org) in source_orgs.iter().enumerate() {
// Assuming all the relevant tuples for this org exist
let mut tuples = vec![];
get_user_creation_tuples(
&org.name,
&user_email,
&org.role.to_string(),
&mut tuples,
);
// Create the org if it does not exist. `org.name` is the id of the org.
// Also it creates necessary ofga tuples for the newly created org
let _ = organization::check_and_create_org(&org.name).await;
if index == 0 {
// this is to allow user call organization api with org
tuples.push(get_user_org_tuple(&user_email, &user_email));
}
tuples_to_add.insert(org.name.to_owned(), tuples);
}
}
let updated_db_user = DBUser {
email: user_email.to_owned(),
first_name: name.to_owned(),
last_name: "".to_owned(),
password: "".to_owned(),
salt: "".to_owned(),
organizations: source_orgs,
is_external: true,
password_ext: Some("".to_owned()),
};
match users::create_new_user(updated_db_user).await {
Ok(_) => {
log::info!("User added to the database");
if openfga_cfg.enabled {
for (_, tuples) in tuples_to_add {
match update_tuples(tuples, vec![]).await {
Ok(_) => {
log::info!("User updated to the openfga");
}
Err(e) => {
log::error!("Error updating user to the openfga: {}", e);
}
}
}
}
}
Err(e) => {
log::error!("Error adding user to the database: {}", e);
}
}
}
}
}
@ -562,8 +562,7 @@ fn format_role_name(org: &str, role: &str) -> String {
#[cfg(feature = "enterprise")]
pub fn format_role_name_only(role: &str) -> String {
let role = RE_ROLE_NAME.replace_all(role, "_").to_string();
role
RE_ROLE_NAME.replace_all(role, "_").to_string()
}
#[cfg(feature = "cloud")]

35
src/handler/http/auth/validator.rs
View File

@ -105,14 +105,13 @@ pub async fn validator(
);
#[cfg(feature = "enterprise")]
if let Some(role) = &res.user_role {
if role.eq(&UserRole::Viewer)
&& req.method().eq(&Method::PUT)
&& path.ends_with(&format!("users/{}", res.user_email))
{
// Viewer should be able to update its own details
return Ok(req);
}
if let Some(role) = &res.user_role
&& role.eq(&UserRole::Viewer)
&& req.method().eq(&Method::PUT)
&& path.ends_with(&format!("users/{}", res.user_email))
{
// Viewer should be able to update its own details
return Ok(req);
}
if auth_info.bypass_check
@ -158,10 +157,10 @@ pub async fn validate_credentials(
path: &str,
) -> Result<TokenValidationResponse, Error> {
let mut path_columns = path.split('/').collect::<Vec<&str>>();
if let Some(v) = path_columns.last() {
if v.is_empty() {
path_columns.pop();
}
if let Some(v) = path_columns.last()
&& v.is_empty()
{
path_columns.pop();
}
let user = if path_columns.last().unwrap_or(&"").eq(&"organizations") {
@ -312,10 +311,10 @@ pub async fn validate_credentials_ext(
let config = get_config();
let password_ext_salt = config.auth.ext_auth_salt.as_str();
let mut path_columns = path.split('/').collect::<Vec<&str>>();
if let Some(v) = path_columns.last() {
if v.is_empty() {
path_columns.pop();
}
if let Some(v) = path_columns.last()
&& v.is_empty()
{
path_columns.pop();
}
let user = if path_columns.last().unwrap_or(&"").eq(&"organizations") {
@ -998,7 +997,7 @@ pub(crate) async fn list_objects_for_user(
/// Helper function to extract the relative path after the base URI and path prefix
fn extract_relative_path(full_path: &str, path_prefix: &str) -> String {
let base_uri = config::get_config().common.base_uri.clone();
let full_prefix = format!("{}{}", base_uri, path_prefix);
let full_prefix = format!("{base_uri}{path_prefix}");
full_path
.strip_prefix(&full_prefix)
.unwrap_or(full_path)
@ -1043,7 +1042,7 @@ fn extract_full_url(req: &ServiceRequest) -> String {
.map(|pq| pq.as_str())
.unwrap_or("");
format!("{}://{}{}", scheme, host, path)
format!("{scheme}://{host}{path}")
}
#[cfg(test)]

8
src/handler/http/models/dashboards.rs
View File

@ -213,10 +213,10 @@ impl ListDashboardsQuery {
// When the title parameter is not set we simply want to return all
// dashboards that match the selected folder so we ignore the page_size
// parameter.
if self.title.is_some_and(|t| !t.is_empty()) {
if let Some(page_size) = self.page_size {
query = query.paginate(page_size, 0)
}
if self.title.is_some_and(|t| !t.is_empty())
&& let Some(page_size) = self.page_size
{
query = query.paginate(page_size, 0)
}
query

11
src/handler/http/request/actions/action.rs
View File

@ -131,7 +131,7 @@ pub async fn serve_action_zip(path: web::Path<(String, Ksuid)>) -> Result<HttpRe
let resp = HttpResponse::Ok()
.insert_header((
"Content-Disposition",
format!("attachment; filename=\"{}\"", file_name),
format!("attachment; filename=\"{file_name}\""),
))
.content_type("application/zip")
.streaming(stream::once(async { Ok::<Bytes, actix_web::Error>(bytes) }));
@ -178,14 +178,13 @@ pub async fn update_action_details(
let mut req = req.into_inner();
// Validate environment variables if they are being updated
if let Some(ref env_vars) = req.environment_variables {
if let Err(e) =
if let Some(ref env_vars) = req.environment_variables
&& let Err(e) =
crate::handler::http::request::actions::action::validate_environment_variables(
env_vars,
)
{
return Ok(MetaHttpResponse::bad_request(e));
}
{
return Ok(MetaHttpResponse::bad_request(e));
}
let sa = match req.service_account.clone() {

2
src/handler/http/request/alerts/deprecated.rs
View File

@ -317,7 +317,7 @@ async fn get_alert(path: web::Path<(String, String, String)>, req: HttpRequest)
if let Ok(scheduled_job) = scheduler::get(
&org_id,
TriggerModule::Alert,
&format!("{}/{}/{}", stream_type, stream_name, name),
&format!("{stream_type}/{stream_name}/{name}"),
)
.await
{

4
src/handler/http/request/clusters/mod.rs
View File

@ -20,7 +20,7 @@ use hashbrown::HashMap;
#[cfg(feature = "enterprise")]
use {
config::meta::cluster::RoleGroup,
o2_enterprise::enterprise::common::config::get_config as get_o2_config, std::io::ErrorKind,
o2_enterprise::enterprise::common::config::get_config as get_o2_config,
};
/// ListClusters
@ -45,7 +45,7 @@ pub async fn list_clusters() -> Result<HttpResponse, Error> {
Some(RoleGroup::Interactive),
)
.await
.map_err(|e| Error::new(ErrorKind::Other, e))?;
.map_err(Error::other)?;
let mut regions = HashMap::with_capacity(clusters.len());
for c in clusters {
let region: &mut Vec<_> = regions.entry(c.region).or_insert_with(Vec::new);

4
src/handler/http/request/logs/ingest.rs
View File

@ -378,7 +378,7 @@ pub async fn otlp_logs_write(
log::error!("[LOGS:OTLP] Invalid proto: {}", e);
return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
http::StatusCode::BAD_REQUEST,
format!("Invalid proto: {}", e),
format!("Invalid proto: {e}"),
)));
}
},
@ -389,7 +389,7 @@ pub async fn otlp_logs_write(
log::error!("[LOGS:OTLP] Invalid json: {}", e);
return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
http::StatusCode::BAD_REQUEST,
format!("Invalid json: {}", e),
format!("Invalid json: {e}"),
)));
}
}

2
src/handler/http/request/logs/loki.rs
View File

@ -150,7 +150,7 @@ fn parse_protobuf_request(
Some("snappy") | None => snap::raw::Decoder::new()
.decompress_vec(&body)
.map_err(|e| LokiError::UnsupportedContentEncoding {
encoding: format!("snappy decompression failed: {}", e),
encoding: format!("snappy decompression failed: {e}"),
})?,
Some("identity") => body.to_vec(),
Some(encoding) => {

10
src/handler/http/request/organization/settings.rs
View File

@ -95,11 +95,11 @@ async fn create(
}
#[cfg(feature = "enterprise")]
if let Some(aggregation_cache_enabled) = settings.aggregation_cache_enabled {
if config::get_config().disk_cache.aggregation_cache_enabled {
field_found = true;
data.aggregation_cache_enabled = aggregation_cache_enabled;
}
if let Some(aggregation_cache_enabled) = settings.aggregation_cache_enabled
&& config::get_config().disk_cache.aggregation_cache_enabled
{
field_found = true;
data.aggregation_cache_enabled = aggregation_cache_enabled;
}
if let Some(enable_streaming_search) = settings.enable_streaming_search {

56
src/handler/http/request/ratelimit/mod.rs
View File

@ -73,9 +73,7 @@ async fn validate_ratelimit_updater(
update_type: &str,
rules: &RatelimitRuleUpdater,
) -> Result<(), anyhow::Error> {
let global_default_rules = get_default_rules()
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
let global_default_rules = get_default_rules().await.map_err(std::io::Error::other)?;
// module-level is org-level
let org_level_rules = infra::table::ratelimit::fetch_rules(
@ -84,7 +82,7 @@ async fn validate_ratelimit_updater(
Some(DEFAULT_GLOBAL_USER_ROLE_IDENTIFIER.to_string()),
)
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
.map_err(std::io::Error::other)?;
let org_level_thresholds: HashMap<(String, ApiGroupOperation), i32> = org_level_rules
.into_iter()
@ -128,8 +126,9 @@ async fn validate_ratelimit_updater(
if update_type == "role" {
let compare_threshold =
org_level_thresholds.get(&(api_group_name.to_string(), operation));
if compare_threshold.is_some() && *compare_threshold.unwrap() < *threshold {
let compare_threshold = compare_threshold.unwrap();
if let Some(compare_threshold) = compare_threshold
&& *compare_threshold < *threshold
{
return Err(anyhow::anyhow!(
"{}:{} threshold must be lower than or equal to {:?}, got {}, because module-level rule limit {}:{} is {:?}",
api_group_name,
@ -183,8 +182,7 @@ pub async fn api_modules(path: web::Path<String>) -> Result<HttpResponse, Error>
let org_id = path.into_inner();
if org_id != QUOTA_PAGE_REQUIRED_ORG {
return Ok(MetaHttpResponse::bad_request(format!(
"org_id: {} has no access",
org_id,
"org_id: {org_id} has no access",
)));
}
@ -235,8 +233,7 @@ pub async fn list_module_ratelimit(
let org_id = path.into_inner();
if org_id != QUOTA_PAGE_REQUIRED_ORG {
return Ok(MetaHttpResponse::bad_request(format!(
"org_id: {} has no access",
org_id,
"org_id: {org_id} has no access",
)));
}
@ -248,9 +245,7 @@ pub async fn list_module_ratelimit(
.await;
}
let global_default_rules = get_default_rules()
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
let global_default_rules = get_default_rules().await.map_err(std::io::Error::other)?;
// module-level is org-level
let all_rules = infra::table::ratelimit::fetch_rules(
global_default_rules.clone(),
@ -258,7 +253,7 @@ pub async fn list_module_ratelimit(
Some(DEFAULT_GLOBAL_USER_ROLE_IDENTIFIER.to_string()),
)
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
.map_err(std::io::Error::other)?;
let info = get_ratelimit_global_default_api_info().await;
let api_group_info = info.api_groups(Some(org_id.as_str()), all_rules).await;
@ -304,8 +299,7 @@ pub async fn list_role_ratelimit(
let org_id = path.into_inner();
if org_id != QUOTA_PAGE_REQUIRED_ORG {
return Ok(MetaHttpResponse::bad_request(format!(
"org_id: {} has no access",
org_id,
"org_id: {org_id} has no access",
)));
}
@ -333,7 +327,7 @@ pub async fn list_role_ratelimit(
Some(DEFAULT_GLOBAL_USER_ROLE_IDENTIFIER.to_string()),
)
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
.map_err(std::io::Error::other)?;
let role_level_rules = infra::table::ratelimit::fetch_rules(
org_level_rules,
@ -341,7 +335,7 @@ pub async fn list_role_ratelimit(
Some(user_role),
)
.await
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
.map_err(std::io::Error::other)?;
let info = get_ratelimit_global_default_api_info().await;
let api_group_info = info
@ -392,16 +386,13 @@ pub async fn update_ratelimit(
let org_id = path.into_inner();
if org_id != QUOTA_PAGE_REQUIRED_ORG {
return Ok(MetaHttpResponse::bad_request(format!(
"org_id: {} has no access",
org_id,
"org_id: {org_id} has no access",
)));
}
let mut bytes = web::BytesMut::new();
while let Some(item) = payload.next().await {
bytes.extend_from_slice(
&item.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?,
);
bytes.extend_from_slice(&item.map_err(std::io::Error::other)?);
}
let updater = match parse_and_validate_ratelimit_payload(bytes).await {
@ -424,8 +415,7 @@ pub async fn update_ratelimit(
validate_ratelimit_updater(org_id.as_str(), &query.get_update_type(), &updater).await
{
return Ok(MetaHttpResponse::bad_request(format!(
"validate ratelimit updater error: {}",
e,
"validate ratelimit updater error: {e}"
)));
}
@ -443,7 +433,7 @@ pub async fn update_ratelimit(
_ => {
return Ok(MetaHttpResponse::bad_request(format!(
"update_type is incorrect: {}, only support module or role",
query.get_update_type(),
query.get_update_type()
)));
}
};
@ -473,7 +463,7 @@ async fn parse_and_validate_ratelimit_payload(
bytes: web::BytesMut,
) -> Result<RatelimitRuleUpdater, String> {
let value: serde_json::Value =
serde_json::from_slice(&bytes).map_err(|e| format!("Failed to parse JSON: {}", e))?;
serde_json::from_slice(&bytes).map_err(|e| format!("Failed to parse JSON: {e}"))?;
let obj = value
.as_object()
@ -484,24 +474,18 @@ async fn parse_and_validate_ratelimit_payload(
for (outer_key, outer_value) in obj {
let inner_obj = outer_value
.as_object()
.ok_or_else(|| format!("Value for key '{}' must be an object", outer_key))?;
.ok_or_else(|| format!("Value for key '{outer_key}' must be an object"))?;
let mut inner_map = HashMap::new();
for (inner_key, inner_value) in inner_obj {
let number = inner_value.as_i64().ok_or_else(|| {
format!(
"Value for key '{}' in group '{}' must be an i32",
inner_key, outer_key
)
format!("Value for key '{inner_key}' in group '{outer_key}' must be an i32")
})?;
if number > i32::MAX as i64 || number < 0 {
return Err(format!(
"The Rate limit value of {} for the key '{}' in group '{}' exceeds the allowed integer range of [{}, {}]",
number,
inner_key,
outer_key,
"The Rate limit value of {number} for the key '{inner_key}' in group '{outer_key}' exceeds the allowed integer range of [{}, {}]",
0,
i32::MAX
));

18
src/handler/http/request/search/around.rs
View File

@ -57,13 +57,13 @@ pub(crate) async fn around(
let mut query_fn = query
.get("query_fn")
.and_then(|v| base64::decode_url(v).ok());
if let Some(vrl_function) = &query_fn {
if !vrl_function.trim().ends_with('.') {
query_fn = Some(format!("{} \n .", vrl_function));
}
if let Some(vrl_function) = &query_fn
&& !vrl_function.trim().ends_with('.')
{
query_fn = Some(format!("{vrl_function} \n ."));
}
let default_sql = format!("SELECT * FROM \"{}\" ", stream_name);
let default_sql = format!("SELECT * FROM \"{stream_name}\" ");
let mut around_sql = if let Some(sql) = sql {
sql
} else {
@ -81,10 +81,10 @@ pub(crate) async fn around(
if let Some(body) = body {
let data: json::Value = json::from_slice(&body).unwrap_or_default();
if let Some(data) = data.as_object() {
if let Some(key) = data.get(TIMESTAMP_COL_NAME) {
if let Some(ts) = key.as_i64() {
around_key = ts;
}
if let Some(key) = data.get(TIMESTAMP_COL_NAME)
&& let Some(ts) = key.as_i64()
{
around_key = ts;
}
for field in DEFAULT_SEARCH_AROUND_FIELDS.iter() {
if let Some(value) = data.get(field) {

39
src/handler/http/request/search/mod.rs
View File

@ -272,8 +272,7 @@ pub async fn search(
{
req.query.start_time = req.query.end_time - max_query_range * 3600 * 1_000_000;
range_error = format!(
"Query duration is modified due to query range restriction of {} hours",
max_query_range
"Query duration is modified due to query range restriction of {max_query_range} hours"
);
}
}
@ -716,7 +715,7 @@ pub async fn build_search_request_per_field(
.and_then(|v| base64::decode_url(v.as_ref()).ok())
.map(|vrl| {
if !vrl.trim().ends_with('.') {
format!("{} \n .", vrl)
format!("{vrl} \n .")
} else {
vrl
}
@ -781,12 +780,12 @@ pub async fn build_search_request_per_field(
query.uses_zo_fn = functions::get_all_transform_keys(org_id)
.await
.iter()
.any(|fn_name| decoded_sql.contains(&format!("{}(", fn_name)));
.any(|fn_name| decoded_sql.contains(&format!("{fn_name}(")));
// pick up where clause from sql
let sql_where_from_query =
match SearchService::sql::pickup_where(&decoded_sql, None) {
Ok(Some(v)) => format!("WHERE {}", v),
Ok(Some(v)) => format!("WHERE {v}"),
Ok(None) => "".to_string(),
Err(e) => {
return Err(Error::other(e));
@ -815,12 +814,12 @@ pub async fn build_search_request_per_field(
return Err(Error::other("Invalid filter format"));
}
let vals = columns[1].split(',').collect::<Vec<_>>().join("','");
let sql_where = format!("WHERE {} IN ('{}')", columns[0], vals);
let sql_where = format!("WHERE {} IN ('{vals}')", columns[0]);
// Define the default_sql here
let default_sql = format!("SELECT {} FROM \"{stream_name}\"", TIMESTAMP_COL_NAME);
let default_sql = format!("SELECT {TIMESTAMP_COL_NAME} FROM \"{stream_name}\"");
query.sql = format!("{} {}", default_sql, sql_where);
query.sql = format!("{default_sql} {sql_where}");
let can_use_distinct_stream = can_use_distinct_stream(
org_id,
@ -917,13 +916,13 @@ async fn values_v1(
.and_then(|v| base64::decode_url(v.as_ref()).ok())
.map(|vrl_function| {
if !vrl_function.trim().ends_with('.') {
format!("{} \n .", vrl_function)
format!("{vrl_function} \n .")
} else {
vrl_function
}
});
let default_sql = format!("SELECT {} FROM \"{stream_name}\"", TIMESTAMP_COL_NAME);
let default_sql = format!("SELECT {TIMESTAMP_COL_NAME} FROM \"{stream_name}\"");
let mut query_sql = match query.get("filter") {
None => default_sql,
Some(v) => {
@ -935,7 +934,7 @@ async fn values_v1(
return Ok(MetaHttpResponse::bad_request("Invalid filter format"));
}
let vals = columns[1].split(',').collect::<Vec<_>>().join("','");
format!("{} WHERE {} IN ('{}')", default_sql, columns[0], vals)
format!("{default_sql} WHERE {} IN ('{vals}')", columns[0])
}
}
};
@ -955,14 +954,14 @@ async fn values_v1(
}
};
if let Some(v) = query.get("sql") {
if let Ok(sql) = base64::decode_url(v) {
uses_fn = functions::get_all_transform_keys(org_id)
.await
.iter()
.any(|fn_name| sql.contains(&format!("{}(", fn_name)));
query_sql = sql;
}
if let Some(v) = query.get("sql")
&& let Ok(sql) = base64::decode_url(v)
{
uses_fn = functions::get_all_transform_keys(org_id)
.await
.iter()
.any(|fn_name| sql.contains(&format!("{fn_name}(")));
query_sql = sql;
};
// pick up where clause from sql
@ -1065,7 +1064,7 @@ async fn values_v1(
let sql_where = if where_str.is_empty() {
"".to_string()
} else {
format!("WHERE {}", where_str)
format!("WHERE {where_str}")
};
for field in &fields {
let http_span = http_span.clone();

35
src/handler/http/request/search/multi_streams.rs
View File

@ -161,10 +161,10 @@ pub async fn search_multi(
.as_ref()
.and_then(|v| base64::decode_url(v).ok());
if let Some(vrl_function) = &query_fn {
if !vrl_function.trim().ends_with('.') {
query_fn = Some(format!("{} \n .", vrl_function));
}
if let Some(vrl_function) = &query_fn
&& !vrl_function.trim().ends_with('.')
{
query_fn = Some(format!("{vrl_function} \n ."));
}
let mut range_error = String::new();
@ -306,7 +306,7 @@ pub async fn search_multi(
req.query.query_fn = query_fn.clone();
}
for fn_name in functions::get_all_transform_keys(&org_id).await {
if req.query.sql.contains(&format!("{}(", fn_name)) {
if req.query.sql.contains(&format!("{fn_name}(")) {
req.query.uses_zo_fn = true;
break;
}
@ -451,24 +451,23 @@ pub async fn search_multi(
log::error!("search error: {:?}", err);
multi_res.function_error =
vec![multi_res.function_error.join(", "), err.to_string()];
if let errors::Error::ErrorCode(code) = err {
if let errors::ErrorCodes::SearchCancelQuery(_) = code {
return Ok(HttpResponse::TooManyRequests().json(
meta::http::HttpResponse::error_code_with_trace_id(
&code,
Some(trace_id),
),
));
}
if let errors::Error::ErrorCode(code) = err
&& let errors::ErrorCodes::SearchCancelQuery(_) = code
{
return Ok(HttpResponse::TooManyRequests().json(
meta::http::HttpResponse::error_code_with_trace_id(&code, Some(trace_id)),
));
}
}
}
}
let mut report_function_usage = false;
multi_res.hits = if query_fn.is_some() && per_query_resp {
multi_res.hits = if let Some(input_fn) = query_fn.as_ref()
&& per_query_resp
{
// compile vrl function & apply the same before returning the response
let mut input_fn = query_fn.unwrap().trim().to_string();
let mut input_fn = input_fn.trim().to_string();
let apply_over_hits = RESULT_ARRAY.is_match(&input_fn);
if apply_over_hits {
@ -823,7 +822,7 @@ pub async fn around_multi(
let mut around_sqls = stream_names
.iter()
.map(|name| format!("SELECT * FROM \"{}\" ", name))
.map(|name| format!("SELECT * FROM \"{name}\" "))
.collect::<Vec<String>>();
if let Some(v) = query.get("sql") {
let sqls = v.split(',').collect::<Vec<&str>>();
@ -845,7 +844,7 @@ pub async fn around_multi(
..Default::default()
};
for (i, stream_name) in stream_names.iter().enumerate() {
let trace_id = format!("{}-{}", trace_id, i);
let trace_id = format!("{trace_id}-{i}");
let search_res = super::around::around(
&trace_id,
http_span.clone(),

56
src/handler/http/request/search/search_inspector.rs
View File

@ -157,8 +157,7 @@ pub async fn get_search_profile(
let mut req: config::meta::search::Request = config::meta::search::Request {
query: Query {
sql: format!(
"SELECT _timestamp, events FROM default WHERE trace_id = '{}' ORDER BY start_time",
query_trace_id
"SELECT _timestamp, events FROM default WHERE trace_id = '{query_trace_id}' ORDER BY start_time"
),
start_time,
end_time,
@ -185,8 +184,7 @@ pub async fn get_search_profile(
{
req.query.start_time = req.query.end_time - max_query_range * 3600 * 1_000_000;
range_error = format!(
"Query duration is modified due to query range restriction of {} hours",
max_query_range
"Query duration is modified due to query range restriction of {max_query_range} hours"
);
}
}
@ -224,7 +222,7 @@ pub async fn get_search_profile(
if !is_root_user(&user_id) {
let user =
match USERS
.get(&format!("{org_id}/{}", user_id))
.get(&format!("{org_id}/{user_id}"))
.and_then(|user_record| {
DBUser::from(&(user_record.clone())).get_user(org_id.clone())
}) {
@ -289,33 +287,33 @@ pub async fn get_search_profile(
};
for hit in res.hits {
if let Some(events_str) = hit.get("events") {
if let Ok(parsed_events) = serde_json::from_str::<Vec<SearchInspectorEvent>>(
if let Some(events_str) = hit.get("events")
&& let Ok(parsed_events) = serde_json::from_str::<Vec<SearchInspectorEvent>>(
events_str.as_str().unwrap_or("[]"),
) {
let mut inspectors = vec![];
let _: Vec<_> = parsed_events
.into_iter()
.map(|event| {
if let Some(mut fields) =
extract_search_inspector_fields(event.name.as_str())
{
if fields.component == Some("summary".to_string()) {
si.sql = fields.sql.unwrap();
let time_range = fields.time_range.unwrap_or_default();
si.start_time = time_range.0;
si.end_time = time_range.1;
si.total_duration = fields.duration.unwrap_or_default();
} else {
fields.timestamp = Some(event._timestamp.to_string());
inspectors.push(fields);
}
)
{
let mut inspectors = vec![];
let _: Vec<_> = parsed_events
.into_iter()
.map(|event| {
if let Some(mut fields) =
extract_search_inspector_fields(event.name.as_str())
{
if fields.component == Some("summary".to_string()) {
si.sql = fields.sql.unwrap();
let time_range = fields.time_range.unwrap_or_default();
si.start_time = time_range.0;
si.end_time = time_range.1;
si.total_duration = fields.duration.unwrap_or_default();
} else {
fields.timestamp = Some(event._timestamp.to_string());
inspectors.push(fields);
}
})
.collect();
}
})
.collect();
events.extend(inspectors);
}
events.extend(inspectors);
}
}

8
src/handler/http/request/search/search_job.rs
View File

@ -643,10 +643,10 @@ async fn cancel_job_inner(
// 3. use job_id to make background_partition_job cancel
let status = status.unwrap();
if status == 1 {
if let Err(e) = cancel_partition_job(job_id).await {
return Ok(MetaHttpResponse::bad_request(e));
}
if status == 1
&& let Err(e) = cancel_partition_job(job_id).await
{
return Ok(MetaHttpResponse::bad_request(e));
}
// 4. use cancel query function to cancel the query

5
src/handler/http/request/service_accounts/mod.rs
View File

@ -166,7 +166,7 @@ pub async fn update(
Err(e) => {
return Err(Error::new(
ErrorKind::InvalidInput,
format!("Invalid query string: {}", e),
format!("Invalid query string: {e}"),
));
}
};
@ -176,8 +176,7 @@ pub async fn update(
"true" => true,
"false" => false,
_ => {
return Err(Error::new(
ErrorKind::Other,
return Err(Error::other(
" 'rotateToken' query param with value 'true' or 'false' allowed",
));
}

14
src/handler/http/request/short_url/mod.rs
View File

@ -117,13 +117,13 @@ pub async fn retrieve(
let original_url = short_url::retrieve(&short_id).await;
// Check if type=ui for JSON response
if let Some(ref type_param) = query.type_param {
if type_param == "ui" {
if let Some(url) = original_url {
return Ok(HttpResponse::Ok().json(url));
} else {
return Ok(HttpResponse::NotFound().finish());
}
if let Some(ref type_param) = query.type_param
&& type_param == "ui"
{
if let Some(url) = original_url {
return Ok(HttpResponse::Ok().json(url));
} else {
return Ok(HttpResponse::NotFound().finish());
}
}

22
src/handler/http/request/status/mod.rs
View File

@ -62,7 +62,6 @@ use {
o2_openfga::config::{
get_config as get_openfga_config, refresh_config as refresh_openfga_config,
},
std::io::ErrorKind,
};
use crate::{
@ -482,7 +481,7 @@ pub async fn redirect(req: HttpRequest) -> Result<HttpResponse, Error> {
let code = match query.get("code") {
Some(code) => code,
None => {
return Err(Error::new(ErrorKind::Other, "no code in request"));
return Err(Error::other("no code in request"));
}
};
let mut audit_message = AuditMessage {
@ -510,7 +509,7 @@ pub async fn redirect(req: HttpRequest) -> Result<HttpResponse, Error> {
// Bad Request
audit_message.response_meta.http_response_code = 400;
audit(audit_message).await;
return Err(Error::new(ErrorKind::Other, "invalid state in request"));
return Err(Error::other("invalid state in request"));
}
},
@ -518,7 +517,7 @@ pub async fn redirect(req: HttpRequest) -> Result<HttpResponse, Error> {
// Bad Request
audit_message.response_meta.http_response_code = 400;
audit(audit_message).await;
return Err(Error::new(ErrorKind::Other, "no state in request"));
return Err(Error::other("no state in request"));
}
};
@ -541,15 +540,14 @@ pub async fn redirect(req: HttpRequest) -> Result<HttpResponse, Error> {
match token_ver {
Ok(res) => {
// check for service accounts , do not to allow login
if let Some(db_user) = db::user::get_user_by_email(&res.0.user_email).await {
if db_user
if let Some(db_user) = db::user::get_user_by_email(&res.0.user_email).await
&& db_user
.organizations
.iter()
.any(|org| org.role.eq(&UserRole::ServiceAccount))
{
return Ok(HttpResponse::Unauthorized()
.json("Service accounts are not allowed to login".to_string()));
}
{
return Ok(HttpResponse::Unauthorized()
.json("Service accounts are not allowed to login".to_string()));
}
audit_message.user_email = res.0.user_email.clone();
@ -590,7 +588,7 @@ pub async fn redirect(req: HttpRequest) -> Result<HttpResponse, Error> {
// store session_id in cluster co-ordinator
let _ = crate::service::session::set_session(&session_id, &access_token).await;
let access_token = format!("session {}", session_id);
let access_token = format!("session {session_id}");
let tokens = json::to_string(&AuthTokens {
access_token,
@ -693,7 +691,7 @@ async fn refresh_token_with_dex(req: actix_web::HttpRequest) -> HttpResponse {
// store session_id in cluster co-ordinator
let _ = crate::service::session::set_session(&session_id, &access_token).await;
let access_token = format!("session {}", session_id);
let access_token = format!("session {session_id}");
let tokens = json::to_string(&AuthTokens {
access_token,

32
src/handler/http/request/stream/mod.rs
View File

@ -13,10 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
cmp::Ordering,
io::{Error, ErrorKind},
};
use std::{cmp::Ordering, io::Error};
use actix_web::{
HttpRequest, HttpResponse, Responder, delete, get, http, http::StatusCode, post, put, web,
@ -95,10 +92,10 @@ async fn schema(
}
// filter by keyword
if let Some(keyword) = query.get("keyword") {
if !keyword.is_empty() {
schema.schema.retain(|f| f.name.contains(keyword));
}
if let Some(keyword) = query.get("keyword")
&& !keyword.is_empty()
{
schema.schema.retain(|f| f.name.contains(keyword));
}
// set total fields
@ -160,7 +157,7 @@ async fn settings(
return Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST,
format!("Stream type '{}' not allowed", stream_type),
format!("Stream type '{stream_type}' not allowed"),
)),
);
}
@ -205,7 +202,7 @@ async fn update_settings(
return Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST,
format!("Stream type '{}' not allowed", stream_type),
format!("Stream type '{stream_type}' not allowed"),
)),
);
}
@ -221,7 +218,7 @@ async fn update_settings(
if cfg.common.inverted_index_old_format && stream_type == StreamType::Logs {
stream_name.to_string()
} else {
format!("{}_{}", stream_name, stream_type)
format!("{stream_name}_{stream_type}")
};
if infra::schema::get(&org_id, &index_stream_name, StreamType::Index)
.await
@ -377,8 +374,7 @@ async fn list(org_id: web::Path<String>, req: HttpRequest) -> impl Responder {
"true" => true,
"false" => false,
_ => {
return Err(Error::new(
ErrorKind::Other,
return Err(Error::other(
" 'fetchSchema' query param with value 'true' or 'false' allowed",
));
}
@ -426,10 +422,10 @@ async fn list(org_id: web::Path<String>, req: HttpRequest) -> impl Responder {
.await;
// filter by keyword
if let Some(keyword) = query.get("keyword") {
if !keyword.is_empty() {
indices.retain(|s| s.name.contains(keyword));
}
if let Some(keyword) = query.get("keyword")
&& !keyword.is_empty()
{
indices.retain(|s| s.name.contains(keyword));
}
// sort by
@ -547,7 +543,7 @@ async fn delete_stream_cache(
let path = if stream_name.eq("_all") {
org_id
} else {
format!("{}/{}/{}", org_id, stream_type, stream_name)
format!("{org_id}/{stream_type}/{stream_name}")
};
match crate::service::search::cluster::cacher::delete_cached_results(path).await {

11
src/handler/http/request/traces/mod.rs
View File

@ -228,7 +228,7 @@ pub async fn get_latest_traces(
}
let max_query_range = crate::common::utils::stream::get_max_query_range(
&[stream_name.clone()],
std::slice::from_ref(&stream_name),
org_id.as_str(),
&user_id,
StreamType::Traces,
@ -238,8 +238,7 @@ pub async fn get_latest_traces(
if max_query_range > 0 && (end_time - start_time) > max_query_range * 3600 * 1_000_000 {
start_time = end_time - max_query_range * 3600 * 1_000_000;
range_error = format!(
"Query duration is modified due to query range restriction of {} hours",
max_query_range
"Query duration is modified due to query range restriction of {max_query_range} hours"
);
}
@ -249,8 +248,7 @@ pub async fn get_latest_traces(
// search
let query_sql = format!(
"SELECT trace_id, min({}) as zo_sql_timestamp, min(start_time) as trace_start_time, max(end_time) as trace_end_time FROM {stream_name}",
TIMESTAMP_COL_NAME
"SELECT trace_id, min({TIMESTAMP_COL_NAME}) as zo_sql_timestamp, min(start_time) as trace_start_time, max(end_time) as trace_end_time FROM {stream_name}"
);
let query_sql = if filter.is_empty() {
format!("{query_sql} GROUP BY trace_id ORDER BY zo_sql_timestamp DESC")
@ -374,8 +372,7 @@ pub async fn get_latest_traces(
.collect::<Vec<String>>()
.join("','");
let query_sql = format!(
"SELECT {}, trace_id, start_time, end_time, duration, service_name, operation_name, span_status FROM {stream_name} WHERE trace_id IN ('{}') ORDER BY {} ASC",
TIMESTAMP_COL_NAME, trace_ids, TIMESTAMP_COL_NAME,
"SELECT {TIMESTAMP_COL_NAME}, trace_id, start_time, end_time, duration, service_name, operation_name, span_status FROM {stream_name} WHERE trace_id IN ('{trace_ids}') ORDER BY {TIMESTAMP_COL_NAME} ASC"
);
req.query.from = 0;
req.query.size = 9999;

4
src/handler/http/request/users/mod.rs
View File

@ -89,7 +89,7 @@ pub async fn list(
// Check if user has access to get users
if get_openfga_config().enabled
&& check_permissions(
Some(format!("_all_{}", org_id)),
Some(format!("_all_{org_id}")),
&org_id,
&user_email.user_id,
"users",
@ -558,7 +558,7 @@ pub async fn get_auth(_req: HttpRequest) -> Result<HttpResponse, Error> {
audit_unauthorized_error(audit_message).await;
return unauthorized_error(resp);
}
format!("q_auth {}", s)
format!("q_auth {s}")
} else if let Some(auth_header) = _req.headers().get("Authorization") {
match auth_header.to_str() {
Ok(auth_header_str) => auth_header_str.to_string(),

13
src/handler/http/request/ws/session.rs
View File

@ -289,11 +289,11 @@ async fn resolve_enterprise_user_id(
// Cluster mode, try to determine user ID
// First check if we're running without router nodes
let router_nodes = get_cached_online_router_nodes().await;
if let Some(nodes) = router_nodes {
if nodes.is_empty() {
// Single node enterprise deployment
return Some(default_user_id.to_string());
}
if let Some(nodes) = router_nodes
&& nodes.is_empty()
{
// Single node enterprise deployment
return Some(default_user_id.to_string());
}
// Next, try to use user_id from the event
@ -538,8 +538,7 @@ pub async fn send_message(req_id: &str, msg: String) -> Result<(), Error> {
session
} else {
return Err(Error::Message(format!(
"[req_id {}] session not found",
req_id
"[req_id {req_id}] session not found"
)));
};

3
src/handler/http/router/middlewares/encoding/encoder.rs
View File

@ -194,8 +194,7 @@ where
if let Some(fut) = this.fut {
let mut encoder = ready!(Pin::new(fut).poll(cx))
.map_err(|_| {
EncoderError::Io(io::Error::new(
io::ErrorKind::Other,
EncoderError::Io(io::Error::other(
"Blocking task was cancelled unexpectedly",
))
})?

6
src/handler/http/router/mod.rs
View File

@ -227,13 +227,11 @@ async fn proxy(
.request(method, &path.target_url)
.send()
.await
.map_err(|e| {
actix_web::error::ErrorInternalServerError(format!("Request failed: {}", e))
})?;
.map_err(|e| actix_web::error::ErrorInternalServerError(format!("Request failed: {e}")))?;
let status = forwarded_resp.status().as_u16();
let body = forwarded_resp.bytes().await.map_err(|e| {
actix_web::error::ErrorInternalServerError(format!("Failed to read the response: {}", e))
actix_web::error::ErrorInternalServerError(format!("Failed to read the response: {e}"))
})?;
Ok(HttpResponse::build(actix_web::http::StatusCode::from_u16(status).unwrap()).body(body))

24
src/handler/tcp_udp/mod.rs
View File

@ -49,12 +49,12 @@ pub async fn udp_server(socket: UdpSocket) {
if input_str != STOP_SRV {
let _ = syslog::ingest(&input_str, addr).await;
}
if let Ok(val) = udp_receiver_rx.try_recv() {
if !val {
log::warn!("UDP server - received the stop signal, exiting.");
break;
}
};
if let Ok(val) = udp_receiver_rx.try_recv()
&& !val
{
log::warn!("UDP server - received the stop signal, exiting.");
break;
}
}
}
@ -84,12 +84,12 @@ pub async fn tls_tcp_server(listener: TcpListener, tls_acceptor: Option<TlsAccep
}
}
if let Ok(val) = tcp_receiver_rx.try_recv() {
if !val {
log::warn!("TCP server - received the stop signal, exiting.");
drop(listener);
break;
}
if let Ok(val) = tcp_receiver_rx.try_recv()
&& !val
{
log::warn!("TCP server - received the stop signal, exiting.");
drop(listener);
break;
}
}
}

44
src/infra/src/cache/file_data/mod.rs vendored
View File

@ -251,16 +251,12 @@ async fn download_from_storage(
// and retry
if data_len != expected_blob_size {
let msg = if i == DOWNLOAD_RETRY_TIMES - 1 {
format!("after {} retries", DOWNLOAD_RETRY_TIMES)
format!("after {DOWNLOAD_RETRY_TIMES} retries")
} else {
"will retry".to_string()
};
log::warn!(
"download file {} found size mismatch with blob store header, expected: {}, actual: {}, {}",
file,
expected_blob_size,
data_len,
msg
"download file {file} found size mismatch with blob store header, expected: {expected_blob_size}, actual: {data_len}, {msg}",
);
tokio::time::sleep(tokio::time::Duration::from_secs(retry_time)).await;
retry_time *= 2;
@ -361,17 +357,18 @@ pub async fn get_opts(
) -> object_store::Result<bytes::Bytes> {
let cfg = config::get_config();
// get from memory cache
if cfg.memory_cache.enabled {
if let Some(v) = memory::get(file, range.clone()).await {
return Ok(v);
}
if cfg.memory_cache.enabled
&& let Some(v) = memory::get(file, range.clone()).await
{
return Ok(v);
}
// get from disk cache
if cfg.disk_cache.enabled {
if let Some(v) = disk::get(file, range.clone()).await {
return Ok(v);
}
if cfg.disk_cache.enabled
&& let Some(v) = disk::get(file, range.clone()).await
{
return Ok(v);
}
// get from storage
if remote {
return match range {
@ -382,7 +379,7 @@ pub async fn get_opts(
Err(object_store::Error::NotFound {
path: file.to_string(),
source: Box::new(std::io::Error::new(std::io::ErrorKind::NotFound, file)),
source: Box::new(std::io::Error::other(file)),
})
}
@ -393,17 +390,18 @@ pub async fn get_size(account: &str, file: &str) -> object_store::Result<usize>
pub async fn get_size_opts(account: &str, file: &str, remote: bool) -> object_store::Result<usize> {
let cfg = config::get_config();
// get from memory cache
if cfg.memory_cache.enabled {
if let Some(v) = memory::get_size(file).await {
return Ok(v);
}
if cfg.memory_cache.enabled
&& let Some(v) = memory::get_size(file).await
{
return Ok(v);
}
// get from disk cache
if cfg.disk_cache.enabled {
if let Some(v) = disk::get_size(file).await {
return Ok(v);
}
if cfg.disk_cache.enabled
&& let Some(v) = disk::get_size(file).await
{
return Ok(v);
}
// get from storage
if remote {
let meta = crate::storage::head(account, file).await?;

276
src/infra/src/cache/storage.rs vendored
View File

@ -268,3 +268,279 @@ pub async fn get_range(
pub async fn head(account: &str, location: &Path) -> Result<ObjectMeta> {
DEFAULT.head(account, location).await
}
#[cfg(test)]
mod tests {
use std::ops::Range;
use bytes::Bytes;
use super::*;
#[test]
fn test_cache_fs_display() {
let cache_fs = CacheFS {};
assert_eq!(cache_fs.to_string(), "CacheFS");
}
#[test]
fn test_cache_fs_new_store() {
let store = CacheFS::new_store();
assert_eq!(store.to_string(), "CacheFS");
}
#[tokio::test]
async fn test_cache_fs_put() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let payload = PutPayload::from(Bytes::from("test data"));
let result = cache_fs.put("default", &location, payload).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_put_opts() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let payload = PutPayload::from(Bytes::from("test data"));
let opts = PutOptions::default();
let result = cache_fs.put_opts("default", &location, payload, opts).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_put_multipart() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let result = cache_fs.put_multipart("default", &location).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_put_multipart_opts() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let opts = PutMultipartOpts::default();
let result = cache_fs
.put_multipart_opts("default", &location, opts)
.await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_get_with_cache_hit() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
// This test would require setting up cache data first
// For now, we test the basic structure
let result = cache_fs.get("default", &location).await;
// The result depends on whether the file exists in cache or storage
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_cache_fs_get_opts_with_cache_hit() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let options = GetOptions::default();
let result = cache_fs.get_opts("default", &location, options).await;
// The result depends on whether the file exists in cache or storage
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_cache_fs_get_range_invalid() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let range = Range { start: 10, end: 5 }; // Invalid range
let result = cache_fs.get_range("default", &location, range).await;
assert!(result.is_err());
// Should return a BadRange error
assert!(matches!(result.unwrap_err(), Error::Generic { .. }));
}
#[tokio::test]
async fn test_cache_fs_get_ranges() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let ranges = vec![Range { start: 0, end: 10 }, Range { start: 10, end: 20 }];
let result = cache_fs.get_ranges("default", &location, &ranges).await;
// The result depends on whether the file exists in cache
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_cache_fs_head_with_cache_hit() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let result = cache_fs.head("default", &location).await;
// The result depends on whether the file exists in cache or storage
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_cache_fs_delete() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
let result = cache_fs.delete("default", &location).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_delete_stream() {
let cache_fs = CacheFS {};
let locations = futures::stream::once(async { Ok(Path::from("test/file.txt")) }).boxed();
let mut result_stream = cache_fs.delete_stream("default", locations);
let result = result_stream.next().await;
assert!(result.is_some());
let result = result.unwrap();
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented { .. }));
}
#[tokio::test]
async fn test_cache_fs_copy() {
let cache_fs = CacheFS {};
let from = Path::from("test/from.txt");
let to = Path::from("test/to.txt");
let result = cache_fs.copy("default", &from, &to).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_rename() {
let cache_fs = CacheFS {};
let from = Path::from("test/from.txt");
let to = Path::from("test/to.txt");
let result = cache_fs.rename("default", &from, &to).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_copy_if_not_exists() {
let cache_fs = CacheFS {};
let from = Path::from("test/from.txt");
let to = Path::from("test/to.txt");
let result = cache_fs.copy_if_not_exists("default", &from, &to).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
#[tokio::test]
async fn test_cache_fs_rename_if_not_exists() {
let cache_fs = CacheFS {};
let from = Path::from("test/from.txt");
let to = Path::from("test/to.txt");
let result = cache_fs.rename_if_not_exists("default", &from, &to).await;
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotImplemented));
}
// Test the public functions that use the DEFAULT instance
#[tokio::test]
async fn test_get_function() {
let path = Path::from("test/file.txt");
let result = get("default", &path).await;
// The result depends on whether the file exists in cache or storage
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_get_opts_function() {
let path = Path::from("test/file.txt");
let options = GetOptions::default();
let result = get_opts("default", &path, options).await;
// The result depends on whether the file exists in cache or storage
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_get_range_function() {
let location = Path::from("test/file.txt");
let range = Range { start: 0, end: 10 };
let result = get_range("default", &location, range).await;
// The result depends on whether the file exists in cache
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
#[tokio::test]
async fn test_head_function() {
let location = Path::from("test/file.txt");
let result = head("default", &location).await;
// The result depends on whether the file exists in cache or storage
// This is a basic test to ensure the function doesn't panic
assert!(result.is_ok() || result.is_err());
}
// Integration test for cache behavior
#[tokio::test]
async fn test_cache_fs_integration() {
let cache_fs = CacheFS {};
let location = Path::from("integration/test.txt");
// Test that the cache FS properly delegates to underlying storage
// when cache is not available
let get_result = cache_fs.get("default", &location).await;
let head_result = cache_fs.head("default", &location).await;
// Both should either succeed (if file exists in storage) or fail appropriately
// This tests the integration between cache and storage layers
assert!(get_result.is_ok() || get_result.is_err());
assert!(head_result.is_ok() || head_result.is_err());
}
// Test error handling for malformed paths
#[tokio::test]
async fn test_cache_fs_malformed_path() {
let cache_fs = CacheFS {};
let location = Path::from(""); // Empty path
let result = cache_fs.get("default", &location).await;
// Should handle empty paths gracefully
assert!(result.is_ok() || result.is_err());
}
// Test with different account names
#[tokio::test]
async fn test_cache_fs_different_accounts() {
let cache_fs = CacheFS {};
let location = Path::from("test/file.txt");
// Test with different account names
let result1 = cache_fs.get("account1", &location).await;
let result2 = cache_fs.get("account2", &location).await;
// Both should handle different accounts appropriately
assert!(result1.is_ok() || result1.is_err());
assert!(result2.is_ok() || result2.is_err());
}
}

42
src/infra/src/db/etcd.rs
View File

@ -163,12 +163,11 @@ impl super::Db for Etcd {
Ok(v) => v,
Err(e) => {
return Err(Error::Message(format!(
"dist_lock key: {}, acquire error: {}",
lock_key, e
"dist_lock key: {lock_key}, acquire error: {e}",
)));
}
};
log::info!("Acquired lock for cluster key: {}", lock_key);
log::info!("Acquired lock for cluster key: {lock_key}");
// get value and update
let value = self.get_key_value(key).await.ok();
@ -178,26 +177,25 @@ impl super::Db for Etcd {
Err(e) => Err(e),
Ok(None) => Ok(()),
Ok(Some((value, new_value))) => {
if let Some(value) = value {
if let Err(e) = self.put(&old_key.unwrap(), value, need_watch, None).await {
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {}", e);
}
log::info!("Released lock for cluster key: {}", lock_key);
return Err(e);
if let Some(value) = value
&& let Err(e) = self.put(&old_key.unwrap(), value, need_watch, None).await
{
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {e}");
}
log::info!("Released lock for cluster key: {lock_key}");
return Err(e);
}
if let Some((new_key, new_value, new_start_dt)) = new_value {
if let Err(e) = self
if let Some((new_key, new_value, new_start_dt)) = new_value
&& let Err(e) = self
.put(&new_key, new_value, need_watch, new_start_dt)
.await
{
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {}", e);
}
log::info!("Released lock for cluster key: {}", lock_key);
return Err(e);
{
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {e}");
}
log::info!("Released lock for cluster key: {lock_key}");
return Err(e);
}
Ok(())
}
@ -205,9 +203,9 @@ impl super::Db for Etcd {
// release lock
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {}", e);
log::error!("dist_lock unlock err: {e}");
}
log::info!("Released lock for cluster key: {}", lock_key);
log::info!("Released lock for cluster key: {lock_key}");
ret
}
@ -444,12 +442,12 @@ impl super::Db for Etcd {
}
let mut client = get_etcd_client().await.clone();
let opt = etcd_client::WatchOptions::new().with_prefix();
log::debug!("[ETCD:watch] prefix: {}", prefix);
log::debug!("[ETCD:watch] prefix: {prefix}");
let (mut _watcher, mut stream) =
match client.watch(key.clone(), Some(opt.clone())).await {
Ok((watcher, stream)) => (watcher, stream),
Err(e) => {
log::error!("[ETCD:watch] prefix: {}, error: {}", key, e);
log::error!("[ETCD:watch] prefix: {key}, error: {e}");
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
continue;
}

135
src/infra/src/db/mysql.rs
View File

@ -135,8 +135,7 @@ impl super::Db for MysqlDb {
.with_label_values(&["select", "meta", ""])
.inc();
let query = format!(
"SELECT value FROM meta WHERE module = '{}' AND key1 = '{}' AND key2 = '{}' ORDER BY start_dt DESC;",
module, key1, key2
"SELECT value FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND key2 = '{key2}' ORDER BY start_dt DESC;"
);
let value: String = match sqlx::query_scalar(&query).fetch_one(&pool).await {
Ok(v) => v,
@ -239,14 +238,13 @@ impl super::Db for MysqlDb {
) -> Result<()> {
let (module, key1, key2) = super::parse_key(key);
let lock_pool = CLIENT.clone();
let lock_key = format!("get_for_update_{}", key);
let lock_key = format!("get_for_update_{key}");
let lock_id = config::utils::hash::gxhash::new().sum64(&lock_key);
let lock_sql = format!(
"SELECT GET_LOCK('{}', {})",
lock_id,
"SELECT GET_LOCK('{lock_id}', {})",
config::get_config().limit.meta_transaction_lock_timeout
);
let unlock_sql = format!("SELECT RELEASE_LOCK('{}')", lock_id);
let unlock_sql = format!("SELECT RELEASE_LOCK('{lock_id}')");
let mut lock_tx = lock_pool.begin().await?;
DB_QUERY_NUMS.with_label_values(&["get_lock", "", ""]).inc();
match sqlx::query_scalar::<_, i64>(&lock_sql)
@ -256,7 +254,7 @@ impl super::Db for MysqlDb {
Ok(v) => {
if v != 1 {
if let Err(e) = lock_tx.rollback().await {
log::error!("[MYSQL] rollback lock for get_for_update error: {}", e);
log::error!("[MYSQL] rollback lock for get_for_update error: {e}");
}
return Err(Error::from(DbError::DBOperError(
"LockTimeout".to_string(),
@ -266,7 +264,7 @@ impl super::Db for MysqlDb {
}
Err(e) => {
if let Err(e) = lock_tx.rollback().await {
log::error!("[MYSQL] rollback lock for get_for_update error: {}", e);
log::error!("[MYSQL] rollback lock for get_for_update error: {e}");
}
return Err(e.into());
}
@ -277,10 +275,10 @@ impl super::Db for MysqlDb {
Ok(tx) => tx,
Err(e) => {
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
return Err(e.into());
}
@ -306,14 +304,14 @@ impl super::Db for MysqlDb {
None
} else {
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback get_for_update error: {}", e);
log::error!("[MYSQL] rollback get_for_update error: {e}");
}
DB_QUERY_NUMS.with_label_values(&["release_lock", "", ""]).inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
return Err(e.into());
}
@ -338,14 +336,14 @@ impl super::Db for MysqlDb {
None
} else {
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback get_for_update error: {}", e);
log::error!("[MYSQL] rollback get_for_update error: {e}");
}
DB_QUERY_NUMS.with_label_values(&["release_lock", "", ""]).inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
return Err(e.into());
}
@ -358,31 +356,31 @@ impl super::Db for MysqlDb {
let (value, new_value) = match update_fn(value) {
Err(e) => {
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback get_for_update error: {}", e);
log::error!("[MYSQL] rollback get_for_update error: {e}");
}
DB_QUERY_NUMS
.with_label_values(&["release_lock", "", ""])
.inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
return Err(e);
}
Ok(None) => {
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback get_for_update error: {}", e);
log::error!("[MYSQL] rollback get_for_update error: {e}");
}
DB_QUERY_NUMS
.with_label_values(&["release_lock", "", ""])
.inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
return Ok(());
}
@ -417,16 +415,16 @@ impl super::Db for MysqlDb {
};
if let Err(e) = ret {
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback get_for_update error: {}", e);
log::error!("[MYSQL] rollback get_for_update error: {e}");
}
DB_QUERY_NUMS
.with_label_values(&["release_lock", "", ""])
.inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
return Err(e.into());
}
@ -451,7 +449,7 @@ impl super::Db for MysqlDb {
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback get_for_update error: {}", e);
log::error!("[MYSQL] rollback get_for_update error: {e}");
}
DB_QUERY_NUMS
.with_label_values(&["release_lock", "", ""])
@ -474,10 +472,10 @@ impl super::Db for MysqlDb {
.with_label_values(&["release_lock", "", ""])
.inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_for_update error: {}", e);
log::error!("[MYSQL] unlock get_for_update error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_for_update error: {}", e);
log::error!("[MYSQL] commit for unlock get_for_update error: {e}");
}
// event watch
@ -518,7 +516,7 @@ impl super::Db for MysqlDb {
.delete(&key, false, true, start_dt)
.await
{
log::error!("[MYSQL] send event error: {}", e);
log::error!("[MYSQL] send event error: {e}");
}
}
});
@ -529,27 +527,22 @@ impl super::Db for MysqlDb {
let (key1, key2) = (key1.replace("'", "''"), key2.replace("'", "''"));
let sql = if with_prefix {
if key1.is_empty() {
format!(r#"DELETE FROM meta WHERE module = '{}';"#, module)
format!(r#"DELETE FROM meta WHERE module = '{module}';"#)
} else if key2.is_empty() {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}';"#,
module, key1
)
format!(r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}';"#)
} else {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}' AND (key2 = '{}' OR key2 LIKE '{}/%');"#,
module, key1, key2, key2
r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND (key2 = '{key2}' OR key2 LIKE '{key2}/%');"#
)
}
} else {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}' AND key2 = '{}';"#,
module, key1, key2
r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND key2 = '{key2}';"#
)
};
let sql = if let Some(start_dt) = start_dt {
sql.replace(';', &format!(" AND start_dt = {};", start_dt))
sql.replace(';', &format!(" AND start_dt = {start_dt};"))
} else {
sql
};
@ -567,15 +560,15 @@ impl super::Db for MysqlDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -599,16 +592,16 @@ impl super::Db for MysqlDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, '' AS value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
.with_label_values(&["select", "meta", ""])
@ -647,19 +640,16 @@ impl super::Db for MysqlDb {
let mut sql =
"SELECT id, module, key1, key2, start_dt, value AS value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!(
"{} AND start_dt >= {} AND start_dt <= {}",
sql, min_dt, max_dt
);
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} AND start_dt >= {min_dt} AND start_dt <= {max_dt}");
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
.with_label_values(&["select", "meta", ""])
@ -677,13 +667,13 @@ impl super::Db for MysqlDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT COUNT(*) AS num FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -777,18 +767,17 @@ async fn add_start_dt_column() -> Result<()> {
sqlx::query(r#"ALTER TABLE meta ADD COLUMN start_dt BIGINT NOT NULL DEFAULT 0;"#)
.execute(&mut *tx)
.await
&& !e.to_string().contains("Duplicate column name")
{
if !e.to_string().contains("Duplicate column name") {
// Check for the specific MySQL error code for duplicate column
log::error!("[MYSQL] Unexpected error in adding column: {}", e);
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] Error in rolling back transaction: {}", e);
}
return Err(e.into());
// Check for the specific MySQL error code for duplicate column
log::error!("[MYSQL] Unexpected error in adding column: {e}");
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] Error in rolling back transaction: {e}");
}
return Err(e.into());
}
if let Err(e) = tx.commit().await {
log::info!("[MYSQL] Error in committing transaction: {}", e);
log::info!("[MYSQL] Error in committing transaction: {e}");
return Err(e.into());
};
@ -818,10 +807,7 @@ async fn create_meta_backup() -> Result<()> {
.await
{
if let Err(e) = tx.rollback().await {
log::error!(
"[MYSQL] rollback create table meta_backup_20240330 error: {}",
e
);
log::error!("[MYSQL] rollback create table meta_backup_20240330 error: {e}");
}
return Err(e.into());
}
@ -844,17 +830,14 @@ async fn create_meta_backup() -> Result<()> {
.await
{
if let Err(e) = tx.rollback().await {
log::error!(
"[MYSQL] rollback insert into meta_backup_20240330 error: {}",
e
);
log::error!("[MYSQL] rollback insert into meta_backup_20240330 error: {e}");
}
return Err(e.into());
}
}
if let Err(e) = tx.commit().await {
log::error!("[MYSQL] commit transaction error: {}", e);
log::error!("[MYSQL] commit transaction error: {e}");
return Err(e.into());
}
@ -907,7 +890,7 @@ pub async fn delete_index(idx_name: &str, table: &str) -> Result<()> {
}
log::info!("[MYSQL] deleting index {} on table {}", idx_name, table);
DB_QUERY_NUMS.with_label_values(&["drop", table, ""]).inc();
let sql = format!("DROP INDEX {} ON {};", idx_name, table);
let sql = format!("DROP INDEX {idx_name} ON {table};");
let start = std::time::Instant::now();
sqlx::query(&sql).execute(&client).await?;
let time = start.elapsed().as_secs_f64();

103
src/infra/src/db/nats.rs
View File

@ -56,7 +56,7 @@ pub async fn init_nats_client(nats_event_sender: mpsc::Sender<async_nats::Event>
NATS_CLIENT
.set(client)
.map_err(|e| Error::Message(format!("[NATS:init] failed to set global client: {}", e)))
.map_err(|e| Error::Message(format!("[NATS:init] failed to set global client: {e}")))
}
pub async fn get_nats_client() -> Result<Client> {
@ -78,7 +78,7 @@ async fn get_bucket_by_key<'a>(
let key = key.trim_start_matches('/');
let bucket_name = key.split('/').next().unwrap();
let mut bucket = jetstream::kv::Config {
bucket: format!("{}{}", prefix, bucket_name),
bucket: format!("{prefix}{bucket_name}"),
num_replicas: cfg.nats.replicas,
history: cfg.nats.history,
..Default::default()
@ -90,8 +90,7 @@ async fn get_bucket_by_key<'a>(
}
let kv = jetstream.create_key_value(bucket).await.map_err(|e| {
Error::Message(format!(
"[NATS:get_bucket_by_key] create jetstream kv error: {}",
e
"[NATS:get_bucket_by_key] create jetstream kv error: {e}"
))
})?;
Ok((kv, key.trim_start_matches(bucket_name)))
@ -122,13 +121,13 @@ impl NatsDb {
if let Some(v) = bucket
.get(&en_key)
.await
.map_err(|e| Error::Message(format!("[NATS:get_key_value] bucket.get error: {}", e)))?
.map_err(|e| Error::Message(format!("[NATS:get_key_value] bucket.get error: {e}")))?
{
return Ok((key.to_string(), v));
}
let keys = keys(&bucket, new_key).await.map_err(|e| {
Error::Message(format!("[NATS:get_key_value] bucket.keys error: {}", e))
})?;
let keys = keys(&bucket, new_key)
.await
.map_err(|e| Error::Message(format!("[NATS:get_key_value] bucket.keys error: {e}")))?;
if keys.is_empty() {
return Err(Error::from(DbError::KeyNotExists(key.to_string())));
}
@ -137,7 +136,7 @@ impl NatsDb {
match bucket
.get(&en_key)
.await
.map_err(|e| Error::Message(format!("[NATS:get_key_value] bucket.get error: {}", e)))?
.map_err(|e| Error::Message(format!("[NATS:get_key_value] bucket.get error: {e}")))?
{
None => Err(Error::from(DbError::KeyNotExists(key.to_string()))),
Some(v) => {
@ -183,13 +182,13 @@ impl super::Db for NatsDb {
if let Some(v) = bucket
.get(&key)
.await
.map_err(|e| Error::Message(format!("[NATS:get] bucket.get error: {}", e)))?
.map_err(|e| Error::Message(format!("[NATS:get] bucket.get error: {e}")))?
{
return Ok(v);
}
let keys = keys(&bucket, new_key)
.await
.map_err(|e| Error::Message(format!("[NATS:get] bucket.keys error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:get] bucket.keys error: {e}")))?;
if keys.is_empty() {
return Err(Error::from(DbError::KeyNotExists(key.to_string())));
}
@ -197,7 +196,7 @@ impl super::Db for NatsDb {
match bucket
.get(key)
.await
.map_err(|e| Error::Message(format!("[NATS:get] bucket.get error: {}", e)))?
.map_err(|e| Error::Message(format!("[NATS:get] bucket.get error: {e}")))?
{
None => Err(Error::from(DbError::KeyNotExists(key.to_string()))),
Some(v) => Ok(v),
@ -221,7 +220,7 @@ impl super::Db for NatsDb {
_ = bucket
.put(&key, value)
.await
.map_err(|e| Error::Message(format!("[NATS:put] bucket.put error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:put] bucket.put error: {e}")))?;
Ok(())
}
@ -238,8 +237,7 @@ impl super::Db for NatsDb {
Ok(v) => v,
Err(e) => {
return Err(Error::Message(format!(
"dist_lock key: {}, acquire error: {}",
lock_key, e
"dist_lock key: {lock_key}, acquire error: {e}"
)));
}
};
@ -253,26 +251,25 @@ impl super::Db for NatsDb {
Err(e) => Err(e),
Ok(None) => Ok(()),
Ok(Some((value, new_value))) => {
if let Some(value) = value {
if let Err(e) = self.put(&old_key.unwrap(), value, need_watch, None).await {
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {}", e);
}
log::info!("Released lock for cluster key: {}", lock_key);
return Err(e);
if let Some(value) = value
&& let Err(e) = self.put(&old_key.unwrap(), value, need_watch, None).await
{
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {e}");
}
log::info!("Released lock for cluster key: {lock_key}");
return Err(e);
}
if let Some((new_key, new_value, new_start_dt)) = new_value {
if let Err(e) = self
if let Some((new_key, new_value, new_start_dt)) = new_value
&& let Err(e) = self
.put(&new_key, new_value, need_watch, new_start_dt)
.await
{
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {}", e);
}
log::info!("Released lock for cluster key: {}", lock_key);
return Err(e);
{
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {e}");
}
log::info!("Released lock for cluster key: {lock_key}");
return Err(e);
}
Ok(())
}
@ -280,9 +277,9 @@ impl super::Db for NatsDb {
// release lock
if let Err(e) = dist_lock::unlock(&locker).await {
log::error!("dist_lock unlock err: {}", e);
log::error!("dist_lock unlock err: {e}");
}
log::info!("Released lock for cluster key: {}", lock_key);
log::info!("Released lock for cluster key: {lock_key}");
ret
}
@ -309,17 +306,17 @@ impl super::Db for NatsDb {
bucket
.purge(key)
.await
.map_err(|e| Error::Message(format!("[NATS:delete] bucket.purge error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:delete] bucket.purge error: {e}")))?;
return Ok(());
}
let keys = keys(&bucket, &new_key)
.await
.map_err(|e| Error::Message(format!("[NATS:delete] bucket.keys error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:delete] bucket.keys error: {e}")))?;
for key in keys {
bucket
.purge(key)
.await
.map_err(|e| Error::Message(format!("[NATS:delete] bucket.purge error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:delete] bucket.purge error: {e}")))?;
}
Ok(())
}
@ -330,7 +327,7 @@ impl super::Db for NatsDb {
let bucket = &bucket;
let keys = keys(bucket, new_key)
.await
.map_err(|e| Error::Message(format!("[NATS:list] bucket.keys error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:list] bucket.keys error: {e}")))?;
if keys.is_empty() {
return Ok(HashMap::new());
}
@ -341,7 +338,7 @@ impl super::Db for NatsDb {
let value = bucket
.get(&encoded_key)
.await
.map_err(|e| Error::Message(format!("[NATS:list] bucket.get error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:list] bucket.get error: {e}")))?;
Ok::<(String, Option<Bytes>), Error>((key, value))
})
.buffer_unordered(get_config().limit.cpu_num)
@ -360,7 +357,7 @@ impl super::Db for NatsDb {
let bucket = &bucket;
let keys = keys(bucket, new_key)
.await
.map_err(|e| Error::Message(format!("[NATS:list_keys] bucket.keys error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:list_keys] bucket.keys error: {e}")))?;
let keys = keys
.into_iter()
.map(|k| bucket_prefix.to_string() + &k)
@ -373,7 +370,7 @@ impl super::Db for NatsDb {
let bucket = &bucket;
let keys = keys(bucket, new_key)
.await
.map_err(|e| Error::Message(format!("[NATS:list_values] bucket.keys error: {}", e)))?;
.map_err(|e| Error::Message(format!("[NATS:list_values] bucket.keys error: {e}")))?;
if keys.is_empty() {
return Ok(vec![]);
}
@ -383,7 +380,7 @@ impl super::Db for NatsDb {
.map(|key| async move {
let encoded_key = key_encode(&key);
let value = bucket.get(&encoded_key).await.map_err(|e| {
Error::Message(format!("[NATS:list_values] bucket.get error: {}", e))
Error::Message(format!("[NATS:list_values] bucket.get error: {e}"))
})?;
Ok::<Option<Bytes>, Error>(value)
})
@ -410,8 +407,7 @@ impl super::Db for NatsDb {
let bucket = &bucket;
let keys = keys(bucket, new_key).await.map_err(|e| {
Error::Message(format!(
"[NATS:list_values_by_start_dt] bucket.keys error: {}",
e
"[NATS:list_values_by_start_dt] bucket.keys error: {e}"
))
})?;
let keys = keys
@ -441,8 +437,7 @@ impl super::Db for NatsDb {
.unwrap_or_default();
let value = bucket.get(&encoded_key).await.map_err(|e| {
Error::Message(format!(
"[NATS:list_values_by_start_dt] bucket.get error: {}",
e
"[NATS:list_values_by_start_dt] bucket.get error: {e}"
))
})?;
Ok::<Option<(i64, Bytes)>, Error>(value.map(|value| (start_dt, value)))
@ -472,7 +467,7 @@ impl super::Db for NatsDb {
let (bucket, new_key) = match get_bucket_by_key(&self_prefix, &prefix).await {
Ok(v) => v,
Err(e) => {
log::error!("[NATS:watch] prefix: {}, get bucket error: {}", prefix, e);
log::error!("[NATS:watch] prefix: {prefix}, get bucket error: {e}");
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
continue;
}
@ -619,10 +614,10 @@ async fn keys(kv: &jetstream::kv::Store, prefix: &str) -> Result<Vec<String>> {
.await?;
let mut keys = Vec::new();
if let Ok(info) = consumer.info().await {
if info.num_pending == 0 {
return Ok(keys);
}
if let Ok(info) = consumer.info().await
&& info.num_pending == 0
{
return Ok(keys);
}
let mut messages = consumer.messages().await?;
while let Ok(Some(message)) = messages.try_next().await {
@ -636,10 +631,10 @@ async fn keys(kv: &jetstream::kv::Store, prefix: &str) -> Result<Vec<String>> {
if key.starts_with(prefix) {
keys.push(key);
}
if let Ok(info) = message.info() {
if info.pending == 0 {
break;
}
if let Ok(info) = message.info()
&& info.pending == 0
{
break;
}
}
keys.sort();
@ -664,7 +659,7 @@ pub(crate) struct Locker {
impl Locker {
pub(crate) fn new(key: &str) -> Self {
Self {
key: format!("/locker{}", key),
key: format!("/locker{key}"),
lock_id: ider::uuid(),
state: Arc::new(AtomicU8::new(0)),
tx: None,

75
src/infra/src/db/postgres.rs
View File

@ -124,8 +124,7 @@ impl super::Db for PostgresDb {
.with_label_values(&["select", "meta", ""])
.inc();
let query = format!(
"SELECT value FROM meta WHERE module = '{}' AND key1 = '{}' AND key2 = '{}' ORDER BY start_dt DESC;",
module, key1, key2
"SELECT value FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND key2 = '{key2}' ORDER BY start_dt DESC;"
);
let value: String = match sqlx::query_scalar(&query).fetch_one(&pool).await {
Ok(v) => v,
@ -229,7 +228,7 @@ impl super::Db for PostgresDb {
let (module, key1, key2) = super::parse_key(key);
let pool = CLIENT.clone();
let mut tx = pool.begin().await?;
let lock_key = format!("get_for_update_{}", key);
let lock_key = format!("get_for_update_{key}");
let lock_id = config::utils::hash::gxhash::new().sum64(&lock_key);
let lock_id = if lock_id > i64::MAX as u64 {
(lock_id >> 1) as i64
@ -429,27 +428,22 @@ impl super::Db for PostgresDb {
let (key1, key2) = (key1.replace("'", "''"), key2.replace("'", "''"));
let sql = if with_prefix {
if key1.is_empty() {
format!(r#"DELETE FROM meta WHERE module = '{}';"#, module)
format!(r#"DELETE FROM meta WHERE module = '{module}';"#)
} else if key2.is_empty() {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}';"#,
module, key1
)
format!(r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}';"#)
} else {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}' AND (key2 = '{}' OR key2 LIKE '{}/%');"#,
module, key1, key2, key2
r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND (key2 = '{key2}' OR key2 LIKE '{key2}/%');"#
)
}
} else {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}' AND key2 = '{}';"#,
module, key1, key2
r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND key2 = '{key2}';"#
)
};
let sql = if let Some(start_dt) = start_dt {
sql.replace(';', &format!(" AND start_dt = {};", start_dt))
sql.replace(';', &format!(" AND start_dt = {start_dt};"))
} else {
sql
};
@ -467,15 +461,15 @@ impl super::Db for PostgresDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -499,15 +493,15 @@ impl super::Db for PostgresDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, '' AS value FROM meta ".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
.with_label_values(&["select", "meta", ""])
@ -545,19 +539,16 @@ impl super::Db for PostgresDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!(
"{} AND start_dt >= {} AND start_dt <= {}",
sql, min_dt, max_dt
);
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} AND start_dt >= {min_dt} AND start_dt <= {max_dt}");
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -576,13 +567,13 @@ impl super::Db for PostgresDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT COUNT(*) AS num FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -679,9 +670,9 @@ async fn add_start_dt_column() -> Result<()> {
.execute(&mut *tx)
.await
{
log::error!("[POSTGRES] Error in adding column start_dt: {}", e);
log::error!("[POSTGRES] Error in adding column start_dt: {e}");
if let Err(e) = tx.rollback().await {
log::error!("[POSTGRES] Error in rolling back transaction: {}", e);
log::error!("[POSTGRES] Error in rolling back transaction: {e}");
}
return Err(e.into());
}
@ -712,18 +703,12 @@ async fn create_meta_backup() -> Result<()> {
.await
{
if let Err(e) = tx.rollback().await {
log::error!(
"[POSTGRES] rollback create table meta_backup_20240330 error: {}",
e
);
log::error!("[POSTGRES] rollback create table meta_backup_20240330 error: {e}");
}
return Err(e.into());
}
if let Err(e) = tx.commit().await {
log::error!(
"[POSTGRES] commit create table meta_backup_20240330 error: {}",
e
);
log::error!("[POSTGRES] commit create table meta_backup_20240330 error: {e}");
return Err(e.into());
}
Ok(())
@ -774,8 +759,8 @@ pub async fn delete_index(idx_name: &str, table: &str) -> Result<()> {
}) {
return Ok(());
}
log::info!("[POSTGRES] deleting index {} on table {}", idx_name, table);
let sql = format!("DROP INDEX IF EXISTS {};", idx_name);
log::info!("[POSTGRES] deleting index {idx_name} on table {table}");
let sql = format!("DROP INDEX IF EXISTS {idx_name};");
DB_QUERY_NUMS.with_label_values(&["drop", table, ""]).inc();
let start = std::time::Instant::now();
sqlx::query(&sql).execute(&client).await?;
@ -783,6 +768,6 @@ pub async fn delete_index(idx_name: &str, table: &str) -> Result<()> {
DB_QUERY_TIME
.with_label_values(&["drop_index", table])
.observe(time);
log::info!("[POSTGRES] index {} deleted successfully", idx_name);
log::info!("[POSTGRES] index {idx_name} deleted successfully");
Ok(())
}

101
src/infra/src/db/sqlite.rs
View File

@ -225,8 +225,7 @@ impl super::Db for SqliteDb {
let (module, key1, key2) = super::parse_key(key);
let pool = CLIENT_RO.clone();
let query = format!(
"SELECT value FROM meta WHERE module = '{}' AND key1 = '{}' AND key2 = '{}' ORDER BY start_dt DESC;",
module, key1, key2
"SELECT value FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND key2 = '{key2}' ORDER BY start_dt DESC;"
);
let value: String = match sqlx::query_scalar(&query).fetch_one(&pool).await {
Ok(v) => v,
@ -267,13 +266,13 @@ impl super::Db for SqliteDb {
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback put meta error: {}", e);
log::error!("[SQLITE] rollback put meta error: {e}");
}
return Err(e.into());
}
// need commit it first to avoid the deadlock of insert and update
if let Err(e) = tx.commit().await {
log::error!("[SQLITE] commit put meta error: {}", e);
log::error!("[SQLITE] commit put meta error: {e}");
return Err(e.into());
}
@ -290,12 +289,12 @@ impl super::Db for SqliteDb {
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback put meta error: {}", e);
log::error!("[SQLITE] rollback put meta error: {e}");
}
return Err(e.into());
}
if let Err(e) = tx.commit().await {
log::error!("[SQLITE] commit put meta error: {}", e);
log::error!("[SQLITE] commit put meta error: {e}");
return Err(e.into());
}
@ -303,8 +302,8 @@ impl super::Db for SqliteDb {
drop(client);
// event watch
if need_watch {
if let Err(e) = CHANNEL
if need_watch
&& let Err(e) = CHANNEL
.watch_tx
.clone()
.send(Event::Put(EventData {
@ -313,9 +312,8 @@ impl super::Db for SqliteDb {
start_dt,
}))
.await
{
log::error!("[SQLITE] send event error: {}", e);
}
{
log::error!("[SQLITE] send event error: {e}");
}
Ok(())
@ -349,7 +347,7 @@ impl super::Db for SqliteDb {
if e.to_string().contains("no rows returned") {
None
} else {
return Err(Error::Message(format!("[SQLITE] get_for_update error: {}", e)));
return Err(Error::Message(format!("[SQLITE] get_for_update error: {e}")));
}
}
}
@ -368,7 +366,7 @@ impl super::Db for SqliteDb {
if e.to_string().contains("no rows returned") {
None
} else {
return Err(Error::Message(format!("[SQLITE] get_for_update error: {}", e)));
return Err(Error::Message(format!("[SQLITE] get_for_update error: {e}")));
}
}
}
@ -379,13 +377,13 @@ impl super::Db for SqliteDb {
let (value, new_value) = match update_fn(value) {
Err(e) => {
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback get_for_update error: {}", e);
log::error!("[SQLITE] rollback get_for_update error: {e}");
}
return Err(e);
}
Ok(None) => {
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback get_for_update error: {}", e);
log::error!("[SQLITE] rollback get_for_update error: {e}");
}
return Ok(());
}
@ -414,7 +412,7 @@ impl super::Db for SqliteDb {
};
if let Err(e) = ret {
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback get_for_update error: {}", e);
log::error!("[SQLITE] rollback get_for_update error: {e}");
}
return Err(e.into());
}
@ -436,14 +434,14 @@ impl super::Db for SqliteDb {
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[POSTGRES] rollback get_for_update error: {}", e);
log::error!("[POSTGRES] rollback get_for_update error: {e}");
}
return Err(e.into());
}
}
if let Err(e) = tx.commit().await {
log::error!("[SQLITE] commit get_for_update error: {}", e);
log::error!("[SQLITE] commit get_for_update error: {e}");
return Err(e.into());
}
@ -457,8 +455,8 @@ impl super::Db for SqliteDb {
} else {
None
};
if new_value.is_some() || value.is_some() {
if let Err(e) = CHANNEL
if (new_value.is_some() || value.is_some())
&& let Err(e) = CHANNEL
.watch_tx
.clone()
.send(Event::Put(EventData {
@ -467,9 +465,8 @@ impl super::Db for SqliteDb {
start_dt,
}))
.await
{
log::error!("[SQLITE] send event error: {}", e);
}
{
log::error!("[SQLITE] send event error: {e}");
}
}
@ -513,7 +510,7 @@ impl super::Db for SqliteDb {
}))
.await
{
log::error!("[SQLITE] send event error: {}", e);
log::error!("[SQLITE] send event error: {e}");
}
}
});
@ -524,27 +521,22 @@ impl super::Db for SqliteDb {
let (key1, key2) = (key1.replace("'", "''"), key2.replace("'", "''"));
let sql = if with_prefix {
if key1.is_empty() {
format!(r#"DELETE FROM meta WHERE module = '{}';"#, module)
format!(r#"DELETE FROM meta WHERE module = '{module}';"#)
} else if key2.is_empty() {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}';"#,
module, key1
)
format!(r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}';"#)
} else {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}' AND (key2 = '{}' OR key2 LIKE '{}/%');"#,
module, key1, key2, key2
r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND (key2 = '{key2}' OR key2 LIKE '{key2}/%');"#
)
}
} else {
format!(
r#"DELETE FROM meta WHERE module = '{}' AND key1 = '{}' AND key2 = '{}';"#,
module, key1, key2
r#"DELETE FROM meta WHERE module = '{module}' AND key1 = '{key1}' AND key2 = '{key2}';"#
)
};
let sql = if let Some(start_dt) = start_dt {
sql.replace(';', &format!(" AND start_dt = {};", start_dt))
sql.replace(';', &format!(" AND start_dt = {start_dt};"))
} else {
sql
};
@ -559,15 +551,15 @@ impl super::Db for SqliteDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
let ret = sqlx::query_as::<_, super::MetaRecord>(&sql)
@ -588,16 +580,16 @@ impl super::Db for SqliteDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, '' AS value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
let ret = sqlx::query_as::<_, super::MetaRecord>(&sql)
.fetch_all(&pool)
@ -632,19 +624,16 @@ impl super::Db for SqliteDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT id, module, key1, key2, start_dt, value FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
sql = format!(
"{} AND start_dt >= {} AND start_dt <= {}",
sql, min_dt, max_dt
);
sql = format!("{} ORDER BY start_dt ASC", sql);
sql = format!("{sql} AND start_dt >= {min_dt} AND start_dt <= {max_dt}");
sql = format!("{sql} ORDER BY start_dt ASC");
let pool = CLIENT_RO.clone();
let ret = sqlx::query_as::<_, super::MetaRecord>(&sql)
@ -660,13 +649,13 @@ impl super::Db for SqliteDb {
let (module, key1, key2) = super::parse_key(prefix);
let mut sql = "SELECT COUNT(*) AS num FROM meta".to_string();
if !module.is_empty() {
sql = format!("{} WHERE module = '{}'", sql, module);
sql = format!("{sql} WHERE module = '{module}'");
}
if !key1.is_empty() {
sql = format!("{} AND key1 = '{}'", sql, key1);
sql = format!("{sql} AND key1 = '{key1}'");
}
if !key2.is_empty() {
sql = format!("{} AND (key2 = '{}' OR key2 LIKE '{}/%')", sql, key2, key2);
sql = format!("{sql} AND (key2 = '{key2}' OR key2 LIKE '{key2}/%')");
}
let pool = CLIENT_RO.clone();
@ -838,9 +827,9 @@ pub async fn delete_index(idx_name: &str, table: &str) -> Result<()> {
}) {
return Ok(());
}
log::info!("[SQLITE] deleting index {} on table {}", idx_name, table);
let sql = format!("DROP INDEX IF EXISTS {};", idx_name);
log::info!("[SQLITE] deleting index {idx_name} on table {table}");
let sql = format!("DROP INDEX IF EXISTS {idx_name};");
sqlx::query(&sql).execute(&*client).await?;
log::info!("[SQLITE] index {} deleted successfully", idx_name);
log::info!("[SQLITE] index {idx_name} deleted successfully");
Ok(())
}

1
src/infra/src/errors/mod.rs
View File

@ -21,6 +21,7 @@ pub mod grpc;
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(ThisError, Debug)]
#[allow(clippy::large_enum_variant)]
pub enum Error {
#[error("IoError# {0}")]
IoError(#[from] std::io::Error),

20
src/infra/src/file_list/mod.rs
View File

@ -521,12 +521,12 @@ pub async fn local_cache_gc() -> Result<()> {
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(3600));
interval.tick().await; // the first tick is immediate
loop {
if let Ok(min_id) = get_min_pk_value().await {
if min_id > 0 {
match LOCAL_CACHE.clean_by_min_pk_value(min_id).await {
Ok(_) => log::info!("[file_list] local cache gc done"),
Err(e) => log::error!("[file_list] local cache gc failed: {}", e),
}
if let Ok(min_id) = get_min_pk_value().await
&& min_id > 0
{
match LOCAL_CACHE.clean_by_min_pk_value(min_id).await {
Ok(_) => log::info!("[file_list] local cache gc done"),
Err(e) => log::error!("[file_list] local cache gc failed: {e}"),
}
}
interval.tick().await;
@ -537,10 +537,10 @@ pub async fn local_cache_gc() -> Result<()> {
}
fn validate_time_range(time_range: Option<(i64, i64)>) -> Result<()> {
if let Some((start, end)) = time_range {
if start > end || start == 0 || end == 0 {
return Err(Error::Message("[file_list] invalid time range".to_string()));
}
if let Some((start, end)) = time_range
&& (start > end || start == 0 || end == 0)
{
return Err(Error::Message("[file_list] invalid time range".to_string()));
}
Ok(())
}

102
src/infra/src/file_list/mysql.rs
View File

@ -379,10 +379,11 @@ SELECT min_ts, max_ts, records, original_size, compressed_size, index_size, flat
time_range: Option<(i64, i64)>,
flattened: Option<bool>,
) -> Result<Vec<FileKey>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -438,10 +439,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
date_range: Option<(String, String)>,
) -> Result<Vec<FileKey>> {
if let Some((start, end)) = date_range.as_ref() {
if start.is_empty() && end.is_empty() {
return Ok(Vec::new());
}
if let Some((start, end)) = date_range.as_ref()
&& start.is_empty()
&& end.is_empty()
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -517,10 +519,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result<Vec<super::FileId>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -633,10 +636,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result<Vec<String>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -694,7 +698,7 @@ SELECT date
lock_id,
config::get_config().limit.meta_transaction_lock_timeout
);
let unlock_sql = format!("SELECT RELEASE_LOCK('{}')", lock_id);
let unlock_sql = format!("SELECT RELEASE_LOCK('{lock_id}')");
let mut lock_tx = lock_pool.begin().await?;
DB_QUERY_NUMS.with_label_values(&["get_lock", "", ""]).inc();
match sqlx::query_scalar::<_, i64>(&lock_sql)
@ -954,16 +958,16 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, CAST(COUNT(*) AS SI
"#,
);
if deleted {
sql = format!("{} AND deleted IS TRUE", sql);
sql = format!("{sql} AND deleted IS TRUE");
}
let sql = match pk_value {
None => format!("{} GROUP BY stream", sql),
Some((0, 0)) => format!("{} GROUP BY stream", sql),
None => format!("{sql} GROUP BY stream"),
Some((0, 0)) => format!("{sql} GROUP BY stream"),
Some((min, max)) => {
if deleted {
format!("{} AND id <= {} GROUP BY stream", sql, max)
format!("{sql} AND id <= {max} GROUP BY stream")
} else {
format!("{} AND id > {} AND id <= {} GROUP BY stream", sql, min, max)
format!("{sql} AND id > {min} AND id <= {max} GROUP BY stream")
}
}
};
@ -994,13 +998,12 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, CAST(COUNT(*) AS SI
) -> Result<Vec<(String, StreamStats)>> {
let sql = if stream_type.is_some() && stream_name.is_some() {
format!(
"SELECT * FROM stream_stats WHERE stream = '{}/{}/{}';",
org_id,
"SELECT * FROM stream_stats WHERE stream = '{org_id}/{}/{}';",
stream_type.unwrap(),
stream_name.unwrap()
)
} else {
format!("SELECT * FROM stream_stats WHERE org = '{}';", org_id)
format!("SELECT * FROM stream_stats WHERE org = '{org_id}';")
};
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -1022,8 +1025,7 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, CAST(COUNT(*) AS SI
stream_name: &str,
) -> Result<()> {
let sql = format!(
"DELETE FROM stream_stats WHERE stream = '{}/{}/{}';",
org_id, stream_type, stream_name
"DELETE FROM stream_stats WHERE stream = '{org_id}/{stream_type}/{stream_name}';"
);
let pool = CLIENT.clone();
DB_QUERY_NUMS
@ -1075,13 +1077,13 @@ INSERT INTO stream_stats
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback insert stream stats error: {}", e);
log::error!("[MYSQL] rollback insert stream stats error: {e}");
}
return Err(e.into());
}
}
if let Err(e) = tx.commit().await {
log::error!("[MYSQL] commit set stream stats error: {}", e);
log::error!("[MYSQL] commit set stream stats error: {e}");
return Err(e.into());
}
@ -1110,7 +1112,7 @@ UPDATE stream_stats
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] rollback set stream stats error: {}", e);
log::error!("[MYSQL] rollback set stream stats error: {e}");
}
return Err(e.into());
}
@ -1154,8 +1156,7 @@ UPDATE stream_stats
Err(e) => {
if let Err(e) = tx.rollback().await {
log::error!(
"[MYSQL] rollback set stream stats error for delete file list: {}",
e
"[MYSQL] rollback set stream stats error for delete file list: {e}"
);
}
return Err(e.into());
@ -1170,7 +1171,7 @@ UPDATE stream_stats
// commit
if let Err(e) = tx.commit().await {
log::error!("[MYSQL] commit set stream stats error: {}", e);
log::error!("[MYSQL] commit set stream stats error: {e}");
return Err(e.into());
}
@ -1226,7 +1227,7 @@ UPDATE stream_stats
{
Ok(r) => r,
Err(e) => {
log::error!("[MYSQL] get file list len error: {}", e);
log::error!("[MYSQL] get file list len error: {e}");
return 0;
}
};
@ -1292,11 +1293,10 @@ UPDATE stream_stats
let lock_key = "file_list_jobs:get_pending_jobs";
let lock_id = config::utils::hash::gxhash::new().sum64(lock_key);
let lock_sql = format!(
"SELECT GET_LOCK('{}', {})",
lock_id,
"SELECT GET_LOCK('{lock_id}', {})",
config::get_config().limit.meta_transaction_lock_timeout
);
let unlock_sql = format!("SELECT RELEASE_LOCK('{}')", lock_id);
let unlock_sql = format!("SELECT RELEASE_LOCK('{lock_id}')");
let mut lock_tx = lock_pool.begin().await?;
DB_QUERY_NUMS.with_label_values(&["get_lock", "", ""]).inc();
match sqlx::query_scalar::<_, i64>(&lock_sql)
@ -1306,7 +1306,7 @@ UPDATE stream_stats
Ok(v) => {
if v != 1 {
if let Err(e) = lock_tx.rollback().await {
log::error!("[MYSQL] rollback lock for get_pending_jobs error: {}", e);
log::error!("[MYSQL] rollback lock for get_pending_jobs error: {e}");
}
return Err(Error::from(DbError::DBOperError(
"LockTimeout".to_string(),
@ -1316,7 +1316,7 @@ UPDATE stream_stats
}
Err(e) => {
if let Err(e) = lock_tx.rollback().await {
log::error!("[MYSQL] rollback lock for get_pending_jobs error: {}", e);
log::error!("[MYSQL] rollback lock for get_pending_jobs error: {e}");
}
return Err(e.into());
}
@ -1327,10 +1327,10 @@ UPDATE stream_stats
Ok(tx) => tx,
Err(e) => {
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_pending_jobs error: {}", e);
log::error!("[MYSQL] unlock get_pending_jobs error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_pending_jobs error: {}", e);
log::error!("[MYSQL] commit for unlock get_pending_jobs error: {e}");
}
return Err(e.into());
}
@ -1364,10 +1364,10 @@ SELECT stream, max(id) as id, CAST(COUNT(*) AS SIGNED) AS num
.with_label_values(&["release_lock", "", ""])
.inc();
if let Err(e) = sqlx::query(&unlock_sql).execute(&mut *lock_tx).await {
log::error!("[MYSQL] unlock get_pending_jobs error: {}", e);
log::error!("[MYSQL] unlock get_pending_jobs error: {e}");
}
if let Err(e) = lock_tx.commit().await {
log::error!("[MYSQL] commit for unlock get_pending_jobs error: {}", e);
log::error!("[MYSQL] commit for unlock get_pending_jobs error: {e}");
}
return Err(e.into());
}
@ -2164,15 +2164,15 @@ async fn add_column(table: &str, column: &str, data_type: &str) -> Result<()> {
let alert_sql = format!("ALTER TABLE {table} ADD COLUMN {column} {data_type};");
let mut tx = pool.begin().await?;
if let Err(e) = sqlx::query(&alert_sql).execute(&mut *tx).await {
if !e.to_string().contains("Duplicate column name") {
// Check for the specific MySQL error code for duplicate column
log::error!("[MYSQL] Unexpected error in adding column {column}: {}", e);
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] Error in rolling back transaction: {}", e);
}
return Err(e.into());
if let Err(e) = sqlx::query(&alert_sql).execute(&mut *tx).await
&& !e.to_string().contains("Duplicate column name")
{
// Check for the specific MySQL error code for duplicate column
log::error!("[MYSQL] Unexpected error in adding column {column}: {}", e);
if let Err(e) = tx.rollback().await {
log::error!("[MYSQL] Error in rolling back transaction: {}", e);
}
return Err(e.into());
}
if let Err(e) = tx.commit().await {
log::info!("[MYSQL] Error in committing transaction: {}", e);

51
src/infra/src/file_list/postgres.rs
View File

@ -392,10 +392,11 @@ SELECT min_ts, max_ts, records, original_size, compressed_size, index_size, flat
time_range: Option<(i64, i64)>,
flattened: Option<bool>,
) -> Result<Vec<FileKey>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -450,10 +451,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
date_range: Option<(String, String)>,
) -> Result<Vec<FileKey>> {
if let Some((start, end)) = date_range.as_ref() {
if start.is_empty() && end.is_empty() {
return Ok(Vec::new());
}
if let Some((start, end)) = date_range.as_ref()
&& start.is_empty()
&& end.is_empty()
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -530,10 +532,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result<Vec<super::FileId>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -646,10 +649,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result<Vec<String>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -894,16 +898,16 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, COUNT(*)::BIGINT AS
"#
);
if deleted {
sql = format!("{} AND deleted IS TRUE", sql);
sql = format!("{sql} AND deleted IS TRUE");
}
let sql = match pk_value {
None => format!("{} GROUP BY stream", sql),
Some((0, 0)) => format!("{} GROUP BY stream", sql),
None => format!("{sql} GROUP BY stream"),
Some((0, 0)) => format!("{sql} GROUP BY stream"),
Some((min, max)) => {
if deleted {
format!("{} AND id <= {} GROUP BY stream", sql, max)
format!("{sql} AND id <= {max} GROUP BY stream")
} else {
format!("{} AND id > {} AND id <= {} GROUP BY stream", sql, min, max)
format!("{sql} AND id > {min} AND id <= {max} GROUP BY stream")
}
}
};
@ -940,7 +944,7 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, COUNT(*)::BIGINT AS
stream_name.unwrap()
)
} else {
format!("SELECT * FROM stream_stats WHERE org = '{}';", org_id)
format!("SELECT * FROM stream_stats WHERE org = '{org_id}';")
};
let pool = CLIENT_RO.clone();
DB_QUERY_NUMS
@ -962,8 +966,7 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, COUNT(*)::BIGINT AS
stream_name: &str,
) -> Result<()> {
let sql = format!(
"DELETE FROM stream_stats WHERE stream = '{}/{}/{}';",
org_id, stream_type, stream_name
"DELETE FROM stream_stats WHERE stream = '{org_id}/{stream_type}/{stream_name}';"
);
let pool = CLIENT.clone();
DB_QUERY_NUMS

71
src/infra/src/file_list/sqlite.rs
View File

@ -195,12 +195,12 @@ impl super::FileList for SqliteFileList {
});
if let Err(e) = query_builder.build().execute(&mut *tx).await {
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback file_list_deleted batch add error: {}", e);
log::error!("[SQLITE] rollback file_list_deleted batch add error: {e}");
}
return Err(e.into());
};
if let Err(e) = tx.commit().await {
log::error!("[SQLITE] commit file_list_deleted batch add error: {}", e);
log::error!("[SQLITE] commit file_list_deleted batch add error: {e}");
return Err(e.into());
}
}
@ -349,10 +349,11 @@ SELECT min_ts, max_ts, records, original_size, compressed_size, index_size, flat
time_range: Option<(i64, i64)>,
flattened: Option<bool>,
) -> Result<Vec<FileKey>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -400,10 +401,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
date_range: Option<(String, String)>,
) -> Result<Vec<FileKey>> {
if let Some((start, end)) = date_range.as_ref() {
if start.is_empty() && end.is_empty() {
return Ok(Vec::new());
}
if let Some((start, end)) = date_range.as_ref()
&& start.is_empty()
&& end.is_empty()
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -464,10 +466,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result<Vec<super::FileId>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -572,10 +575,11 @@ SELECT id, account, stream, date, file, deleted, min_ts, max_ts, records, origin
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result<Vec<String>> {
if let Some((start, end)) = time_range {
if start == 0 && end == 0 {
return Ok(Vec::new());
}
if let Some((start, end)) = time_range
&& start == 0
&& end == 0
{
return Ok(Vec::new());
}
let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
@ -729,16 +733,16 @@ SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_nu
"#,
);
if deleted {
sql = format!("{} AND deleted IS TRUE", sql);
sql = format!("{sql} AND deleted IS TRUE");
}
let sql = match pk_value {
None => format!("{} GROUP BY stream", sql),
Some((0, 0)) => format!("{} GROUP BY stream", sql),
None => format!("{sql} GROUP BY stream"),
Some((0, 0)) => format!("{sql} GROUP BY stream"),
Some((min, max)) => {
if deleted {
format!("{} AND id <= {} GROUP BY stream", sql, max)
format!("{sql} AND id <= {max} GROUP BY stream")
} else {
format!("{} AND id > {} AND id <= {} GROUP BY stream", sql, min, max)
format!("{sql} AND id > {min} AND id <= {max} GROUP BY stream")
}
}
};
@ -760,13 +764,12 @@ SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_nu
) -> Result<Vec<(String, StreamStats)>> {
let sql = if stream_type.is_some() && stream_name.is_some() {
format!(
"SELECT * FROM stream_stats WHERE stream = '{}/{}/{}';",
org_id,
"SELECT * FROM stream_stats WHERE stream = '{org_id}/{}/{}';",
stream_type.unwrap(),
stream_name.unwrap()
)
} else {
format!("SELECT * FROM stream_stats WHERE org = '{}';", org_id)
format!("SELECT * FROM stream_stats WHERE org = '{org_id}';")
};
let pool = CLIENT_RO.clone();
let ret = sqlx::query_as::<_, super::StatsRecord>(&sql)
@ -785,8 +788,7 @@ SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_nu
stream_name: &str,
) -> Result<()> {
let sql = format!(
"DELETE FROM stream_stats WHERE stream = '{}/{}/{}';",
org_id, stream_type, stream_name
"DELETE FROM stream_stats WHERE stream = '{org_id}/{stream_type}/{stream_name}';"
);
let client = CLIENT_RW.clone();
let client = client.lock().await;
@ -834,13 +836,13 @@ SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_nu
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback insert stream stats error: {}", e);
log::error!("[SQLITE] rollback insert stream stats error: {e}");
}
return Err(e.into());
}
}
if let Err(e) = tx.commit().await {
log::error!("[SQLITE] commit set stream stats error: {}", e);
log::error!("[SQLITE] commit set stream stats error: {e}");
return Err(e.into());
}
@ -866,7 +868,7 @@ UPDATE stream_stats
.await
{
if let Err(e) = tx.rollback().await {
log::error!("[SQLITE] rollback set stream stats error: {}", e);
log::error!("[SQLITE] rollback set stream stats error: {e}");
}
return Err(e.into());
}
@ -893,8 +895,7 @@ UPDATE stream_stats
Err(e) => {
if let Err(e) = tx.rollback().await {
log::error!(
"[SQLITE] rollback set stream stats error for delete file list: {}",
e
"[SQLITE] rollback set stream stats error for delete file list: {e}"
);
}
return Err(e.into());
@ -905,7 +906,7 @@ UPDATE stream_stats
// commit
if let Err(e) = tx.commit().await {
log::error!("[SQLITE] commit set stream stats error: {}", e);
log::error!("[SQLITE] commit set stream stats error: {e}");
return Err(e.into());
}
@ -951,7 +952,7 @@ UPDATE stream_stats
{
Ok(r) => r,
Err(e) => {
log::error!("[SQLITE] get file list len error: {}", e);
log::error!("[SQLITE] get file list len error: {e}");
return 0;
}
};

2
src/infra/src/queue/mod.rs
View File

@ -78,7 +78,7 @@ impl Message {
Message::Nats(msg) => msg
.ack()
.await
.map_err(|e| Error::Message(format!("ack error:{}", e)))?,
.map_err(|e| Error::Message(format!("ack error:{e}")))?,
}
Ok(())
}

31
src/infra/src/queue/nats.rs
View File

@ -103,8 +103,8 @@ impl super::Queue for NatsQueue {
let client = get_nats_client().await?;
let jetstream = jetstream::new(client);
let stream = jetstream.get_stream(&stream_name).await.map_err(|e| {
log::error!("Failed to get nats stream {}: {}", stream_name, e);
Error::Message(format!("Failed to get nats stream {}: {}", stream_name, e))
log::error!("Failed to get nats stream {stream_name}: {e}");
Error::Message(format!("Failed to get nats stream {stream_name}: {e}"))
})?;
let config = jetstream::consumer::pull::Config {
name: Some(consumer_name.to_string()),
@ -120,39 +120,24 @@ impl super::Queue for NatsQueue {
.get_or_create_consumer(&consumer_name, config)
.await
.map_err(|e| {
log::error!(
"Failed to get_or_create nats for stream {}: {}",
stream_name,
e
);
log::error!("Failed to get_or_create nats for stream {stream_name}: {e}");
Error::Message(format!(
"Failed to get_or_create nats for stream {}: {}",
stream_name, e
"Failed to get_or_create nats for stream {stream_name}: {e}"
))
})?;
// Consume messages from the consumer
let mut messages = consumer.messages().await.map_err(|e| {
log::error!(
"Failed to get nats consumer messages for stream {}: {}",
stream_name,
e
);
log::error!("Failed to get nats consumer messages for stream {stream_name}: {e}");
Error::Message(format!(
"Failed to get nats consumer messages for stream {}: {}",
stream_name, e
"Failed to get nats consumer messages for stream {stream_name}: {e}"
))
})?;
while let Ok(Some(message)) = messages.try_next().await {
let message = super::Message::Nats(message);
tx.send(message).await.map_err(|e| {
log::error!(
"Failed to send nats message for stream {}: {}",
stream_name,
e
);
log::error!("Failed to send nats message for stream {stream_name}: {e}");
Error::Message(format!(
"Failed to send nats message for stream {}: {}",
stream_name, e
"Failed to send nats message for stream {stream_name}: {e}"
))
})?;
}

49
src/infra/src/scheduler/mysql.rs
View File

@ -216,7 +216,7 @@ INSERT IGNORE INTO scheduled_jobs (org, module, module_key, is_realtime, is_sile
// It will send event even if the alert is not realtime alert.
// But that is okay, for non-realtime alerts, since the triggers are not
// present in the cache at all, it will just do nothing.
let key = format!("{TRIGGERS_KEY}{}/{}/{}", module, org, key);
let key = format!("{TRIGGERS_KEY}{module}/{org}/{key}");
let cluster_coordinator = db::get_coordinator().await;
cluster_coordinator.delete(&key, false, true, None).await?;
}
@ -388,11 +388,10 @@ INSERT IGNORE INTO scheduled_jobs (org, module, module_key, is_realtime, is_sile
let lock_key = "scheduler_pull_lock".to_string();
let lock_id = config::utils::hash::gxhash::new().sum64(&lock_key);
let lock_sql = format!(
"SELECT GET_LOCK('{}', {})",
lock_id,
"SELECT GET_LOCK('{lock_id}', {})",
config::get_config().limit.meta_transaction_lock_timeout
);
let unlock_sql = format!("SELECT RELEASE_LOCK('{}')", lock_id);
let unlock_sql = format!("SELECT RELEASE_LOCK('{lock_id}')");
let mut lock_tx = lock_pool.begin().await?;
DB_QUERY_NUMS
.with_label_values(&["get_lock", "scheduled_jobs", ""])
@ -404,10 +403,7 @@ INSERT IGNORE INTO scheduled_jobs (org, module, module_key, is_realtime, is_sile
Ok(v) => {
if v != 1 {
if let Err(e) = lock_tx.rollback().await {
log::error!(
"[SCHEDULER] rollback lock for pull scheduled_jobs error: {}",
e
);
log::error!("[SCHEDULER] rollback lock for pull scheduled_jobs error: {e}");
}
return Err(Error::from(DbError::DBOperError(
"LockTimeout".to_string(),
@ -417,10 +413,7 @@ INSERT IGNORE INTO scheduled_jobs (org, module, module_key, is_realtime, is_sile
}
Err(e) => {
if let Err(e) = lock_tx.rollback().await {
log::error!(
"[SCHEDULER] rollback lock for pull scheduled_jobs error: {}",
e
);
log::error!("[SCHEDULER] rollback lock for pull scheduled_jobs error: {e}");
}
return Err(e.into());
}
@ -434,10 +427,7 @@ INSERT IGNORE INTO scheduled_jobs (org, module, module_key, is_realtime, is_sile
log::error!("[SCHEDULER] unlock pull scheduled_jobs error: {}", e);
}
if let Err(e) = lock_tx.commit().await {
log::error!(
"[SCHEDULER] commit for unlock pull scheduled_jobs error: {}",
e
);
log::error!("[SCHEDULER] commit for unlock pull scheduled_jobs error: {e}");
}
return Err(e.into());
}
@ -474,10 +464,7 @@ LIMIT ?;
log::error!("[SCHEDULER] unlock pull scheduled_jobs error: {}", e);
}
if let Err(e) = lock_tx.commit().await {
log::error!(
"[SCHEDULER] commit for unlock pull scheduled_jobs error: {}",
e
);
log::error!("[SCHEDULER] commit for unlock pull scheduled_jobs error: {e}");
}
return Err(e.into());
}
@ -498,10 +485,7 @@ LIMIT ?;
log::error!("[SCHEDULER] unlock pull scheduled_jobs error: {}", e);
}
if let Err(e) = lock_tx.commit().await {
log::error!(
"[SCHEDULER] commit for unlock pull scheduled_jobs error: {}",
e
);
log::error!("[SCHEDULER] commit for unlock pull scheduled_jobs error: {e}");
}
return Ok(vec![]);
}
@ -572,10 +556,7 @@ WHERE id IN ({});",
log::error!("[SCHEDULER] unlock pull scheduled_jobs error: {}", e);
}
if let Err(e) = lock_tx.commit().await {
log::error!(
"[SCHEDULER] commit for unlock pull scheduled_jobs error: {}",
e
);
log::error!("[SCHEDULER] commit for unlock pull scheduled_jobs error: {e}");
}
let query = format!(
@ -610,8 +591,7 @@ WHERE id IN ({});",
Ok(job) => job,
Err(_) => {
return Err(Error::from(DbError::KeyNotExists(format!(
"{org}/{}/{key}",
module
"{org}/{module}/{key}"
))));
}
};
@ -766,12 +746,11 @@ async fn add_data_column() -> Result<()> {
if let Err(e) = sqlx::query(r#"ALTER TABLE scheduled_jobs ADD COLUMN data LONGTEXT NOT NULL;"#)
.execute(&pool)
.await
&& !e.to_string().contains("Duplicate column name")
{
if !e.to_string().contains("Duplicate column name") {
// Check for the specific MySQL error code for duplicate column
log::error!("[MYSQL] Unexpected error in adding column: {}", e);
return Err(e.into());
}
// Check for the specific MySQL error code for duplicate column
log::error!("[MYSQL] Unexpected error in adding column: {e}");
return Err(e.into());
}
Ok(())
}

5
src/infra/src/scheduler/postgres.rs
View File

@ -221,7 +221,7 @@ INSERT INTO scheduled_jobs (org, module, module_key, is_realtime, is_silenced, s
// It will send event even if the alert is not realtime alert.
// But that is okay, for non-realtime alerts, since the triggers are not
// present in the cache at all, it will just do nothing.
let key = format!("{TRIGGERS_KEY}{}/{}/{}", module, org, key);
let key = format!("{TRIGGERS_KEY}{module}/{org}/{key}");
let cluster_coordinator = db::get_coordinator().await;
cluster_coordinator.delete(&key, false, true, None).await?;
}
@ -474,8 +474,7 @@ WHERE org = $1 AND module = $2 AND module_key = $3;"#;
Ok(job) => job,
Err(_) => {
return Err(Error::from(DbError::KeyNotExists(format!(
"{org}/{}/{key}",
module
"{org}/{module}/{key}"
))));
}
};

5
src/infra/src/scheduler/sqlite.rs
View File

@ -204,7 +204,7 @@ INSERT INTO scheduled_jobs (org, module, module_key, is_realtime, is_silenced, s
// For status update of triggers, we don't need to send put events
// to cluster coordinator for now as it only changes the status and retries
// fields of scheduled jobs and not anything else
let key = format!("{TRIGGERS_KEY}{}/{}/{}", module, org, key);
let key = format!("{TRIGGERS_KEY}{module}/{org}/{key}");
let cluster_coordinator = db::get_coordinator().await;
cluster_coordinator.delete(&key, false, true, None).await?;
}
@ -419,8 +419,7 @@ WHERE org = $1 AND module = $2 AND module_key = $3;"#;
Ok(job) => job,
Err(_) => {
return Err(Error::from(DbError::KeyNotExists(format!(
"{org}/{}/{key}",
module
"{org}/{module}/{key}"
))));
}
};

27
src/infra/src/schema/mod.rs
View File

@ -159,11 +159,11 @@ pub async fn get_versions(
}
if let Some(last_index) = last_schema_index {
if last_index > 0 {
if let Some((_, data)) = versions.get(last_index - 1) {
// older version of schema before start_dt should be added in start
schemas.insert(0, data.clone());
}
if last_index > 0
&& let Some((_, data)) = versions.get(last_index - 1)
{
// older version of schema before start_dt should be added in start
schemas.insert(0, data.clone());
}
} else {
// this is latest version of schema hence added in end
@ -201,7 +201,7 @@ pub async fn get_settings(
stream_name: &str,
stream_type: StreamType,
) -> Option<StreamSettings> {
let key = format!("{}/{}/{}", org_id, stream_type, stream_name);
let key = format!("{org_id}/{stream_type}/{stream_name}");
// Try to get from read lock first
if let Some(settings) = get_stream_settings_atomic(&key) {
@ -408,8 +408,7 @@ pub async fn merge(
Some(s) => s,
None => {
return Err(Error::Message(format!(
"Error parsing latest schema for schema: {}",
key
"Error parsing latest schema for schema: {key}"
)));
}
};
@ -433,18 +432,22 @@ pub async fn merge(
.collect::<Vec<_>>();
let need_new_version = !schema_version_changes.is_empty();
if need_new_version && start_dt.is_some() {
if need_new_version && let Some(start_dt) = start_dt {
// update old version end_dt
let mut metadata = latest_schema.metadata().clone();
metadata.insert("end_dt".to_string(), start_dt.unwrap().to_string());
metadata.insert("end_dt".to_string(), start_dt.to_string());
let prev_schema = vec![latest_schema.clone().with_metadata(metadata)];
let mut new_metadata = latest_schema.metadata().clone();
new_metadata.insert("start_dt".to_string(), start_dt.unwrap().to_string());
new_metadata.insert("start_dt".to_string(), start_dt.to_string());
let new_schema = vec![final_schema.clone().with_metadata(new_metadata)];
tx.send(Some((final_schema, field_datatype_delta))).unwrap();
Ok(Some((
Some(json::to_vec(&prev_schema).unwrap().into()),
Some((key, json::to_vec(&new_schema).unwrap().into(), start_dt)),
Some((
key,
json::to_vec(&new_schema).unwrap().into(),
Some(start_dt),
)),
)))
} else {
// just update the latest schema

8
src/infra/src/storage/accounts.rs
View File

@ -109,10 +109,10 @@ impl StorageClientFactory {
/// Get the client for the given name.
/// If the name is not found, return the default client.
pub fn get_client_by_name(&self, name: &str) -> &dyn ObjectStore {
if !name.is_empty() {
if let Some(client) = self.accounts.get(name) {
return client;
}
if !name.is_empty()
&& let Some(client) = self.accounts.get(name)
{
return client;
}
self.accounts
.get(DEFAULT_ACCOUNT)

10
src/infra/src/storage/mod.rs
View File

@ -325,8 +325,8 @@ pub enum Error {
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::OutOfRange(s) => write!(f, "Out of range: {}", s),
Self::BadRange(s) => write!(f, "Bad range: {}", s),
Self::OutOfRange(s) => write!(f, "Out of range: {s}"),
Self::BadRange(s) => write!(f, "Bad range: {s}"),
}
}
}
@ -355,15 +355,13 @@ impl std::fmt::Display for InvalidGetRange {
Self::StartTooLarge { requested, length } => {
write!(
f,
"Start too large: requested {} but length is {}",
requested, length
"Start too large: requested {requested} but length is {length}"
)
}
Self::Inconsistent { start, end } => {
write!(
f,
"Inconsistent range: start {} is greater than end {}",
start, end
"Inconsistent range: start {start} is greater than end {end}"
)
}
}

2
src/infra/src/storage/remote.rs
View File

@ -383,7 +383,7 @@ fn init_client(config: StorageConfig) -> Box<dyn object_store::ObjectStore> {
_ => match init_aws_config(config) {
Ok(client) => Box::new(client),
Err(e) => {
panic!("{} init config error: {:?}", provider, e);
panic!("{provider} init config error: {e:?}");
}
},
}

2
src/infra/src/table/dashboards.rs
View File

@ -304,7 +304,7 @@ pub async fn delete_from_folder(
// Delete any report that referenced the dashboard.
for r in report_models {
let rslt = r.delete(&txn).await;
println!("delete report result: {:?}", rslt);
println!("delete report result: {rslt:?}");
}
let _ = dashboard_model.delete(&txn).await?;

2
src/infra/src/table/migration/m20250109_092400_recreate_tables_with_ksuids.rs
View File

@ -177,7 +177,7 @@ mod legacy_folders {
for folder in folders {
let ksuid = ksuid_from_hash(&folder).to_string();
let mut am = folder.into_active_model();
println!("folder ksuid: {}", ksuid);
println!("folder ksuid: {ksuid}");
am.ksuid = Set(Some(ksuid));
am.update(conn).await?;
}

2
src/infra/src/table/migration/m20250113_144600_create_unique_folder_name_idx.rs
View File

@ -82,7 +82,7 @@ async fn rename_duplicates<C: ConnectionTrait>(conn: &C) -> Result<(), DbErr> {
let name = f.name.clone();
let ksuid = svix_ksuid::Ksuid::new(None, None);
let mut am = f.into_active_model();
am.name = Set(format!("{} {}", name, ksuid));
am.name = Set(format!("{name} {ksuid}"));
am.update(conn).await?;
}
}

Some files were not shown because too many files have changed in this diff Show More