Compare commits
1 Commits
master
...
revert-668
| Author | SHA1 | Date |
|---|---|---|
opengauss_bot
|
0c687859c3 |
|
|
@ -1 +0,0 @@
|
|||
*.jar filter=lfs diff=lfs merge=lfs -text
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
# Build artifacts
|
||||
target/
|
||||
*.jar
|
||||
*.war
|
||||
*.ear
|
||||
|
||||
# IDE configuration files
|
||||
.project
|
||||
.classpath
|
||||
.settings/
|
||||
.idea/
|
||||
*.iml
|
||||
*.ipr
|
||||
*.iws
|
||||
|
||||
# Log files
|
||||
*.log
|
||||
|
||||
# Maven files
|
||||
pom.xml.tag
|
||||
pom.xml.releaseBackup
|
||||
pom.xml.versionsBackup
|
||||
pom.xml.next
|
||||
release.properties
|
||||
dependency-reduced-pom.xml
|
||||
buildNumber.properties
|
||||
.mvn/timing.properties
|
||||
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{
|
||||
"cmake.sourceDirectory": "D:/examples/openGaussSharedTableParallelHashJoin"
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,64 +0,0 @@
|
|||
import ollama
|
||||
import psycopg2
|
||||
|
||||
table_name = "item_collection_website_content_crawler"
|
||||
|
||||
def embedding(text):
|
||||
vector = ollama.embeddings(model="nomic-embed-text", prompt=text)
|
||||
return vector["embedding"]
|
||||
|
||||
conn = psycopg2.connect(
|
||||
database="postgres",
|
||||
user="hly",
|
||||
password="Hly@1234",
|
||||
host="192.168.219.134",
|
||||
port="5432"
|
||||
)
|
||||
|
||||
cur = conn.cursor()
|
||||
# 设置搜索路径
|
||||
cur.execute("SET search_path TO public;")
|
||||
# 创建索引
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_{}_embedding_hnsw ON {} USING hnsw (embedding vector_l2_ops);".format(table_name, table_name))
|
||||
conn.commit()
|
||||
|
||||
question = "介绍一下openGauss DataVec"
|
||||
emb_data = embedding(question)
|
||||
dimensions = len(emb_data)
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute("select document_content from {} order by embedding <-> '{}' limit 3;".format(table_name, emb_data))
|
||||
conn.commit()
|
||||
|
||||
rows = cur.fetchall()
|
||||
print(rows)
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
# 指定容器所在主机或容器名、端口（如果需要的话）, 或通过设置环境变量
|
||||
# client = ollama.Client(host="http://<宿主机 IP>:11434")
|
||||
|
||||
context = rows
|
||||
# 测试
|
||||
# context = ""
|
||||
|
||||
SYSTEM_PROMPT = "你作为一个对话 AI 助手，结合上下文信息简练高效的回答用户提出的问题"
|
||||
USER_PROMPT = f"请结合{context}信息来回答{question}的问题，不需要额外的无用回答"
|
||||
|
||||
response = ollama.chat(
|
||||
model="deepseek-r1:1.5b",
|
||||
stream=True,
|
||||
messages=[
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": USER_PROMPT}
|
||||
]
|
||||
)
|
||||
|
||||
print("\n\n")
|
||||
# 实时打印每一段内容
|
||||
for chunk in response:
|
||||
print(chunk["message"]["content"], end="", flush=True)
|
||||
|
||||
# print(response["message"]["content"])
|
||||
|
|
@ -1,395 +0,0 @@
|
|||
# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "annotated-types"
|
||||
version = "0.7.0"
|
||||
description = "Reusable constraint types to use with typing.Annotated"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
|
||||
{file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.10.0"
|
||||
description = "High-level concurrency and networking framework on top of asyncio or Trio"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1"},
|
||||
{file = "anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
idna = ">=2.8"
|
||||
sniffio = ">=1.1"
|
||||
typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""}
|
||||
|
||||
[package.extras]
|
||||
trio = ["trio (>=0.26.1)"]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2025.8.3"
|
||||
description = "Python package for providing Mozilla's CA Bundle."
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"},
|
||||
{file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h11"
|
||||
version = "0.16.0"
|
||||
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"},
|
||||
{file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "1.0.9"
|
||||
description = "A minimal low-level HTTP client."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"},
|
||||
{file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
certifi = "*"
|
||||
h11 = ">=0.16"
|
||||
|
||||
[package.extras]
|
||||
asyncio = ["anyio (>=4.0,<5.0)"]
|
||||
http2 = ["h2 (>=3,<5)"]
|
||||
socks = ["socksio (==1.*)"]
|
||||
trio = ["trio (>=0.22.0,<1.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "httpx"
|
||||
version = "0.28.1"
|
||||
description = "The next generation HTTP client."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"},
|
||||
{file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
anyio = "*"
|
||||
certifi = "*"
|
||||
httpcore = "==1.*"
|
||||
idna = "*"
|
||||
|
||||
[package.extras]
|
||||
brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""]
|
||||
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
|
||||
http2 = ["h2 (>=3,<5)"]
|
||||
socks = ["socksio (==1.*)"]
|
||||
zstd = ["zstandard (>=0.18.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.10"
|
||||
description = "Internationalized Domain Names in Applications (IDNA)"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"},
|
||||
{file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "ollama"
|
||||
version = "0.5.4"
|
||||
description = "The official Python client for Ollama."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "ollama-0.5.4-py3-none-any.whl", hash = "sha256:6374c9bb4f2a371b3583c09786112ba85b006516745689c172a7e28af4d4d1a2"},
|
||||
{file = "ollama-0.5.4.tar.gz", hash = "sha256:75857505a5d42e5e58114a1b78cc8c24596d8866863359d8a2329946a9b6d6f3"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
httpx = ">=0.27"
|
||||
pydantic = ">=2.9"
|
||||
|
||||
[[package]]
|
||||
name = "psycopg2-binary"
|
||||
version = "2.9.9"
|
||||
description = "psycopg2 - Python-PostgreSQL Database Adapter"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"},
|
||||
{file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"},
|
||||
{file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"},
|
||||
{file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"},
|
||||
{file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic"
|
||||
version = "2.11.9"
|
||||
description = "Data validation using Python type hints"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "pydantic-2.11.9-py3-none-any.whl", hash = "sha256:c42dd626f5cfc1c6950ce6205ea58c93efa406da65f479dcb4029d5934857da2"},
|
||||
{file = "pydantic-2.11.9.tar.gz", hash = "sha256:6b8ffda597a14812a7975c90b82a8a2e777d9257aba3453f973acd3c032a18e2"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
annotated-types = ">=0.6.0"
|
||||
pydantic-core = "2.33.2"
|
||||
typing-extensions = ">=4.12.2"
|
||||
typing-inspection = ">=0.4.0"
|
||||
|
||||
[package.extras]
|
||||
email = ["email-validator (>=2.0.0)"]
|
||||
timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic-core"
|
||||
version = "2.33.2"
|
||||
description = "Core functionality for Pydantic validation and serialization"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22"},
|
||||
{file = "pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab"},
|
||||
{file = "pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2"},
|
||||
{file = "pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5"},
|
||||
{file = "pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-win32.whl", hash = "sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9"},
|
||||
{file = "pydantic_core-2.33.2-cp39-cp39-win_amd64.whl", hash = "sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c"},
|
||||
{file = "pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb"},
|
||||
{file = "pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039"},
|
||||
{file = "pydantic_core-2.33.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27"},
|
||||
{file = "pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
|
||||
|
||||
[[package]]
|
||||
name = "sniffio"
|
||||
version = "1.3.1"
|
||||
description = "Sniff out which async library your code is running under"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
|
||||
{file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.15.0"
|
||||
description = "Backported and Experimental Type Hints for Python 3.9+"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"},
|
||||
{file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-inspection"
|
||||
version = "0.4.1"
|
||||
description = "Runtime typing introspection tools"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51"},
|
||||
{file = "typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
typing-extensions = ">=4.12.0"
|
||||
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.11"
|
||||
content-hash = "132dfa86c9b3a4e9221cb6ce4a3fea6bb11593b0c0b9f168615adc2aaf34db3a"
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
name = "chatbot-ollama"
|
||||
version = "0.1.0"
|
||||
description = "A chatbot implementation using Ollama."
|
||||
authors = ["huliyan <liyanhu03@163.com>"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.11"
|
||||
ollama = ">=0.2.0"
|
||||
psycopg2-binary = "2.9.9"
|
||||
|
|
@ -1,312 +0,0 @@
|
|||
# Airbyte实践案例: 从Apify Dataset到openGauss DataVec
|
||||
|
||||
Airbyte目前有300+ Source可选，本文选择使用频率很高、非常流行的Apify Dataset作为source，destination则为openGauss DataVec。
|
||||
|
||||
具体而言，本教程将使用Apify的Website Content Crawler Actor抓取openGauss DataVec的官方文档，存入Apify Dataset，之后通过airbyte平台写入openGauss DataVec向量数据库。再本地部署deepseek-r1大模型和nomic-embed-text嵌入模型，搭建本地AI助手。
|
||||
|
||||
## 前置准备
|
||||
|
||||
- 本地部署的airbyte
|
||||
- apify账号
|
||||
- docker
|
||||
|
||||
本文实践所采用的操作系统为 **openEuler 22.03 LTS**。为了确保各个组件之间能够实现无缝兼容并顺利运行，我们选择使用 **python3.11**。
|
||||
|
||||
|
||||
|
||||
## Apify抓取openGauss DataVec官方文档
|
||||
|
||||
登录Apify控制台网站 [Apify Console](https://console.apify.com/)
|
||||
|
||||
选择Apify Store -> Website Content Crawler
|
||||
|
||||
|
||||
|
||||
点击后进行简单配置，这里选择抓取一页openGauss DataVec官方文档
|
||||
|
||||
- `Start URLs`填 `https://docs.opengauss.org/zh/docs/latest/docs/DataVec/DataVec-Overview.html`
|
||||
|
||||
- `Crawler type`选择第三个`Raw HTTP client(Cheerio)-Fastest, but doesn't render JavaScript content.`
|
||||
- `HTML Processing` -> `Keep HTML elements (CSS selector)` 填入 `article`
|
||||
|
||||
其他保留默认值即可。
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
点击start，获取到结果如下。
|
||||
|
||||
|
||||
|
||||
点击左边导航栏Storage，查看最新捕获内容的`Dataset ID`，并修改name为`openGauss DataVec介绍`
|
||||
|
||||
|
||||
|
||||
复制`Dataset ID`，等下airbyte需要用到。
|
||||
|
||||
点击左边栏`Settings` -> `API& Integrations`复制 `Personal API tokens`，等下airbyte也需要用到。
|
||||
|
||||
|
||||
|
||||
## 配置Airbyte-Apify Dataset
|
||||
|
||||
进入airbyte界面，点击左边栏`Source` ，选择`Apify Dataset`，填充我们之前复制的`Dataset ID`和 `Personal API tokens`，然后`set up source`
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 部署openGauss DataVec
|
||||
|
||||
命令行拉取openGauss镜像并运行
|
||||
|
||||
```bash
|
||||
# 拉取镜像
|
||||
docker pull opengauss/opengauss:latest
|
||||
|
||||
# 运行容器
|
||||
docker run --name opengauss --privileged=true -d -e GS_PASSWORD=Test@123 -v /opengauss:/var/lib/opengauss -p 5432:5432 opengauss/opengauss:latest
|
||||
```
|
||||
|
||||
创建新用户(因为openGauss不支持初始用户通过远程连接登录)
|
||||
|
||||
```bash
|
||||
# 进入opengauss容器
|
||||
docker exec -it opengauss bash
|
||||
|
||||
# 切换用户
|
||||
su omm
|
||||
# 连接opengauss
|
||||
gsql -d postgres
|
||||
# 创建新用户
|
||||
CREATE USER airbyte WITH PASSWORD 'Airbyte@123';
|
||||
# 分配权限
|
||||
GRANT ALL PRIVILEGES TO airbyte;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO airbyte;
|
||||
GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO airbyte;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO airbyte;
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 部署deepseek-r1大模型和nomic-embed-text嵌入模型
|
||||
|
||||
命令行拉取ollama镜像并运行
|
||||
|
||||
```bash
|
||||
# 拉取镜像
|
||||
docker pull ollama/ollama
|
||||
|
||||
# 运行镜像
|
||||
docker run -d -p 11434:11434 --name ollama ollama/ollama:latest
|
||||
# 进入容器
|
||||
docker exec -it ollama bash
|
||||
# 拉取模型
|
||||
ollama pull nomic-embed-text
|
||||
ollama pull deepseek-r1:1.5b # 根据电脑配置选择适当参数的模型
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 配置airbyte-openGauss DataVec
|
||||
|
||||
### 加载openGauss DataVec destination connector镜像
|
||||
|
||||
进入airbyte主界面，点击`Setting` -> `Destinations` -> `+New connector`
|
||||
|
||||
|
||||
|
||||
填写表单如下
|
||||
|
||||
- `Connector display name`：`openGauss DataVec`
|
||||
- `Docker repository name`: `lucashu123/airbyte-connector-opengauss-datavec`
|
||||
- `Docker image tag`：`latest`
|
||||
|
||||
|
||||
|
||||
之后便可在Destination -> Custom中选择openGauss DataVec作为destination。
|
||||
|
||||
|
||||
|
||||
### 填写配置
|
||||
|
||||
配置openGauss DataVec有3个部分。
|
||||
|
||||
- `Processing`—决定将source中获取的内容的哪些字段作为元数据，哪些字段作为文本字段，分别由`Fields to store as metadata`和`Text fields to embed`控制(若不填，默认选择所有字段)。以及将所有文本字段拼接成一个总文本后，如何进行分块，由`Chunk size`和`Text splitter`等控制。
|
||||
- `Embedding`—选择嵌入向量模型API，将每个chunk分块文本嵌入向量。可以选OpenAI API，自部署的OpenAI-compatible API等。这里我们选择自部署的`nomic-embed-text嵌入模型`。
|
||||
- `Indexing`—填写openGauss向量数据库的连接配置。
|
||||
|
||||
配置如下：
|
||||
|
||||
- `Processing`部分
|
||||
- `chunk size`: 100
|
||||
- `Fields to store as metadata`: `metadata`
|
||||
- `Text fields to embed`: `markdown`
|
||||
- `Text splitter`选择`By Markdown header`，`Split level`填充`2`
|
||||
- `Embedding`部分，选择`OpenAI-compatible`
|
||||
- `Base URL`: http://localhost:11434/v1 (替换为上面部署ollama容器所在机器的ip地址)
|
||||
- `Embedding dimensions`: 768
|
||||
- `Model name` : `nomic-embed-text`
|
||||
- `Indexing`部分
|
||||
- `Host`: `localhost` (替换为上面部署openGauss容器所在机器的ip地址)
|
||||
- `Database`: `postgres`
|
||||
- `Username`: `airbyte` (上面创建的新用户)
|
||||
- `Password`: `Airbyte@123` (上面创建的新用户密码)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## 建立连接
|
||||
|
||||
进入airbyte主界面点击左边栏`connections` -> `+ New connection`
|
||||
|
||||
`Define source`选择上面配置好的`Apify Dataset`，`Define destination`选择上面配置好的`openGauss DataVec`
|
||||
|
||||
`Select streams`中有4个stream，我们只选择`item_collection_website_content_crawler`流，其中只勾选`markdown`和`metadata`字段
|
||||
|
||||
|
||||
|
||||
`Configure connection`中的`Schedule type`我们选择调度方式为`Manual`，因为我们选择的Apify Dataset的内容是固定的，手动同步一次即可，无需定时按频率同步。
|
||||
|
||||
## 开始同步
|
||||
|
||||
建立好连接后，即可点击`Sync now`开始同步，大概花费5min，等待同步结束。
|
||||
|
||||
|
||||
|
||||
所有流将被存储到同名的表中。如果表不存在将创建表。表将具有以下列：
|
||||
|
||||
- `document_id`（字符串）- 文档的唯一标识符，通过stream名 + stream的主键构成，若没有主键，则uuid随机生成`document_id`
|
||||
- `chunk_id`（字符串）- 块的唯一标识符，通过将块编号附加到 document_id 创建
|
||||
- `metadata`（变体）- 文档的元数据，以键值对形式存储
|
||||
- `document_content`（字符串）- 块的文本内容
|
||||
- `embedding`（向量）- 块的嵌入，以浮点数列表形式存储
|
||||
|
||||
## 搭建AI助手进行测试
|
||||
|
||||
源代码如下，利用本地部署的deepseek-r1作大模型
|
||||
|
||||
```python
|
||||
import ollama
|
||||
import psycopg2
|
||||
|
||||
table_name = "item_collection_website_content_crawler"
|
||||
|
||||
def embedding(text):
|
||||
vector = ollama.embeddings(model="nomic-embed-text", prompt=text)
|
||||
return vector["embedding"]
|
||||
|
||||
# 替换为自己的配置
|
||||
conn = psycopg2.connect(
|
||||
database="postgres",
|
||||
user="airbyte",
|
||||
password="Airbyte@123",
|
||||
host="192.168.219.134",
|
||||
port="5432"
|
||||
)
|
||||
|
||||
cur = conn.cursor()
|
||||
# 设置搜索路径
|
||||
cur.execute("SET search_path TO public;")
|
||||
# 创建索引
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_{}_embedding_hnsw ON {} USING hnsw (embedding vector_l2_ops);".format(table_name, table_name))
|
||||
conn.commit()
|
||||
|
||||
question = "介绍一下openGauss DataVec"
|
||||
emb_data = embedding(question)
|
||||
dimensions = len(emb_data)
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute("select document_content from {} order by embedding <-> '{}' limit 3;".format(table_name, emb_data))
|
||||
conn.commit()
|
||||
|
||||
rows = cur.fetchall()
|
||||
print(rows)
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
# 指定容器所在主机或容器名、端口（如果需要的话）, 或通过设置环境变量
|
||||
# client = ollama.Client(host="http://<宿主机 IP>:11434")
|
||||
|
||||
context = rows
|
||||
# 测试
|
||||
# context = ""
|
||||
|
||||
SYSTEM_PROMPT = "你作为一个对话 AI 助手，结合上下文信息简练高效的回答用户提出的问题"
|
||||
USER_PROMPT = f"请结合{context}信息来回答{question}的问题，不需要额外的无用回答"
|
||||
|
||||
response = ollama.chat(
|
||||
model="deepseek-r1:1.5b",
|
||||
stream=True,
|
||||
messages=[
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": USER_PROMPT}
|
||||
]
|
||||
)
|
||||
|
||||
print("\n\n")
|
||||
# 实时打印每一段内容
|
||||
for chunk in response:
|
||||
print(chunk["message"]["content"], end="", flush=True)
|
||||
```
|
||||
|
||||
问题为`介绍一下openGauss DataVec`。当没有为大模型提供DataVec查询结果作为上下文时的回答如下
|
||||
|
||||
```
|
||||
<think>
|
||||
好，我现在需要帮用户介绍一下openGauss DataVec的问题。用户要求答案要简洁高效，不需要额外的信息。
|
||||
|
||||
首先，我得了解DataVec的基本概念和结构。DataVec是openGauss的一个数据结构，用于存储数据向量，类似于Java中的List或Python的List。
|
||||
|
||||
接下来，我要考虑用户可能对DataVec的使用情况感兴趣。也许他们是在开发一个需要高扩展性的数据库系统，或者在进行数据管理和查询 优化的工作中。因此，在回答时可以提到DataVec的优势，比如快速查找和插入记录，以及支持聚合操作，这样能帮助用户更好地理解其价 值。
|
||||
|
||||
另外，用户可能关心如何高效存储大量数据。DataVec的随机访问特性可能能满足他们的需求，尤其是在处理高并发或大表的情况下。
|
||||
|
||||
最后，总结一下DataVec适合什么样的场景，比如在需要动态查询、快速插入和数据管理中的应用。
|
||||
</think>
|
||||
|
||||
openGauss DataVec是一个高效的数据库数据结构，专为存储动态数据向量设计。与传统列表类型数据结构相比，DataVec的优势在于：
|
||||
|
||||
1. **快速查找**：支持从随机位置获取记录信息，适用于动态查询场景。
|
||||
2. **快速插入**：无需重新初始化数据结构，可以快速添加新记录。
|
||||
3. **支持聚合操作**：适合进行统计、汇总等操作。
|
||||
|
||||
例如，在大数据分析或实时系统中，DataVec能够高效管理大量数据向量，满足高扩展性和灵活性的需求。
|
||||
```
|
||||
|
||||
幻觉比较严重，当我们添加DataVec查询结果作为大模型补充上下文时的回答如下
|
||||
|
||||
```
|
||||
<think>
|
||||
好的，我现在需要帮助用户解决一个关于openGauss DataVec的问题。根据用户提供的上下文，我看到他们提到了DataVec向量数据库和它的多语言生态对接，并且提供了详细的说明。
|
||||
|
||||
首先，我要理解用户的需求是什么。他们想知道详细介绍openGauss DataVec的内容，但不需要额外的信息，所以我只需要提取关键点。
|
||||
|
||||
接下来，我会回顾一下提供的信息：DataVec是一个基于openGauss的向量引擎，支持存储、检索大规模高维向量数据，适用于智能知识检索和RAG应用。它还提到了支持的各种功能，如最近邻搜索、L2距离、余弦距离、内积等，并且用熟悉的SQL语法操作。
|
||||
|
||||
然后，我会考虑用户可能的身份和使用场景。他们可能是开发人员或者研究人员，需要在大数据环境中高效地处理向量数据。因此，我应该强调DataVec的性能优势以及其对多种语言的支持。
|
||||
|
||||
最后，我会整理这些信息，确保回答简明扼要，涵盖主要功能、支持的语言生态和技术特点。
|
||||
</think>
|
||||
|
||||
openGauss DataVec是一个基于openGauss的向量引擎，提供向量数据类型的存储和检索。在处理大规模高维向量数据时，能够快速、准确地检索结果，并适用于智能知识检索和RAG（Retrieval-Augmented Generation）等复杂应用场景的智能应用。
|
||||
|
||||
DataVec支持的各种功能包括：
|
||||
|
||||
1. **精确和近似的最近邻搜索**：快速找到最接近的向量。
|
||||
2. **L2距离&余弦距离&内积**：多种距离计算方式，满足不同场景的需求。
|
||||
3. **向量索引**：高效的索引结构，提升查询性能。
|
||||
4. **向量操作函数和操作符**：灵活的操作符组合，增强功能扩展。
|
||||
|
||||
此外，DataVec提供了多语言生态对接，支持通过Python、Java、Node.js等编程语言的API调用，使用户能够快速集成向量数据库能力。另 外，它还采用开源组件，优化了RAG场景下的兼容性，提供了多样化的选择。
|
||||
|
||||
总体而言，openGauss DataVec在处理向量数据时表现出色，结合了高效的性能和丰富的功能支持，满足复杂智能应用的需求。
|
||||
```
|
||||
|
||||
可见，AI助手回答效果更加精确和可靠。
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
## openGauss向量数据库对接airbyte
|
||||
|
||||
本项目为开源之夏2025的项目——openGauss向量数据库对接airbyte。
|
||||
|
||||
主要内容为
|
||||
|
||||
1. 开发destination为openGauss DataVec向量数据库的airbyte连接器(connector)；
|
||||
|
||||
2. 输出实践案例教程和详细使用文档。
|
||||
|
||||
### 目录结构
|
||||
```text
|
||||
Airbyte_Destination_openGaussDataVec/
|
||||
├─ destination-opengauss-datavec/
|
||||
├─ Practical-Case-Tutorial/
|
||||
├─ images/
|
||||
├─ airbyte对接openGauss DataVec.md
|
||||
├─ openGauss_DataVec_Connector_Document.md
|
||||
├─ openGauss_DataVec_Connector_Document.zh.md
|
||||
└─ README.md
|
||||
```
|
||||
|
||||
#### 目录与文件说明
|
||||
- **destination-opengauss-datavec/**: openGauss DataVec目标连接器(destination connector)源代码
|
||||
- **Practical-Case-Tutorial/**: 实践案例与教程，包含演示与依赖配置。
|
||||
- **images/**: 文档中使用的图片资源文件夹，存储所有Markdown文件引用的图片。
|
||||
- **airbyte对接openGauss DataVec.md**: airbyte本地部署和openGauss DataVec连接器镜像加载教程。
|
||||
- **openGauss_DataVec_Connector_Document.md**: openGauss DataVec连接器配置说明的英文文档。
|
||||
- **openGauss_DataVec_Connector_Document.zh.md**: openGauss DataVec连接器配置说明的中文文档。
|
||||
- **README.md**: 总说明。
|
||||
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
# Airbyte对接openGauss DataVec
|
||||
|
||||
本教程讲述如何将openGauss DataVec作为airbyte的destination连接器。
|
||||
|
||||
## 环境准备
|
||||
|
||||
- 安装较高版本的Docker 或 Docker Desktop
|
||||
- 配置代理
|
||||
|
||||
## 本地部署airbyte
|
||||
|
||||
参考官方教程：[Quickstart | Airbyte Docs](https://docs.airbyte.com/platform/using-airbyte/getting-started/oss-quickstart)
|
||||
|
||||
本教程以Linux系统为例进行部署。
|
||||
|
||||
### 1.安装abctl
|
||||
|
||||
abctl是airbyte部署和管理的命令行工具。
|
||||
|
||||
```bash
|
||||
curl -LsfS https://get.airbyte.com | bash -
|
||||
```
|
||||
|
||||
成功后可以看见`abctl install succeeded.`
|
||||
|
||||
### 2.运行airbyte
|
||||
|
||||
运行命令
|
||||
|
||||
```bash
|
||||
abctl local install --insecure-cookies --low-resource-mode
|
||||
```
|
||||
|
||||
- `--insecure-cookies` 表示通过http进行连接，针对不想配置ssl证书使用安全https连接时适用，可以不用
|
||||
- `--low-resource-mode`表示在低资源环境下(少于4个CPUs)运行，不能使用Connector Builder工具。可以不用
|
||||
|
||||
可通过`abctl local install --help`查看更多参数详情。上述2个参数可根据自身情况选择是否使用。
|
||||
|
||||
|
||||
|
||||
整个过程大概需要花费30min。安装完成后，可在浏览器中输入 [http://localhost:8000](http://localhost:8000/) 进入登录界面。第一次登录会要求填写邮箱，公司等信息。
|
||||
|
||||
> 温馨提示：若安装过程中遇见报错，优先考虑重启机器和更换稳定代理
|
||||
|
||||
### 3.查看凭证
|
||||
|
||||
查看默认密码命令
|
||||
|
||||
```bash
|
||||
abctl local credentials
|
||||
```
|
||||
|
||||
输出大致为
|
||||
|
||||
```bash
|
||||
Credentials:
|
||||
Email: user@example.com
|
||||
Password: a-random-password
|
||||
Client-Id: 03ef466c-5558-4ca5-856b-4960ba7c161b
|
||||
Client-Secret: m2UjnDO4iyBQ3IsRiy5GG3LaZWP6xs9I
|
||||
```
|
||||
|
||||
若要修改密码，采用下面命令。
|
||||
|
||||
```bash
|
||||
abctl local credentials --password YourStrongPasswordExample
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 加载openGauss DataVec connector镜像
|
||||
|
||||
访问 [http://localhost:8000](http://localhost:8000/) (部署airbyte所在机器的ip)登录后，点击Setting -> Destinations -> +New connector
|
||||
|
||||
|
||||
|
||||
填写表单如下
|
||||
|
||||
- `Connector display name`：`openGauss DataVec`
|
||||
- `Docker repository name`: `lucashu123/airbyte-connector-opengauss-datavec`
|
||||
- `Docker image tag`：`latest`
|
||||
|
||||
|
||||
|
||||
> Connector documentation URL选项有点问题，填充URL后也不会显示文档
|
||||
|
||||
点击`Add`，等待2-3min后成功加载。若超过5min，大概率是因为代理网络问题。
|
||||
|
||||
|
||||
|
||||
之后便可在Destination -> Custom中选择openGauss DataVec作为destination。
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
# OpenGauss DataVec Destination
|
||||
|
||||
This is the repository for the OpenGauss DataVec destination connector, written in Python.
|
||||
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/opengauss-datavec).(todo)
|
||||
|
||||
## Overview
|
||||
|
||||
This destination allows you to sync data to OpenGauss database with vector storage capabilities using the DataVec extension. It supports:
|
||||
|
||||
- **Vector Embeddings**: Store text data as vector embeddings using various embedding providers (OpenAI, Cohere, Azure OpenAI, etc.)
|
||||
- **Multiple Sync Modes**: Full refresh, incremental append, and append with deduplication
|
||||
- **Flexible Text Processing**: Configurable text chunking and field mapping
|
||||
- **OpenGauss DataVec Integration**: Native support for OpenGauss vector operations and distance functions
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- OpenGauss database with DataVec extension installed
|
||||
- Database user with appropriate permissions
|
||||
- API key for chosen embedding provider (or use fake embeddings for testing)
|
||||
|
||||
## Local development
|
||||
|
||||
### Prerequisites
|
||||
**To iterate on this connector, make sure to complete this prerequisites section.**
|
||||
|
||||
#### Minimum Python version required `= 3.10.0`
|
||||
|
||||
### Installing the connector
|
||||
From this connector directory, run:
|
||||
```bash
|
||||
poetry install --with dev.
|
||||
```
|
||||
|
||||
#### Create credentials
|
||||
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/opengauss-datavec)(todo)
|
||||
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_opengauss_datavec/spec.json` file.
|
||||
Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information.
|
||||
See `integration_tests/sample_config.json` for a sample config file.
|
||||
|
||||
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination opengauss-datavec test creds`
|
||||
and place them into `secrets/config.json`.
|
||||
|
||||
### Locally running the connector
|
||||
```
|
||||
poetry run python main.py spec
|
||||
poetry run python main.py check --config secrets/config.json
|
||||
cat examples/messages.jsonl | poetry run python main.py write --config secrets/config.json --catalog examples/configured_catalog.json
|
||||
```
|
||||
|
||||
### Locally running the connector docker image
|
||||
|
||||
#### Use `airbyte-ci` to build your connector
|
||||
The Airbyte way of building this connector is to use our `airbyte-ci` tool.
|
||||
You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1).
|
||||
Then running the following command will build your connector:
|
||||
|
||||
```bash
|
||||
airbyte-ci connectors --name destination-opengauss-datavec build
|
||||
```
|
||||
Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-opengauss-datavec:dev`.
|
||||
|
||||
##### Customizing our build process
|
||||
When contributing on our connector you might need to customize the build process to add a system dependency or set an env var.
|
||||
You can customize our build process by adding a `build_customization.py` module to your connector.
|
||||
This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively.
|
||||
It will be imported at runtime by our build process and the functions will be called if they exist.
|
||||
|
||||
Here is an example of a `build_customization.py` module:
|
||||
```python
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Feel free to check the dagger documentation for more information on the Container object and its methods.
|
||||
# https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/
|
||||
from dagger import Container
|
||||
|
||||
|
||||
async def pre_connector_install(base_image_container: Container) -> Container:
|
||||
return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value")
|
||||
|
||||
async def post_connector_install(connector_container: Container) -> Container:
|
||||
return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value")
|
||||
```
|
||||
|
||||
#### Build your own connector image
|
||||
This connector is built using our dynamic built process in `airbyte-ci`.
|
||||
The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`.
|
||||
The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py).
|
||||
It does not rely on a Dockerfile.
|
||||
|
||||
If you would like to patch our connector and build your own a simple approach would be to:
|
||||
|
||||
1. Create your own Dockerfile based on the latest version of the connector image.
|
||||
```Dockerfile
|
||||
# todo
|
||||
FROM airbyte/destination-opengauss-datavec:latest
|
||||
|
||||
COPY . ./airbyte/integration_code
|
||||
RUN pip install ./airbyte/integration_code
|
||||
|
||||
# The entrypoint and default env vars are already set in the base image
|
||||
# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
|
||||
# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
|
||||
```
|
||||
Please use this as an example. This is not optimized.
|
||||
|
||||
2. Build your image:
|
||||
```bash
|
||||
docker build -t airbyte/destination-opengauss-datavec:dev .
|
||||
# Running the spec command against your patched connector
|
||||
docker run airbyte/destination-opengauss-datavec:dev spec
|
||||
```
|
||||
#### Run
|
||||
Then run any of the connector commands as follows:
|
||||
```
|
||||
docker run --rm airbyte/destination-opengauss-datavec:dev spec
|
||||
docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-opengauss-datavec:dev check --config /secrets/config.json
|
||||
# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages
|
||||
cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-opengauss-datavec:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
||||
```
|
||||
## Testing
|
||||
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
||||
```bash
|
||||
airbyte-ci connectors --name=destination-opengauss-datavec test
|
||||
```
|
||||
|
||||
### Unit Tests
|
||||
To run unit tests locally, from the connector directory run:
|
||||
```
|
||||
poetry run pytest -s unit_tests
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector).
|
||||
|
||||
To run integration tests locally, make sure you have a secrets/config.json as explained above, and then run:
|
||||
```
|
||||
poetry run pytest -s integration_tests
|
||||
```
|
||||
|
||||
### Customizing acceptance Tests
|
||||
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
||||
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
||||
|
||||
### Using `airbyte-ci` to run tests
|
||||
See [airbyte-ci documentation](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command)
|
||||
|
||||
## Dependency Management
|
||||
All of your dependencies should go in `pyproject.toml`
|
||||
* required for your connector to work need to go to `[tool.poetry.dependencies]` list.
|
||||
* required for the testing need to go to `[tool.poetry.group.dev.dependencies]` list
|
||||
|
||||
### Publishing a new version of the connector
|
||||
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
||||
1. Make sure your changes are passing unit and integration tests.
|
||||
1. Bump the connector version in `pyproject.toml` -- increment the value appropriately (we use [SemVer](https://semver.org/)).
|
||||
1. Update the `metadata.yaml` file with the new version.
|
||||
1. Create a Pull Request.
|
||||
1. Pat yourself on the back for being an awesome contributor.
|
||||
1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
acceptance_tests:
|
||||
spec:
|
||||
tests:
|
||||
- spec_path: integration_tests/spec.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "0.0.1"
|
||||
connector_image: airbyte/destination-opengauss-datavec:dev
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# openGauss DataVec Destination Connector Bootstrap
|
||||
|
||||
This destination does three things:
|
||||
|
||||
- Split records into chunks and separates metadata from text data
|
||||
- Embeds text data into an embedding vector
|
||||
- Stores the metadata and embedding vector in a vector database
|
||||
|
||||
The record processing is using the text split components from https://python.langchain.com/docs/modules/data_connection/document_transformers/.
|
||||
|
||||
There are various possible providers for generating embeddings, delivered as part of the CDK (`airbyte_cdk.destinations.vector_db_based`).
|
||||
|
||||
Embedded documents are stored in the openGauss DataVec vector database.
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
cd /home/hly/airbyte/airbyte-ci/connectors/pipelines/
|
||||
eval "$(poetry env activate)"
|
||||
|
||||
airbyte-ci connectors --name destination-opengauss-datavec build -t dev
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from .destination import DestinationOpenGaussDataVec
|
||||
|
||||
__all__ = ["DestinationOpenGaussDataVec"]
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
"""Catalog provider implementation.
|
||||
|
||||
A catalog provider wraps a configured catalog and configured streams. This class is responsible for
|
||||
providing information about the catalog and streams. A catalog provider can also be updated with new
|
||||
streams as they are discovered, providing a thin layer of abstraction over the configured catalog.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, final
|
||||
|
||||
from airbyte import exceptions as exc
|
||||
from airbyte_cdk.models import DestinationSyncMode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream
|
||||
|
||||
|
||||
class CatalogProvider:
|
||||
"""A catalog provider wraps a configured catalog and configured streams.
|
||||
|
||||
This class is responsible for providing information about the catalog and streams.
|
||||
|
||||
Note:
|
||||
- The catalog provider is not responsible for managing the catalog or streams but it may
|
||||
be updated with new streams as they are discovered.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
configured_catalog: ConfiguredAirbyteCatalog,
|
||||
) -> None:
|
||||
"""Initialize the catalog manager with a catalog object reference.
|
||||
|
||||
Since the catalog is passed by reference, the catalog manager may be updated with new
|
||||
streams as they are discovered.
|
||||
"""
|
||||
self._catalog: ConfiguredAirbyteCatalog = configured_catalog
|
||||
|
||||
@property
|
||||
def configured_catalog(self) -> ConfiguredAirbyteCatalog:
|
||||
return self._catalog
|
||||
|
||||
@property
|
||||
def stream_names(self) -> list[str]:
|
||||
return list({stream.stream.name for stream in self.configured_catalog.streams})
|
||||
|
||||
def get_configured_stream_info(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> ConfiguredAirbyteStream:
|
||||
"""Return the column definitions for the given stream."""
|
||||
if not self.configured_catalog:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Cannot get stream JSON schema without a catalog.",
|
||||
)
|
||||
|
||||
matching_streams: list[ConfiguredAirbyteStream] = [
|
||||
stream
|
||||
for stream in self.configured_catalog.streams
|
||||
if stream.stream.name == stream_name
|
||||
]
|
||||
if not matching_streams:
|
||||
raise exc.AirbyteStreamNotFoundError(
|
||||
stream_name=stream_name,
|
||||
context={
|
||||
"available_streams": [
|
||||
stream.stream.name for stream in self.configured_catalog.streams
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
if len(matching_streams) > 1:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Multiple streams found with same name.",
|
||||
context={
|
||||
"stream_name": stream_name,
|
||||
},
|
||||
)
|
||||
|
||||
return matching_streams[0]
|
||||
|
||||
@final
|
||||
def get_stream_json_schema(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Return the column definitions for the given stream."""
|
||||
return self.get_configured_stream_info(stream_name).stream.json_schema
|
||||
|
||||
def get_stream_properties(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> dict[str, dict]:
|
||||
"""Return the names of the top-level properties for the given stream."""
|
||||
return self.get_stream_json_schema(stream_name)["properties"]
|
||||
|
||||
def get_destination_sync_mode(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> DestinationSyncMode:
|
||||
"""Return the destination sync mode for the given stream."""
|
||||
return self.get_configured_stream_info(stream_name).destination_sync_mode
|
||||
|
|
@ -1,313 +0,0 @@
|
|||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
"""Abstract base class for Processors, including SQL processors.
|
||||
|
||||
Processors accept Airbyte messages as input from STDIN or from another input stream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import io
|
||||
import queue
|
||||
import sys
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Union, cast, final
|
||||
|
||||
from airbyte import exceptions as exc
|
||||
from airbyte.strategies import WriteStrategy
|
||||
from airbyte_cdk.models import (
|
||||
AirbyteMessage,
|
||||
AirbyteRecordMessage,
|
||||
AirbyteStateMessage,
|
||||
AirbyteStateType,
|
||||
AirbyteStreamState,
|
||||
Type,
|
||||
)
|
||||
|
||||
from destination_opengauss_datavec.common.state.state_writers import StateWriterBase, StdOutStateWriter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
from airbyte._batch_handles import BatchHandle
|
||||
|
||||
from destination_opengauss_datavec.common.catalog.catalog_providers import CatalogProvider
|
||||
from destination_opengauss_datavec.common.state.state_writers import StateWriterBase
|
||||
|
||||
|
||||
class AirbyteMessageParsingError(Exception):
|
||||
"""Raised when an Airbyte message is invalid or cannot be parsed."""
|
||||
|
||||
|
||||
class RecordProcessorBase(abc.ABC):
|
||||
"""Abstract base class for classes which can process Airbyte messages from a source.
|
||||
|
||||
This class is responsible for all aspects of handling Airbyte protocol.
|
||||
|
||||
The class should be passed a catalog manager and stream manager class to handle the
|
||||
catalog and state aspects of the protocol.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
catalog_provider: CatalogProvider,
|
||||
state_writer: Union[StateWriterBase, None] = None,
|
||||
) -> None:
|
||||
"""Initialize the processor.
|
||||
|
||||
If a state writer is not provided, the processor will use the default (STDOUT) state writer.
|
||||
"""
|
||||
self._catalog_provider: Union[CatalogProvider, None] = catalog_provider
|
||||
self._state_writer: Union[StateWriterBase, None] = state_writer or StdOutStateWriter()
|
||||
|
||||
self._pending_state_messages: dict[str, list[AirbyteStateMessage]] = defaultdict(list, {})
|
||||
self._finalized_state_messages: dict[
|
||||
str,
|
||||
list[AirbyteStateMessage],
|
||||
] = defaultdict(list, {})
|
||||
|
||||
self._setup()
|
||||
|
||||
@property
|
||||
def expected_streams(self) -> set[str]:
|
||||
"""Return the expected stream names."""
|
||||
return set(self.catalog_provider.stream_names)
|
||||
|
||||
@property
|
||||
def catalog_provider(
|
||||
self,
|
||||
) -> CatalogProvider:
|
||||
"""Return the catalog manager.
|
||||
|
||||
Subclasses should set this property to a valid catalog manager instance if one
|
||||
is not explicitly passed to the constructor.
|
||||
|
||||
Raises:
|
||||
PyAirbyteInternalError: If the catalog manager is not set.
|
||||
"""
|
||||
if not self._catalog_provider:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Catalog manager should exist but does not.",
|
||||
)
|
||||
|
||||
return self._catalog_provider
|
||||
|
||||
@property
|
||||
def state_writer(
|
||||
self,
|
||||
) -> StateWriterBase:
|
||||
"""Return the state writer instance.
|
||||
|
||||
Subclasses should set this property to a valid state manager instance if one
|
||||
is not explicitly passed to the constructor.
|
||||
|
||||
Raises:
|
||||
PyAirbyteInternalError: If the state manager is not set.
|
||||
"""
|
||||
if not self._state_writer:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="State manager should exist but does not.",
|
||||
)
|
||||
|
||||
return self._state_writer
|
||||
|
||||
@final
|
||||
def process_stdin(
|
||||
self,
|
||||
*,
|
||||
write_strategy: WriteStrategy = WriteStrategy.AUTO,
|
||||
) -> None:
|
||||
"""Process the input stream from stdin.
|
||||
|
||||
Return a list of summaries for testing.
|
||||
"""
|
||||
input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8")
|
||||
self.process_input_stream(
|
||||
input_stream,
|
||||
write_strategy=write_strategy,
|
||||
)
|
||||
|
||||
@final
|
||||
def _airbyte_messages_from_buffer(
|
||||
self,
|
||||
buffer: io.TextIOBase,
|
||||
) -> Iterator[AirbyteMessage]:
|
||||
"""Yield messages from a buffer."""
|
||||
yield from (AirbyteMessage.parse_raw(line) for line in buffer)
|
||||
|
||||
@final
|
||||
def process_input_stream(
|
||||
self,
|
||||
input_stream: io.TextIOBase,
|
||||
*,
|
||||
write_strategy: WriteStrategy = WriteStrategy.AUTO,
|
||||
) -> None:
|
||||
"""Parse the input stream and process data in batches.
|
||||
|
||||
Return a list of summaries for testing.
|
||||
"""
|
||||
messages = self._airbyte_messages_from_buffer(input_stream)
|
||||
self.process_airbyte_messages(
|
||||
messages,
|
||||
write_strategy=write_strategy,
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
def process_record_message(
|
||||
self,
|
||||
record_msg: AirbyteRecordMessage,
|
||||
stream_schema: dict,
|
||||
) -> None:
|
||||
"""Write a record.
|
||||
|
||||
This method is called for each record message.
|
||||
|
||||
In most cases, the SQL processor will not perform any action, but will pass this along to to
|
||||
the file processor.
|
||||
"""
|
||||
|
||||
def process_airbyte_messages_as_generator(
|
||||
self,
|
||||
messages: Iterable[AirbyteMessage],
|
||||
*,
|
||||
write_strategy: WriteStrategy,
|
||||
) -> Iterable[AirbyteMessage]:
|
||||
"""
|
||||
This is copied from PyAirbyte's RecordProcessorBase class.
|
||||
|
||||
This version will _also_ yield `AirbyteMessage` objects which would otherwise only
|
||||
be pushed to STDOUT or sent to the state writer.
|
||||
|
||||
TODO:
|
||||
- In the future, we should optimize this to emit STATE messages as soon as their records
|
||||
are committed, rather than waiting until the end of the input stream.
|
||||
"""
|
||||
# Create a queue to store output messages
|
||||
output_queue: queue.Queue[AirbyteMessage] = queue.Queue()
|
||||
|
||||
class WrappedStateWriter(StateWriterBase):
|
||||
def __init__(self, wrapped: Union[StateWriterBase, None]) -> None:
|
||||
self.wrapped = wrapped
|
||||
super().__init__()
|
||||
|
||||
def write_state(self, state_message: AirbyteStateMessage) -> None:
|
||||
"""First add state message to output queue, then call the wrapped state writer."""
|
||||
output_queue.put(state_message)
|
||||
if self.wrapped:
|
||||
return self.wrapped.write_state(state_message)
|
||||
|
||||
self._state_writer = WrappedStateWriter(self._state_writer)
|
||||
self.process_airbyte_messages(
|
||||
messages=messages,
|
||||
write_strategy=write_strategy,
|
||||
)
|
||||
|
||||
# Restore the original state writer
|
||||
self._state_writer = self._state_writer.wrapped
|
||||
|
||||
# Yield all messages from the output queue
|
||||
yield from output_queue.queue
|
||||
|
||||
@final
|
||||
def process_airbyte_messages(
|
||||
self,
|
||||
messages: Iterable[AirbyteMessage],
|
||||
*,
|
||||
write_strategy: WriteStrategy,
|
||||
) -> None:
|
||||
"""Process a stream of Airbyte messages."""
|
||||
if not isinstance(write_strategy, WriteStrategy):
|
||||
raise exc.AirbyteInternalError(
|
||||
message="Invalid `write_strategy` argument. Expected instance of WriteStrategy.",
|
||||
context={"write_strategy": write_strategy},
|
||||
)
|
||||
|
||||
stream_schemas: dict[str, dict] = {}
|
||||
|
||||
# Process messages, writing to batches as we go
|
||||
for message in messages:
|
||||
if message.type is Type.RECORD:
|
||||
record_msg = cast(AirbyteRecordMessage, message.record)
|
||||
stream_name = record_msg.stream
|
||||
|
||||
if stream_name not in stream_schemas:
|
||||
stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema(
|
||||
stream_name=stream_name
|
||||
)
|
||||
|
||||
self.process_record_message(
|
||||
record_msg,
|
||||
stream_schema=stream_schemas[stream_name],
|
||||
)
|
||||
|
||||
elif message.type is Type.STATE:
|
||||
state_msg = cast(AirbyteStateMessage, message.state)
|
||||
if state_msg.type in {AirbyteStateType.GLOBAL, AirbyteStateType.LEGACY}:
|
||||
self._pending_state_messages[f"_{state_msg.type}"].append(state_msg)
|
||||
elif state_msg.type is AirbyteStateType.STREAM:
|
||||
stream_state = cast(AirbyteStreamState, state_msg.stream)
|
||||
stream_name = stream_state.stream_descriptor.name
|
||||
self._pending_state_messages[stream_name].append(state_msg)
|
||||
else:
|
||||
warnings.warn(
|
||||
f"Unexpected state message type. State message was: {state_msg}",
|
||||
stacklevel=2,
|
||||
)
|
||||
self._pending_state_messages[f"_{state_msg.type}"].append(state_msg)
|
||||
|
||||
else:
|
||||
# Ignore unexpected or unhandled message types:
|
||||
# Type.LOG, Type.TRACE, Type.CONTROL, etc.
|
||||
pass
|
||||
|
||||
# We've finished processing input data.
|
||||
# Finalize all received records and state messages:
|
||||
self.write_all_stream_data(
|
||||
write_strategy=write_strategy,
|
||||
)
|
||||
|
||||
self.cleanup_all()
|
||||
|
||||
def write_all_stream_data(self, write_strategy: WriteStrategy) -> None:
|
||||
"""Finalize any pending writes."""
|
||||
for stream_name in self.catalog_provider.stream_names:
|
||||
self.write_stream_data(
|
||||
stream_name,
|
||||
write_strategy=write_strategy,
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
def write_stream_data(
|
||||
self,
|
||||
stream_name: str,
|
||||
write_strategy: WriteStrategy,
|
||||
) -> list[BatchHandle]:
|
||||
"""Write pending stream data to the cache."""
|
||||
...
|
||||
|
||||
def _finalize_state_messages(
|
||||
self,
|
||||
state_messages: list[AirbyteStateMessage],
|
||||
) -> None:
|
||||
"""Handle state messages by passing them to the catalog manager."""
|
||||
if state_messages:
|
||||
self.state_writer.write_state(
|
||||
state_message=state_messages[-1],
|
||||
)
|
||||
|
||||
def _setup(self) -> None: # noqa: B027 # Intentionally empty, not abstract
|
||||
"""Create the database.
|
||||
|
||||
By default this is a no-op but subclasses can override this method to prepare
|
||||
any necessary resources.
|
||||
"""
|
||||
pass
|
||||
|
||||
def cleanup_all(self) -> None: # noqa: B027 # Intentionally empty, not abstract
|
||||
"""Clean up all resources.
|
||||
|
||||
The default implementation is a no-op.
|
||||
"""
|
||||
pass
|
||||
|
|
@ -1,993 +0,0 @@
|
|||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
"""The base SQL Cache implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import contextlib
|
||||
import enum
|
||||
import hashlib
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Optional, Union, cast, final
|
||||
|
||||
import pandas as pd
|
||||
import sqlalchemy
|
||||
import ulid
|
||||
from airbyte import exceptions as exc
|
||||
from airbyte._util.name_normalizers import LowerCaseNormalizer
|
||||
from airbyte.constants import AB_EXTRACTED_AT_COLUMN, AB_META_COLUMN, AB_RAW_ID_COLUMN, DEBUG_MODE
|
||||
from airbyte.progress import progress
|
||||
from airbyte.strategies import WriteStrategy
|
||||
from airbyte.types import SQLTypeConverter
|
||||
from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode
|
||||
from pandas import Index
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import Column, Table, and_, create_engine, insert, null, select, text, update
|
||||
from sqlalchemy.sql.elements import TextClause
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from destination_opengauss_datavec.common.destinations.record_processor import RecordProcessorBase
|
||||
from destination_opengauss_datavec.common.state.state_writers import StdOutStateWriter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from airbyte._batch_handles import BatchHandle
|
||||
from airbyte._processors.file.base import FileWriterBase
|
||||
from airbyte.secrets.base import SecretString
|
||||
from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage
|
||||
from sqlalchemy.engine import Connection, Engine
|
||||
from sqlalchemy.engine.cursor import CursorResult
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.sql.base import Executable
|
||||
from sqlalchemy.sql.type_api import TypeEngine
|
||||
|
||||
from destination_opengauss_datavec.common.catalog.catalog_providers import CatalogProvider
|
||||
from destination_opengauss_datavec.common.state.state_writers import StateWriterBase
|
||||
|
||||
|
||||
class RecordDedupeMode(enum.Enum):
|
||||
APPEND = "append"
|
||||
REPLACE = "replace"
|
||||
|
||||
|
||||
class SQLRuntimeError(Exception):
|
||||
"""Raised when an SQL operation fails."""
|
||||
|
||||
|
||||
class SqlConfig(BaseModel, abc.ABC):
|
||||
"""Common configuration for SQL connections."""
|
||||
|
||||
schema_name: str
|
||||
"""The name of the schema to write to."""
|
||||
|
||||
table_prefix: Optional[str] = ""
|
||||
"""A prefix to add to created table names."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_sql_alchemy_url(self) -> SecretString:
|
||||
"""Returns a SQL Alchemy URL."""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_database_name(self) -> str:
|
||||
"""Return the name of the database."""
|
||||
...
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Attempt to connect, and raise `AirbyteConnectionError` if the connection fails."""
|
||||
engine = self.get_sql_engine()
|
||||
try:
|
||||
connection = engine.connect()
|
||||
connection.close()
|
||||
except Exception as ex:
|
||||
raise exc.AirbyteConnectionError(
|
||||
message="Could not connect to the database.",
|
||||
guidance="Check the connection settings and try again.",
|
||||
) from ex
|
||||
|
||||
def get_sql_engine(self) -> Engine:
|
||||
"""Return a new SQL engine to use."""
|
||||
return create_engine(
|
||||
url=self.get_sql_alchemy_url(),
|
||||
echo=DEBUG_MODE,
|
||||
execution_options={
|
||||
"schema_translate_map": {None: self.schema_name},
|
||||
},
|
||||
)
|
||||
|
||||
def get_vendor_client(self) -> object:
|
||||
"""Return the vendor-specific client object.
|
||||
|
||||
This is used for vendor-specific operations.
|
||||
|
||||
Raises `NotImplementedError` if a custom vendor client is not defined.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"The type '{type(self).__name__}' does not define a custom client."
|
||||
)
|
||||
|
||||
|
||||
class SqlProcessorBase(RecordProcessorBase):
|
||||
"""A base class to be used for SQL Caches."""
|
||||
|
||||
type_converter_class: type[SQLTypeConverter] = SQLTypeConverter
|
||||
"""The type converter class to use for converting JSON schema types to SQL types."""
|
||||
|
||||
normalizer = LowerCaseNormalizer
|
||||
"""The name normalizer to user for table and column name normalization."""
|
||||
|
||||
file_writer_class: type[FileWriterBase]
|
||||
"""The file writer class to use for writing files to the cache."""
|
||||
|
||||
supports_merge_insert = False
|
||||
"""True if the database supports the MERGE INTO syntax."""
|
||||
|
||||
# Constructor:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
sql_config: SqlConfig,
|
||||
catalog_provider: CatalogProvider,
|
||||
state_writer: Union[StateWriterBase, None] = None,
|
||||
file_writer: Union[FileWriterBase, None] = None,
|
||||
temp_dir: Union[Path, None] = None,
|
||||
temp_file_cleanup: bool,
|
||||
) -> None:
|
||||
if not temp_dir and not file_writer:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Either `temp_dir` or `file_writer` must be provided.",
|
||||
)
|
||||
|
||||
state_writer = state_writer or StdOutStateWriter()
|
||||
|
||||
self._sql_config: SqlConfig = sql_config
|
||||
|
||||
super().__init__(
|
||||
state_writer=state_writer,
|
||||
catalog_provider=catalog_provider,
|
||||
)
|
||||
self.file_writer = file_writer or self.file_writer_class(
|
||||
cache_dir=cast(Path, temp_dir),
|
||||
cleanup=temp_file_cleanup,
|
||||
)
|
||||
self.type_converter = self.type_converter_class()
|
||||
self._cached_table_definitions: dict[str, sqlalchemy.Table] = {}
|
||||
self._ensure_schema_exists()
|
||||
|
||||
# Public interface:
|
||||
|
||||
@property
|
||||
def sql_config(self) -> SqlConfig:
|
||||
return self._sql_config
|
||||
|
||||
def get_sql_alchemy_url(self) -> SecretString:
|
||||
"""Return the SQLAlchemy URL to use."""
|
||||
return self.sql_config.get_sql_alchemy_url()
|
||||
|
||||
@final
|
||||
@cached_property
|
||||
def database_name(self) -> str:
|
||||
"""Return the name of the database."""
|
||||
return self.sql_config.get_database_name()
|
||||
|
||||
@final
|
||||
def get_sql_engine(self) -> Engine:
|
||||
"""Return a new SQL engine to use."""
|
||||
return self.sql_config.get_sql_engine()
|
||||
|
||||
@contextmanager
|
||||
def get_sql_connection(self) -> Generator[sqlalchemy.engine.Connection, None, None]:
|
||||
"""A context manager which returns a new SQL connection for running queries.
|
||||
|
||||
If the connection needs to close, it will be closed automatically.
|
||||
"""
|
||||
with self.get_sql_engine().begin() as connection:
|
||||
self._init_connection_settings(connection)
|
||||
yield connection
|
||||
|
||||
connection.close()
|
||||
del connection
|
||||
|
||||
def get_sql_table_name(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> str:
|
||||
"""Return the name of the SQL table for the given stream."""
|
||||
table_prefix = self.sql_config.table_prefix
|
||||
|
||||
# TODO: Add default prefix based on the source name.
|
||||
long_name = f"{table_prefix}{stream_name}"
|
||||
normalized_name = self.normalizer.normalize(long_name)
|
||||
|
||||
if len(normalized_name) <= 63:
|
||||
return normalized_name
|
||||
|
||||
# The max length for an identifier in postgres is 63 characters.
|
||||
# We use a part of the stream name for readability, and a hash for uniqueness to avoid collisions.
|
||||
hash_suffix = hashlib.md5(normalized_name.encode("utf-8")).hexdigest()
|
||||
|
||||
# 63 characters max, 1 for the underscore, 32 for the hash
|
||||
truncate_limit = 63 - 1 - 32
|
||||
|
||||
truncated_name = normalized_name[:truncate_limit]
|
||||
short_name = f"{truncated_name}_{hash_suffix}"
|
||||
logger.warning(
|
||||
f"Table name '{normalized_name}' is too long ({len(normalized_name)} > 63). "
|
||||
f"Truncating to '{short_name}'."
|
||||
)
|
||||
|
||||
return short_name
|
||||
|
||||
@final
|
||||
def get_sql_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> sqlalchemy.Table:
|
||||
"""Return the main table object for the stream."""
|
||||
return self._get_table_by_name(
|
||||
self.get_sql_table_name(stream_name),
|
||||
)
|
||||
|
||||
# Record processing:
|
||||
|
||||
def process_record_message(
|
||||
self,
|
||||
record_msg: AirbyteRecordMessage,
|
||||
stream_schema: dict,
|
||||
) -> None:
|
||||
"""Write a record to the cache.
|
||||
|
||||
This method is called for each record message, before the batch is written.
|
||||
|
||||
In most cases, the SQL processor will not perform any action, but will pass this along to to
|
||||
the file processor.
|
||||
"""
|
||||
self.file_writer.process_record_message(
|
||||
record_msg,
|
||||
stream_schema=stream_schema,
|
||||
)
|
||||
|
||||
# Protected members (non-public interface):
|
||||
|
||||
def _init_connection_settings(self, connection: Connection) -> None:
|
||||
"""This is called automatically whenever a new connection is created.
|
||||
|
||||
By default this is a no-op. Subclasses can use this to set connection settings, such as
|
||||
timezone, case-sensitivity settings, and other session-level variables.
|
||||
"""
|
||||
pass
|
||||
|
||||
def _invalidate_table_cache(
|
||||
self,
|
||||
table_name: str,
|
||||
) -> None:
|
||||
"""Invalidate the the named table cache.
|
||||
|
||||
This should be called whenever the table schema is known to have changed.
|
||||
"""
|
||||
if table_name in self._cached_table_definitions:
|
||||
del self._cached_table_definitions[table_name]
|
||||
|
||||
def _get_table_by_name(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
force_refresh: bool = False,
|
||||
shallow_okay: bool = False,
|
||||
) -> sqlalchemy.Table:
|
||||
"""Return a table object from a table name.
|
||||
|
||||
If 'shallow_okay' is True, the table will be returned without requiring properties to
|
||||
be read from the database.
|
||||
|
||||
To prevent unnecessary round-trips to the database, the table is cached after the first
|
||||
query. To ignore the cache and force a refresh, set 'force_refresh' to True.
|
||||
"""
|
||||
if force_refresh and shallow_okay:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Cannot force refresh and use shallow query at the same time."
|
||||
)
|
||||
|
||||
if force_refresh and table_name in self._cached_table_definitions:
|
||||
self._invalidate_table_cache(table_name)
|
||||
|
||||
if table_name not in self._cached_table_definitions:
|
||||
if shallow_okay:
|
||||
# Return a shallow instance, without column declarations. Do not cache
|
||||
# the table definition in this case.
|
||||
return sqlalchemy.Table(
|
||||
table_name,
|
||||
sqlalchemy.MetaData(schema=self.sql_config.schema_name),
|
||||
)
|
||||
|
||||
self._cached_table_definitions[table_name] = sqlalchemy.Table(
|
||||
table_name,
|
||||
sqlalchemy.MetaData(schema=self.sql_config.schema_name),
|
||||
autoload_with=self.get_sql_engine(),
|
||||
)
|
||||
|
||||
return self._cached_table_definitions[table_name]
|
||||
|
||||
def _ensure_schema_exists(
|
||||
self,
|
||||
) -> None:
|
||||
"""Return a new (unique) temporary table name."""
|
||||
schema_name = self.sql_config.schema_name
|
||||
if schema_name in self._get_schemas_list():
|
||||
return
|
||||
|
||||
sql = f"CREATE SCHEMA IF NOT EXISTS {schema_name}"
|
||||
|
||||
try:
|
||||
self._execute_sql(sql)
|
||||
except Exception as ex:
|
||||
# Ignore schema exists errors.
|
||||
if "already exists" not in str(ex):
|
||||
raise
|
||||
|
||||
if DEBUG_MODE:
|
||||
found_schemas = self._get_schemas_list()
|
||||
assert schema_name in found_schemas, (
|
||||
f"Schema {schema_name} was not created. Found: {found_schemas}"
|
||||
)
|
||||
|
||||
def _quote_identifier(self, identifier: str) -> str:
|
||||
"""Return the given identifier, quoted."""
|
||||
return f'"{identifier}"'
|
||||
|
||||
@final
|
||||
def _get_temp_table_name(
|
||||
self,
|
||||
stream_name: str,
|
||||
batch_id: Union[str, None] = None, # ULID of the batch
|
||||
) -> str:
|
||||
"""Return a new (unique) temporary table name."""
|
||||
batch_id = batch_id or str(ulid.ULID())
|
||||
long_name = f"{stream_name}_{batch_id}"
|
||||
normalized_name = self.normalizer.normalize(long_name)
|
||||
|
||||
if len(normalized_name) <= 63:
|
||||
return normalized_name
|
||||
|
||||
# The max length for an identifier in postgres is 63 characters.
|
||||
# We use a part of the stream name for readability, and a hash for uniqueness to avoid collisions.
|
||||
hash_suffix = hashlib.md5(normalized_name.encode("utf-8")).hexdigest()
|
||||
|
||||
# 63 characters max, 1 for the underscore, 32 for the hash
|
||||
truncate_limit = 63 - 1 - 32
|
||||
|
||||
truncated_name = normalized_name[:truncate_limit]
|
||||
|
||||
return f"{truncated_name}_{hash_suffix}"
|
||||
|
||||
def _fully_qualified(
|
||||
self,
|
||||
table_name: str,
|
||||
) -> str:
|
||||
"""Return the fully qualified name of the given table."""
|
||||
return f"{self.sql_config.schema_name}.{self._quote_identifier(table_name)}"
|
||||
|
||||
@final
|
||||
def _create_table_for_loading(
|
||||
self,
|
||||
/,
|
||||
stream_name: str,
|
||||
batch_id: str,
|
||||
) -> str:
|
||||
"""Create a new table for loading data."""
|
||||
temp_table_name = self._get_temp_table_name(stream_name, batch_id)
|
||||
column_definition_str = ",\n ".join(
|
||||
f"{self._quote_identifier(column_name)} {sql_type}"
|
||||
for column_name, sql_type in self._get_sql_column_definitions(stream_name).items()
|
||||
)
|
||||
self._create_table(temp_table_name, column_definition_str)
|
||||
|
||||
return temp_table_name
|
||||
|
||||
def _get_tables_list(
|
||||
self,
|
||||
) -> list[str]:
|
||||
"""Return a list of all tables in the database."""
|
||||
with self.get_sql_connection() as conn:
|
||||
inspector: Inspector = sqlalchemy.inspect(conn)
|
||||
return inspector.get_table_names(schema=self.sql_config.schema_name)
|
||||
|
||||
def _get_schemas_list(
|
||||
self,
|
||||
database_name: Union[str, None] = None,
|
||||
) -> list[str]:
|
||||
"""Return a list of all tables in the database."""
|
||||
inspector: Inspector = sqlalchemy.inspect(self.get_sql_engine())
|
||||
database_name = database_name or self.database_name
|
||||
found_schemas = inspector.get_schema_names()
|
||||
return [
|
||||
found_schema.split(".")[-1].strip('"')
|
||||
for found_schema in found_schemas
|
||||
if "." not in found_schema
|
||||
or (found_schema.split(".")[0].lower().strip('"') == database_name.lower())
|
||||
]
|
||||
|
||||
def _ensure_final_table_exists(
|
||||
self,
|
||||
stream_name: str,
|
||||
*,
|
||||
create_if_missing: bool = True,
|
||||
) -> str:
|
||||
"""Create the final table if it doesn't already exist.
|
||||
|
||||
Return the table name.
|
||||
"""
|
||||
table_name = self.get_sql_table_name(stream_name)
|
||||
did_exist = self._table_exists(table_name)
|
||||
if not did_exist and create_if_missing:
|
||||
column_definition_str = ",\n ".join(
|
||||
f"{self._quote_identifier(column_name)} {sql_type}"
|
||||
for column_name, sql_type in self._get_sql_column_definitions(
|
||||
stream_name,
|
||||
).items()
|
||||
)
|
||||
self._create_table(table_name, column_definition_str)
|
||||
|
||||
return table_name
|
||||
|
||||
def _ensure_compatible_table_schema(
|
||||
self,
|
||||
stream_name: str,
|
||||
table_name: str,
|
||||
) -> None:
|
||||
"""Return true if the given table is compatible with the stream's schema.
|
||||
|
||||
Raises an exception if the table schema is not compatible with the schema of the
|
||||
input stream.
|
||||
|
||||
TODO:
|
||||
- Expand this to check for column types and sizes.
|
||||
"""
|
||||
self._add_missing_columns_to_table(
|
||||
stream_name=stream_name,
|
||||
table_name=table_name,
|
||||
)
|
||||
|
||||
@final
|
||||
def _create_table(
|
||||
self,
|
||||
table_name: str,
|
||||
column_definition_str: str,
|
||||
primary_keys: Union[list[str], None] = None,
|
||||
) -> None:
|
||||
if primary_keys:
|
||||
pk_str = ", ".join(primary_keys)
|
||||
column_definition_str += f",\n PRIMARY KEY ({pk_str})"
|
||||
|
||||
cmd = f"""
|
||||
CREATE TABLE {self._fully_qualified(table_name)} (
|
||||
{column_definition_str}
|
||||
)
|
||||
"""
|
||||
_ = self._execute_sql(cmd)
|
||||
|
||||
def _get_sql_column_definitions(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> dict[str, sqlalchemy.types.TypeEngine]:
|
||||
"""Return the column definitions for the given stream."""
|
||||
columns: dict[str, sqlalchemy.types.TypeEngine] = {}
|
||||
properties = self.catalog_provider.get_stream_properties(stream_name)
|
||||
for property_name, json_schema_property_def in properties.items():
|
||||
clean_prop_name = self.normalizer.normalize(property_name)
|
||||
columns[clean_prop_name] = self.type_converter.to_sql_type(
|
||||
json_schema_property_def,
|
||||
)
|
||||
|
||||
columns[AB_RAW_ID_COLUMN] = self.type_converter_class.get_string_type()
|
||||
columns[AB_EXTRACTED_AT_COLUMN] = sqlalchemy.TIMESTAMP()
|
||||
columns[AB_META_COLUMN] = self.type_converter_class.get_json_type()
|
||||
|
||||
return columns
|
||||
|
||||
@final
|
||||
def write_stream_data(
|
||||
self,
|
||||
stream_name: str,
|
||||
write_strategy: WriteStrategy,
|
||||
) -> list[BatchHandle]:
|
||||
"""Finalize all uncommitted batches.
|
||||
|
||||
This is a generic 'final' SQL implementation, which should not be overridden.
|
||||
|
||||
Returns a mapping of batch IDs to batch handles, for those processed batches.
|
||||
|
||||
TODO: Add a dedupe step here to remove duplicates from the temp table.
|
||||
Some sources will send us duplicate records within the same stream,
|
||||
although this is a fairly rare edge case we can ignore in V1.
|
||||
"""
|
||||
# Flush any pending writes
|
||||
self.file_writer.flush_active_batches()
|
||||
|
||||
with self.finalizing_batches(stream_name) as batches_to_finalize:
|
||||
# Make sure the target schema and target table exist.
|
||||
self._ensure_schema_exists()
|
||||
final_table_name = self._ensure_final_table_exists(
|
||||
stream_name,
|
||||
create_if_missing=True,
|
||||
)
|
||||
|
||||
if not batches_to_finalize:
|
||||
# If there are no batches to finalize, return after ensuring the table exists.
|
||||
return []
|
||||
|
||||
files: list[Path] = []
|
||||
# Get a list of all files to finalize from all pending batches.
|
||||
for batch_handle in batches_to_finalize:
|
||||
files += batch_handle.files
|
||||
# Use the max batch ID as the batch ID for table names.
|
||||
max_batch_id = max(batch.batch_id for batch in batches_to_finalize)
|
||||
|
||||
temp_table_name = self._write_files_to_new_table(
|
||||
files=files,
|
||||
stream_name=stream_name,
|
||||
batch_id=max_batch_id,
|
||||
)
|
||||
try:
|
||||
self._write_temp_table_to_final_table(
|
||||
stream_name=stream_name,
|
||||
temp_table_name=temp_table_name,
|
||||
final_table_name=final_table_name,
|
||||
write_strategy=write_strategy,
|
||||
)
|
||||
finally:
|
||||
self._drop_temp_table(temp_table_name, if_exists=True)
|
||||
|
||||
progress.log_stream_finalized(stream_name)
|
||||
|
||||
# Return the batch handles as measure of work completed.
|
||||
return batches_to_finalize
|
||||
|
||||
@final
|
||||
def cleanup_all(self) -> None:
|
||||
"""Clean resources."""
|
||||
self.file_writer.cleanup_all()
|
||||
|
||||
# Finalizing context manager
|
||||
|
||||
@final
|
||||
@contextlib.contextmanager
|
||||
def finalizing_batches(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> Generator[list[BatchHandle], str, None]:
|
||||
"""Context manager to use for finalizing batches, if applicable.
|
||||
|
||||
Returns a mapping of batch IDs to batch handles, for those processed batches.
|
||||
"""
|
||||
batches_to_finalize: list[BatchHandle] = self.file_writer.get_pending_batches(stream_name)
|
||||
state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[
|
||||
stream_name
|
||||
].copy()
|
||||
self._pending_state_messages[stream_name].clear()
|
||||
|
||||
progress.log_batches_finalizing(stream_name, len(batches_to_finalize))
|
||||
yield batches_to_finalize
|
||||
self._finalize_state_messages(state_messages_to_finalize)
|
||||
progress.log_batches_finalized(stream_name, len(batches_to_finalize))
|
||||
|
||||
for batch_handle in batches_to_finalize:
|
||||
batch_handle.finalized = True
|
||||
|
||||
self._finalized_state_messages[stream_name] += state_messages_to_finalize
|
||||
|
||||
def _execute_sql(self, sql: Union[str, TextClause, Executable]) -> CursorResult:
|
||||
"""Execute the given SQL statement."""
|
||||
if isinstance(sql, str):
|
||||
sql = text(sql)
|
||||
if isinstance(sql, TextClause):
|
||||
sql = sql.execution_options(
|
||||
autocommit=True,
|
||||
)
|
||||
|
||||
with self.get_sql_connection() as conn:
|
||||
try:
|
||||
result = conn.execute(sql)
|
||||
except (
|
||||
sqlalchemy.exc.ProgrammingError,
|
||||
sqlalchemy.exc.SQLAlchemyError,
|
||||
) as ex:
|
||||
msg = f"Error when executing SQL:\n{sql}\n{type(ex).__name__}{ex!s}"
|
||||
raise SQLRuntimeError(msg) from None # from ex
|
||||
|
||||
return result
|
||||
|
||||
def _drop_temp_table(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
if_exists: bool = True,
|
||||
) -> None:
|
||||
"""Drop the given table."""
|
||||
exists_str = "IF EXISTS" if if_exists else ""
|
||||
self._execute_sql(f"DROP TABLE {exists_str} {self._fully_qualified(table_name)}")
|
||||
|
||||
def _write_files_to_new_table(
|
||||
self,
|
||||
files: list[Path],
|
||||
stream_name: str,
|
||||
batch_id: str,
|
||||
) -> str:
|
||||
"""Write a file(s) to a new table.
|
||||
|
||||
This is a generic implementation, which can be overridden by subclasses
|
||||
to improve performance.
|
||||
"""
|
||||
temp_table_name = self._create_table_for_loading(stream_name, batch_id)
|
||||
for file_path in files:
|
||||
dataframe = pd.read_json(file_path, lines=True)
|
||||
|
||||
sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions(
|
||||
stream_name
|
||||
)
|
||||
|
||||
# Remove fields that are not in the schema
|
||||
for col_name in dataframe.columns:
|
||||
if col_name not in sql_column_definitions:
|
||||
dataframe = dataframe.drop(columns=col_name)
|
||||
|
||||
# Pandas will auto-create the table if it doesn't exist, which we don't want.
|
||||
if not self._table_exists(temp_table_name):
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Table does not exist after creation.",
|
||||
context={
|
||||
"temp_table_name": temp_table_name,
|
||||
},
|
||||
)
|
||||
|
||||
# Normalize all column names to lower case.
|
||||
dataframe.columns = Index([self.normalizer.normalize(col) for col in dataframe.columns])
|
||||
|
||||
# Write the data to the table.
|
||||
dataframe.to_sql(
|
||||
temp_table_name,
|
||||
self.get_sql_alchemy_url(),
|
||||
schema=self.sql_config.schema_name,
|
||||
if_exists="append",
|
||||
index=False,
|
||||
dtype=sql_column_definitions,
|
||||
)
|
||||
return temp_table_name
|
||||
|
||||
def _add_column_to_table(
|
||||
self,
|
||||
table: Table,
|
||||
column_name: str,
|
||||
column_type: sqlalchemy.types.TypeEngine,
|
||||
) -> None:
|
||||
"""Add a column to the given table."""
|
||||
self._execute_sql(
|
||||
text(
|
||||
f"ALTER TABLE {self._fully_qualified(table.name)} "
|
||||
f"ADD COLUMN {column_name} {column_type}"
|
||||
),
|
||||
)
|
||||
|
||||
def _add_missing_columns_to_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
table_name: str,
|
||||
) -> None:
|
||||
"""Add missing columns to the table.
|
||||
|
||||
This is a no-op if all columns are already present.
|
||||
"""
|
||||
columns = self._get_sql_column_definitions(stream_name)
|
||||
# First check without forcing a refresh of the cache (faster). If nothing is missing,
|
||||
# then we're done.
|
||||
table = self._get_table_by_name(
|
||||
table_name,
|
||||
force_refresh=False,
|
||||
)
|
||||
missing_columns: bool = any(column_name not in table.columns for column_name in columns)
|
||||
|
||||
if missing_columns:
|
||||
# If we found missing columns, refresh the cache and then take action on anything
|
||||
# that's still confirmed missing.
|
||||
columns_added = False
|
||||
table = self._get_table_by_name(
|
||||
table_name,
|
||||
force_refresh=True,
|
||||
)
|
||||
for column_name, column_type in columns.items():
|
||||
if column_name not in table.columns:
|
||||
self._add_column_to_table(table, column_name, column_type)
|
||||
columns_added = True
|
||||
|
||||
if columns_added:
|
||||
# We've added columns, so invalidate the cache.
|
||||
self._invalidate_table_cache(table_name)
|
||||
|
||||
@final
|
||||
def _write_temp_table_to_final_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
temp_table_name: str,
|
||||
final_table_name: str,
|
||||
write_strategy: WriteStrategy,
|
||||
) -> None:
|
||||
"""Write the temp table into the final table using the provided write strategy."""
|
||||
has_pks: bool = bool(self._get_primary_keys(stream_name))
|
||||
has_incremental_key: bool = bool(self._get_incremental_key(stream_name))
|
||||
if write_strategy == WriteStrategy.MERGE and not has_pks:
|
||||
raise exc.PyAirbyteInputError(
|
||||
message="Cannot use merge strategy on a stream with no primary keys.",
|
||||
context={
|
||||
"stream_name": stream_name,
|
||||
},
|
||||
)
|
||||
|
||||
if write_strategy == WriteStrategy.AUTO:
|
||||
configured_destination_sync_mode: DestinationSyncMode = (
|
||||
self.catalog_provider.get_destination_sync_mode(stream_name)
|
||||
)
|
||||
if configured_destination_sync_mode == DestinationSyncMode.overwrite:
|
||||
write_strategy = WriteStrategy.REPLACE
|
||||
elif configured_destination_sync_mode == DestinationSyncMode.append:
|
||||
write_strategy = WriteStrategy.APPEND
|
||||
elif configured_destination_sync_mode == DestinationSyncMode.append_dedup:
|
||||
write_strategy = WriteStrategy.MERGE
|
||||
|
||||
# TODO: Consider removing the rest of these cases if they are dead code.
|
||||
elif has_pks:
|
||||
write_strategy = WriteStrategy.MERGE
|
||||
elif has_incremental_key:
|
||||
write_strategy = WriteStrategy.APPEND
|
||||
else:
|
||||
write_strategy = WriteStrategy.REPLACE
|
||||
|
||||
if write_strategy == WriteStrategy.REPLACE:
|
||||
# Note: No need to check for schema compatibility
|
||||
# here, because we are fully replacing the table.
|
||||
self._swap_temp_table_with_final_table(
|
||||
stream_name=stream_name,
|
||||
temp_table_name=temp_table_name,
|
||||
final_table_name=final_table_name,
|
||||
)
|
||||
return
|
||||
|
||||
if write_strategy == WriteStrategy.APPEND:
|
||||
self._ensure_compatible_table_schema(
|
||||
stream_name=stream_name,
|
||||
table_name=final_table_name,
|
||||
)
|
||||
self._append_temp_table_to_final_table(
|
||||
stream_name=stream_name,
|
||||
temp_table_name=temp_table_name,
|
||||
final_table_name=final_table_name,
|
||||
)
|
||||
return
|
||||
|
||||
if write_strategy == WriteStrategy.MERGE:
|
||||
self._ensure_compatible_table_schema(
|
||||
stream_name=stream_name,
|
||||
table_name=final_table_name,
|
||||
)
|
||||
if not self.supports_merge_insert:
|
||||
# Fallback to emulated merge if the database does not support merge natively.
|
||||
self._emulated_merge_temp_table_to_final_table(
|
||||
stream_name=stream_name,
|
||||
temp_table_name=temp_table_name,
|
||||
final_table_name=final_table_name,
|
||||
)
|
||||
return
|
||||
|
||||
self._merge_temp_table_to_final_table(
|
||||
stream_name=stream_name,
|
||||
temp_table_name=temp_table_name,
|
||||
final_table_name=final_table_name,
|
||||
)
|
||||
return
|
||||
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Write strategy is not supported.",
|
||||
context={
|
||||
"write_strategy": write_strategy,
|
||||
},
|
||||
)
|
||||
|
||||
def _append_temp_table_to_final_table(
|
||||
self,
|
||||
temp_table_name: str,
|
||||
final_table_name: str,
|
||||
stream_name: str,
|
||||
) -> None:
|
||||
nl = "\n"
|
||||
columns = [self._quote_identifier(c) for c in self._get_sql_column_definitions(stream_name)]
|
||||
self._execute_sql(
|
||||
f"""
|
||||
INSERT INTO {self._fully_qualified(final_table_name)} (
|
||||
{f",{nl} ".join(columns)}
|
||||
)
|
||||
SELECT
|
||||
{f",{nl} ".join(columns)}
|
||||
FROM {self._fully_qualified(temp_table_name)}
|
||||
""",
|
||||
)
|
||||
|
||||
def _get_primary_keys(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> list[str]:
|
||||
pks = self.catalog_provider.get_configured_stream_info(stream_name).primary_key
|
||||
if not pks:
|
||||
return []
|
||||
|
||||
joined_pks = [".".join(pk) for pk in pks]
|
||||
for pk in joined_pks:
|
||||
if "." in pk:
|
||||
msg = f"Nested primary keys are not yet supported. Found: {pk}"
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
return joined_pks
|
||||
|
||||
def _get_incremental_key(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> Union[str, None]:
|
||||
return self.catalog_provider.get_configured_stream_info(stream_name).cursor_field
|
||||
|
||||
def _swap_temp_table_with_final_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
temp_table_name: str,
|
||||
final_table_name: str,
|
||||
) -> None:
|
||||
"""Merge the temp table into the main one.
|
||||
|
||||
This implementation requires MERGE support in the SQL DB.
|
||||
Databases that do not support this syntax can override this method.
|
||||
"""
|
||||
if final_table_name is None:
|
||||
raise exc.PyAirbyteInternalError(message="Arg 'final_table_name' cannot be None.")
|
||||
if temp_table_name is None:
|
||||
raise exc.PyAirbyteInternalError(message="Arg 'temp_table_name' cannot be None.")
|
||||
|
||||
_ = stream_name
|
||||
deletion_name = f"{final_table_name}_deleteme"
|
||||
commands = "\n".join([
|
||||
f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};",
|
||||
f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME TO {final_table_name};",
|
||||
f"DROP TABLE {self._fully_qualified(deletion_name)};",
|
||||
])
|
||||
self._execute_sql(commands)
|
||||
|
||||
def _merge_temp_table_to_final_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
temp_table_name: str,
|
||||
final_table_name: str,
|
||||
) -> None:
|
||||
"""Merge the temp table into the main one.
|
||||
|
||||
This implementation requires MERGE support in the SQL DB.
|
||||
Databases that do not support this syntax can override this method.
|
||||
"""
|
||||
nl = "\n"
|
||||
columns = {self._quote_identifier(c) for c in self._get_sql_column_definitions(stream_name)}
|
||||
pk_columns = {self._quote_identifier(c) for c in self._get_primary_keys(stream_name)}
|
||||
non_pk_columns = columns - pk_columns
|
||||
join_clause = f"{nl} AND ".join(f"tmp.{pk_col} = final.{pk_col}" for pk_col in pk_columns)
|
||||
set_clause = f"{nl} , ".join(f"{col} = tmp.{col}" for col in non_pk_columns)
|
||||
self._execute_sql(
|
||||
f"""
|
||||
MERGE INTO {self._fully_qualified(final_table_name)} final
|
||||
USING (
|
||||
SELECT *
|
||||
FROM {self._fully_qualified(temp_table_name)}
|
||||
) AS tmp
|
||||
ON {join_clause}
|
||||
WHEN MATCHED THEN UPDATE
|
||||
SET
|
||||
{set_clause}
|
||||
WHEN NOT MATCHED THEN INSERT
|
||||
(
|
||||
{f",{nl} ".join(columns)}
|
||||
)
|
||||
VALUES (
|
||||
tmp.{f",{nl} tmp.".join(columns)}
|
||||
);
|
||||
""",
|
||||
)
|
||||
|
||||
def _get_column_by_name(self, table: Union[str, Table], column_name: str) -> Column:
|
||||
"""Return the column object for the given column name.
|
||||
|
||||
This method is case-insensitive.
|
||||
"""
|
||||
if isinstance(table, str):
|
||||
table = self._get_table_by_name(table)
|
||||
try:
|
||||
# Try to get the column in a case-insensitive manner
|
||||
return next(col for col in table.c if col.name.lower() == column_name.lower())
|
||||
except StopIteration:
|
||||
raise exc.PyAirbyteInternalError(
|
||||
message="Could not find matching column.",
|
||||
context={
|
||||
"table": table,
|
||||
"column_name": column_name,
|
||||
},
|
||||
) from None
|
||||
|
||||
def _emulated_merge_temp_table_to_final_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
temp_table_name: str,
|
||||
final_table_name: str,
|
||||
) -> None:
|
||||
"""Emulate the merge operation using a series of SQL commands.
|
||||
|
||||
This is a fallback implementation for databases that do not support MERGE.
|
||||
"""
|
||||
final_table = self._get_table_by_name(final_table_name)
|
||||
temp_table = self._get_table_by_name(temp_table_name)
|
||||
pk_columns = self._get_primary_keys(stream_name)
|
||||
|
||||
columns_to_update: set[str] = self._get_sql_column_definitions(
|
||||
stream_name=stream_name
|
||||
).keys() - set(pk_columns)
|
||||
|
||||
# Create a dictionary mapping columns in users_final to users_stage for updating
|
||||
update_values = {
|
||||
self._get_column_by_name(final_table, column): (
|
||||
self._get_column_by_name(temp_table, column)
|
||||
)
|
||||
for column in columns_to_update
|
||||
}
|
||||
|
||||
# Craft the WHERE clause for composite primary keys
|
||||
join_conditions = [
|
||||
self._get_column_by_name(final_table, pk_column)
|
||||
== self._get_column_by_name(temp_table, pk_column)
|
||||
for pk_column in pk_columns
|
||||
]
|
||||
join_clause = and_(*join_conditions)
|
||||
|
||||
# Craft the UPDATE statement
|
||||
update_stmt = update(final_table).values(update_values).where(join_clause)
|
||||
|
||||
# Define a join between temp_table and final_table
|
||||
joined_table = temp_table.outerjoin(final_table, join_clause)
|
||||
|
||||
# Define a condition that checks for records in temp_table that do not have a corresponding
|
||||
# record in final_table
|
||||
where_not_exists_clause = self._get_column_by_name(final_table, pk_columns[0]) == null()
|
||||
|
||||
# Select records from temp_table that are not in final_table
|
||||
select_new_records_stmt = (
|
||||
select([temp_table]).select_from(joined_table).where(where_not_exists_clause)
|
||||
)
|
||||
|
||||
# Craft the INSERT statement using the select statement
|
||||
insert_new_records_stmt = insert(final_table).from_select(
|
||||
names=[column.name for column in temp_table.columns], select=select_new_records_stmt
|
||||
)
|
||||
|
||||
if DEBUG_MODE:
|
||||
print(str(update_stmt))
|
||||
print(str(insert_new_records_stmt))
|
||||
|
||||
with self.get_sql_connection() as conn:
|
||||
conn.execute(update_stmt)
|
||||
conn.execute(insert_new_records_stmt)
|
||||
|
||||
def _table_exists(
|
||||
self,
|
||||
table_name: str,
|
||||
) -> bool:
|
||||
"""Return true if the given table exists.
|
||||
|
||||
Subclasses may override this method to provide a more efficient implementation.
|
||||
"""
|
||||
return table_name in self._get_tables_list()
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
|
||||
"""State writer implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from airbyte_cdk.models.airbyte_protocol import AirbyteStateMessage
|
||||
|
||||
|
||||
class StateWriterBase(abc.ABC):
|
||||
"""A class to write state artifacts."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def write_state(
|
||||
self,
|
||||
state_message: AirbyteStateMessage,
|
||||
) -> None:
|
||||
"""Save or 'write' a state artifact."""
|
||||
...
|
||||
|
||||
|
||||
class StdOutStateWriter(StateWriterBase):
|
||||
"""A state writer that writes state artifacts to stdout.
|
||||
|
||||
This is required when we are functioning as a "Destination" in the Airbyte protocol, and
|
||||
an orchestrator is responsible for saving those state artifacts.
|
||||
"""
|
||||
|
||||
def write_state(
|
||||
self,
|
||||
state_message: AirbyteStateMessage,
|
||||
) -> None:
|
||||
"""Save or 'write' a state artifact."""
|
||||
print(state_message.json())
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class OpenGaussDatavecIndexingModel(BaseModel):
|
||||
host: str = Field(
|
||||
...,
|
||||
title="Host",
|
||||
order=1,
|
||||
description="Hostname of the database.",
|
||||
)
|
||||
database: str = Field(
|
||||
...,
|
||||
title="Database",
|
||||
order=2,
|
||||
description="Enter the name of the database that you want to sync data into",
|
||||
examples=["AIRBYTE_DATABASE"],
|
||||
)
|
||||
username: str = Field(
|
||||
...,
|
||||
title="Username",
|
||||
order=3,
|
||||
description="Enter the name of the user you want to use to access the database",
|
||||
examples=["root"],
|
||||
)
|
||||
password: str = Field(
|
||||
...,
|
||||
title="Password",
|
||||
order=4,
|
||||
description="Enter the password of the user you want to use to access the database",
|
||||
)
|
||||
|
||||
port: int = Field(
|
||||
default=5432,
|
||||
title="Port",
|
||||
description="Enter the port you want to use to access the database",
|
||||
examples=["5432"],
|
||||
)
|
||||
default_schema: str = Field(
|
||||
default="public",
|
||||
title="Default Schema",
|
||||
order=6,
|
||||
description="Enter the name of the default schema",
|
||||
examples=["AIRBYTE_SCHEMA"],
|
||||
)
|
||||
|
||||
|
||||
class Config:
|
||||
title = "Indexing"
|
||||
schema_extra = {
|
||||
"description": "Indexing configuration",
|
||||
"group": "indexing",
|
||||
}
|
||||
|
||||
|
||||
class ConfigModel(VectorDBConfigModel):
|
||||
indexing: OpenGaussDatavecIndexingModel
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import tempfile
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping, Optional
|
||||
|
||||
from airbyte.secrets import SecretString
|
||||
from airbyte.strategies import WriteStrategy
|
||||
from airbyte_cdk.destinations import Destination
|
||||
from airbyte_cdk.models import (
|
||||
AirbyteConnectionStatus,
|
||||
AirbyteMessage,
|
||||
ConfiguredAirbyteCatalog,
|
||||
ConnectorSpecification,
|
||||
DestinationSyncMode,
|
||||
Status,
|
||||
)
|
||||
|
||||
from destination_opengauss_datavec import opengauss_processor
|
||||
from destination_opengauss_datavec.config import ConfigModel
|
||||
from destination_opengauss_datavec.common.catalog.catalog_providers import CatalogProvider
|
||||
|
||||
|
||||
BATCH_SIZE = 150
|
||||
|
||||
|
||||
class DestinationOpenGaussDataVec(Destination):
|
||||
sql_processor: opengauss_processor.OpenGaussDataVecProcessor
|
||||
|
||||
def _init_sql_processor(
|
||||
self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None
|
||||
) -> None:
|
||||
self.sql_processor = opengauss_processor.OpenGaussDataVecProcessor(
|
||||
sql_config=opengauss_processor.OpenGaussConfig(
|
||||
host=config.indexing.host,
|
||||
port=config.indexing.port,
|
||||
database=config.indexing.database,
|
||||
schema_name=config.indexing.default_schema,
|
||||
username=config.indexing.username,
|
||||
password=SecretString(config.indexing.password),
|
||||
),
|
||||
splitter_config=config.processing,
|
||||
embedder_config=config.embedding, # type: ignore [arg-type]
|
||||
catalog_provider=CatalogProvider(configured_catalog),
|
||||
temp_dir=Path(tempfile.mkdtemp()),
|
||||
temp_file_cleanup=True,
|
||||
)
|
||||
|
||||
def write(
|
||||
self,
|
||||
config: Mapping[str, Any],
|
||||
configured_catalog: ConfiguredAirbyteCatalog,
|
||||
input_messages: Iterable[AirbyteMessage],
|
||||
) -> Iterable[AirbyteMessage]:
|
||||
parsed_config = ConfigModel.parse_obj(config)
|
||||
self._init_sql_processor(config=parsed_config, configured_catalog=configured_catalog)
|
||||
yield from self.sql_processor.process_airbyte_messages_as_generator(
|
||||
messages=input_messages,
|
||||
write_strategy=WriteStrategy.AUTO,
|
||||
)
|
||||
|
||||
def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
|
||||
_ = logger # Unused
|
||||
try:
|
||||
parsed_config = ConfigModel.parse_obj(config)
|
||||
self._init_sql_processor(config=parsed_config)
|
||||
self.sql_processor.sql_config.get_sql_engine().connect()
|
||||
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
|
||||
except Exception as e:
|
||||
return AirbyteConnectionStatus(
|
||||
status=Status.FAILED, message=f"An exception occurred: {repr(e)}"
|
||||
)
|
||||
|
||||
def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
|
||||
return ConnectorSpecification(
|
||||
documentationUrl="https://docs.airbyte.com/integrations/destinations/opengauss-datavec",
|
||||
supportsIncremental=True,
|
||||
supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup],
|
||||
connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined]
|
||||
)
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
"""Globals and constants for the destination."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DOCUMENT_ID_COLUMN = "document_id"
|
||||
CHUNK_ID_COLUMN = "chunk_id"
|
||||
METADATA_COLUMN = "metadata"
|
||||
DOCUMENT_CONTENT_COLUMN = "document_content"
|
||||
EMBEDDING_COLUMN = "embedding"
|
||||
|
|
@ -1,275 +0,0 @@
|
|||
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
"""An OpenGauss DataVec implementation of the SQL processor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Union
|
||||
|
||||
import dpath
|
||||
import sqlalchemy
|
||||
from airbyte.secrets import SecretString
|
||||
from airbyte._processors.file.jsonl import JsonlWriter
|
||||
from airbyte_cdk.destinations.vector_db_based import embedder
|
||||
from airbyte_cdk.destinations.vector_db_based.document_processor import (
|
||||
DocumentProcessor as DocumentSplitter,
|
||||
)
|
||||
from airbyte_cdk.destinations.vector_db_based.document_processor import (
|
||||
ProcessingConfigModel as DocumentSplitterConfig,
|
||||
)
|
||||
from airbyte_cdk.models import AirbyteRecordMessage
|
||||
from sqlalchemy import (
|
||||
insert,
|
||||
select,
|
||||
delete,
|
||||
)
|
||||
from overrides import overrides
|
||||
from typing_extensions import Protocol
|
||||
from destination_opengauss_datavec.common.catalog.catalog_providers import CatalogProvider
|
||||
from destination_opengauss_datavec.globals import (
|
||||
CHUNK_ID_COLUMN,
|
||||
DOCUMENT_CONTENT_COLUMN,
|
||||
DOCUMENT_ID_COLUMN,
|
||||
EMBEDDING_COLUMN,
|
||||
METADATA_COLUMN,
|
||||
)
|
||||
|
||||
|
||||
# OpenGauss DataVec vector type definition
|
||||
from opengauss_sqlalchemy.usertype import Vector
|
||||
|
||||
# Import the SQL config base class
|
||||
from destination_opengauss_datavec.common.sql.sql_processor import SqlConfig, SqlProcessorBase
|
||||
|
||||
|
||||
class OpenGaussConfig(SqlConfig):
|
||||
"""Configuration for the OpenGauss DataVec cache.
|
||||
|
||||
Also inherits config from the JsonlWriter, which is responsible for writing files to disk.
|
||||
"""
|
||||
|
||||
host: str
|
||||
port: int
|
||||
database: str
|
||||
username: str
|
||||
password: Union[SecretString, str]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
database: str,
|
||||
username: str,
|
||||
password: Union[SecretString, str],
|
||||
schema_name: str = "public",
|
||||
):
|
||||
# Initialize parent SqlConfig
|
||||
super().__init__(
|
||||
schema_name=schema_name,
|
||||
host=host,
|
||||
port=port,
|
||||
database=database,
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
|
||||
@overrides
|
||||
def get_sql_alchemy_url(self) -> SecretString:
|
||||
"""Return the SQLAlchemy URL to use."""
|
||||
url = sqlalchemy.engine.URL.create(
|
||||
drivername="opengauss+psycopg2",
|
||||
username=str(self.username),
|
||||
password=str(self.password),
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
database=self.database,
|
||||
)
|
||||
return SecretString(str(url))
|
||||
|
||||
@overrides
|
||||
def get_database_name(self) -> str:
|
||||
"""Return the name of the database."""
|
||||
return self.database
|
||||
|
||||
|
||||
class EmbeddingConfig(Protocol):
|
||||
"""A protocol for embedding configuration."""
|
||||
mode: str
|
||||
|
||||
|
||||
class OpenGaussDataVecProcessor(SqlProcessorBase):
|
||||
"""An OpenGauss DataVec implementation of the SQL Processor."""
|
||||
|
||||
supports_merge_insert = False
|
||||
"""We use the emulated merge code path because each primary key has multiple rows (chunks)."""
|
||||
|
||||
sql_config: OpenGaussConfig
|
||||
"""The configuration for the OpenGauss DataVec processor, including the vector length."""
|
||||
|
||||
splitter_config: DocumentSplitterConfig
|
||||
"""The configuration for the document splitter."""
|
||||
|
||||
file_writer_class = JsonlWriter
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sql_config: OpenGaussConfig,
|
||||
splitter_config: DocumentSplitterConfig,
|
||||
embedder_config: EmbeddingConfig,
|
||||
catalog_provider: CatalogProvider,
|
||||
temp_dir: Path,
|
||||
temp_file_cleanup: bool = True,
|
||||
) -> None:
|
||||
"""Initialize the OpenGauss DataVec processor."""
|
||||
self.splitter_config = splitter_config
|
||||
self.embedder_config = embedder_config
|
||||
super().__init__(
|
||||
sql_config=sql_config,
|
||||
catalog_provider=catalog_provider,
|
||||
temp_dir=temp_dir,
|
||||
temp_file_cleanup=temp_file_cleanup,
|
||||
)
|
||||
|
||||
def _get_sql_column_definitions(
|
||||
self,
|
||||
stream_name: str,
|
||||
) -> dict[str, sqlalchemy.types.TypeEngine]:
|
||||
"""Return the column definitions for the given stream."""
|
||||
_ = stream_name # unused
|
||||
return {
|
||||
DOCUMENT_ID_COLUMN: sqlalchemy.String(),
|
||||
CHUNK_ID_COLUMN: sqlalchemy.String(),
|
||||
METADATA_COLUMN: sqlalchemy.JSON(),
|
||||
DOCUMENT_CONTENT_COLUMN: sqlalchemy.Text(),
|
||||
EMBEDDING_COLUMN: Vector(self.embedding_dimensions),
|
||||
}
|
||||
|
||||
def _emulated_merge_temp_table_to_final_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
temp_table_name: str,
|
||||
final_table_name: str,
|
||||
) -> None:
|
||||
"""Emulate the merge operation using a series of SQL commands."""
|
||||
final_table = self._get_table_by_name(final_table_name)
|
||||
temp_table = self._get_table_by_name(temp_table_name)
|
||||
|
||||
columns_list: list[str] = list(
|
||||
self._get_sql_column_definitions(stream_name=stream_name).keys()
|
||||
)
|
||||
|
||||
subquery = select(temp_table.c[DOCUMENT_ID_COLUMN])
|
||||
delete_statement = delete(final_table).where(
|
||||
final_table.c[DOCUMENT_ID_COLUMN].in_(subquery)
|
||||
)
|
||||
|
||||
select_all_from_temp = select(temp_table)
|
||||
insert_statement = insert(final_table).from_select(
|
||||
columns_list, select_all_from_temp
|
||||
)
|
||||
|
||||
with self.get_sql_connection() as conn:
|
||||
# This is a transactional operation to avoid "outages"
|
||||
conn.execute(delete_statement)
|
||||
conn.execute(insert_statement)
|
||||
|
||||
def process_record_message(
|
||||
self,
|
||||
record_msg: AirbyteRecordMessage,
|
||||
stream_schema: dict,
|
||||
) -> None:
|
||||
"""Write a record to the cache."""
|
||||
document_chunks, id_to_delete = self.splitter.process(record_msg)
|
||||
|
||||
_ = id_to_delete # unused
|
||||
|
||||
embeddings = self.embedder.embed_documents(
|
||||
documents=document_chunks,
|
||||
)
|
||||
for i, chunk in enumerate(document_chunks, start=0):
|
||||
new_data: dict[str, Any] = {
|
||||
DOCUMENT_ID_COLUMN: self._create_document_id(record_msg),
|
||||
CHUNK_ID_COLUMN: str(uuid.uuid4().int),
|
||||
METADATA_COLUMN: chunk.metadata,
|
||||
DOCUMENT_CONTENT_COLUMN: chunk.page_content,
|
||||
EMBEDDING_COLUMN: embeddings[i],
|
||||
}
|
||||
|
||||
self.file_writer.process_record_message(
|
||||
record_msg=AirbyteRecordMessage(
|
||||
namespace=record_msg.namespace,
|
||||
stream=record_msg.stream,
|
||||
data=new_data,
|
||||
emitted_at=record_msg.emitted_at,
|
||||
),
|
||||
stream_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
DOCUMENT_ID_COLUMN: {"type": "string"},
|
||||
CHUNK_ID_COLUMN: {"type": "string"},
|
||||
METADATA_COLUMN: {"type": "object"},
|
||||
DOCUMENT_CONTENT_COLUMN: {"type": "string"},
|
||||
EMBEDDING_COLUMN: {
|
||||
"type": "array",
|
||||
"items": {"type": "float"},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def _add_missing_columns_to_table(
|
||||
self,
|
||||
stream_name: str,
|
||||
table_name: str,
|
||||
) -> None:
|
||||
"""Add missing columns to the table.
|
||||
|
||||
This is a no-op because metadata scans do not work with the `VECTOR` data type.
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def embedder(self) -> embedder.Embedder:
|
||||
return embedder.create_from_config(
|
||||
embedding_config=self.embedder_config, # type: ignore [arg-type]
|
||||
processing_config=self.splitter_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def embedding_dimensions(self) -> int:
|
||||
"""Return the number of dimensions for the embeddings."""
|
||||
return self.embedder.embedding_dimensions
|
||||
|
||||
@property
|
||||
def splitter(self) -> DocumentSplitter:
|
||||
return DocumentSplitter(
|
||||
config=self.splitter_config,
|
||||
catalog=self.catalog_provider.configured_catalog,
|
||||
)
|
||||
|
||||
def _create_document_id(self, record_msg: AirbyteRecordMessage) -> str:
|
||||
"""Create document id based on the primary key values."""
|
||||
stream_name = record_msg.stream
|
||||
primary_key = self._get_record_primary_key(record_msg=record_msg)
|
||||
if primary_key is not None:
|
||||
return f"Stream_{stream_name}_Key_{primary_key}"
|
||||
return str(uuid.uuid4().int)
|
||||
|
||||
def _get_record_primary_key(self, record_msg: AirbyteRecordMessage) -> Union[str, None]:
|
||||
"""Create primary key for the record by appending the primary keys."""
|
||||
stream_name = record_msg.stream
|
||||
primary_keys = self._get_primary_keys(stream_name)
|
||||
|
||||
if not primary_keys:
|
||||
return None
|
||||
|
||||
primary_key = []
|
||||
for key in primary_keys:
|
||||
try:
|
||||
primary_key.append(str(dpath.get(record_msg.data, key)))
|
||||
except KeyError:
|
||||
primary_key.append("__not_found__")
|
||||
# return a stringified version of all primary keys
|
||||
stringified_primary_key = "_".join(primary_key)
|
||||
return stringified_primary_key
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "example_stream_3",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": false,
|
||||
"default_cursor_field": ["column_name"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "example_stream_3",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": false,
|
||||
"default_cursor_field": ["column_name"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "append_dedup"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "example_stream_3",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": false,
|
||||
"default_cursor_field": ["column_name"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "append"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "this_is_a_very_long_stream_name_that_is_designed_to_exceed_the_sixty_three_character_limit_for_postgres",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": false,
|
||||
"default_cursor_field": ["column_name"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "append"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
{"type": "RECORD", "record": {"stream": "example_stream_3", "data": { "title": "value1", "field2": "value2", "str_col":"text1", "int_col":1 }, "emitted_at": 1625383200000}}
|
||||
{"type": "RECORD", "record": {"stream": "example_stream_3", "data": { "title": "value2", "field2": "value2", "str_col":"text2", "int_col":2 }, "emitted_at": 1625383200000}}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{"type": "RECORD", "record": {"stream": "example_stream_3", "data": { "title": "value1", "field2": "value2", "str_col":"text1", "int_col":1 }, "emitted_at": 1625383200000}}
|
||||
{"type": "STATE", "state": {"type": "STREAM", "stream": {"stream_descriptor": {"name": "example_stream_3", "namespace": null}, "stream_state": {"__ab_no_cursor_state_message": true}}, "global_": null, "data": null, "sourceStats": {"recordCount": 1.0, "rejectedRecordCount": null}, "destinationStats": null, "id": 2}}
|
||||
{"type": "RECORD", "record": {"stream": "example_stream_3", "data": { "title": "value2", "field2": "value2", "str_col":"text2", "int_col":2 }, "emitted_at": 1625383200000}}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{"type": "RECORD", "record": {"stream": "this_is_a_very_long_stream_name_that_is_designed_to_exceed_the_sixty_three_character_limit_for_postgres", "data": { "title": "value1", "field2": "value2", "str_col":"text1", "int_col":1 }, "emitted_at": 1625383200000}}
|
||||
{"type": "STATE", "state": {"type": "STREAM", "stream": {"stream_descriptor": {"name": "this_is_a_very_long_stream_name_that_is_designed_to_exceed_the_sixty_three_character_limit_for_postgres", "namespace": null}, "stream_state": {"__ab_no_cursor_state_message": true}}, "global_": null, "data": null, "sourceStats": {"recordCount": 1.0, "rejectedRecordCount": null}, "destinationStats": null, "id": 2}}
|
||||
{"type": "RECORD", "record": {"stream": "this_is_a_very_long_stream_name_that_is_designed_to_exceed_the_sixty_three_character_limit_for_postgres", "data": { "title": "value2", "field2": "value2", "str_col":"text2", "int_col":2 }, "emitted_at": 1625383200000}}
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 5.4 KiB |
|
|
@ -1,363 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from airbyte_cdk.models.airbyte_protocol import AirbyteStateType
|
||||
import psycopg2
|
||||
from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest
|
||||
from airbyte_cdk.models import DestinationSyncMode, Status
|
||||
|
||||
from destination_opengauss_datavec.destination import DestinationOpenGaussDataVec
|
||||
|
||||
|
||||
class OpenGaussDataVecIntegrationTest(BaseIntegrationTest):
|
||||
def setUp(self):
|
||||
with open("secrets/config.json", "r") as f:
|
||||
self.config = json.loads(f.read())
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_check_valid_config(self):
|
||||
outcome = DestinationOpenGaussDataVec().check(logging.getLogger("airbyte"), self.config)
|
||||
assert outcome.status == Status.SUCCEEDED
|
||||
|
||||
def test_check_invalid_config(self):
|
||||
outcome = DestinationOpenGaussDataVec().check(
|
||||
logging.getLogger("airbyte"),
|
||||
{
|
||||
"processing": {
|
||||
"text_fields": ["str_col"],
|
||||
"chunk_size": 1000,
|
||||
"metadata_fields": ["int_col"],
|
||||
},
|
||||
"embedding": {"mode": "openai", "openai_key": "mykey"},
|
||||
"indexing": {
|
||||
"host": "invalid_host",
|
||||
"port": 9999,
|
||||
"database": "invalid_database",
|
||||
"default_schema": "invalid_schema",
|
||||
"username": "invalid_user",
|
||||
"password": "invalid_password",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert outcome.status == Status.FAILED
|
||||
|
||||
def _get_db_connection(self):
|
||||
return psycopg2.connect(
|
||||
dbname=self.config["indexing"]["database"],
|
||||
user=self.config["indexing"]["username"],
|
||||
password=self.config["indexing"]["password"],
|
||||
host=self.config["indexing"]["host"],
|
||||
port=self.config["indexing"]["port"],
|
||||
)
|
||||
|
||||
def _get_record_count(self, table_name):
|
||||
"""Return the number of records in the table."""
|
||||
conn = self._get_db_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table_name};")
|
||||
result = cursor.fetchone()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return result[0]
|
||||
|
||||
def _get_all_records(self, table_name) -> list[dict[str, Any]]:
|
||||
"""Return all records from the table as a list of dictionaries."""
|
||||
conn = self._get_db_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"SELECT * FROM {table_name};")
|
||||
column_names = [desc[0] for desc in cursor.description]
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in cursor.fetchall():
|
||||
result.append(dict(zip(column_names, row)))
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
def _delete_table(self, table_name):
|
||||
conn = self._get_db_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"DROP TABLE IF EXISTS {table_name};")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def _run_cosine_similarity(self, query_vector, table_name):
|
||||
conn = self._get_db_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = f"""
|
||||
SELECT DOCUMENT_CONTENT
|
||||
FROM {table_name}
|
||||
ORDER BY embedding <-> CAST({query_vector} AS VECTOR(1536))
|
||||
LIMIT 1
|
||||
"""
|
||||
cursor.execute(query)
|
||||
result = cursor.fetchone()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
def test_write(self):
|
||||
self._delete_table("mystream")
|
||||
catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
first_state_message = self._state({"state": "1"})
|
||||
first_state_message.state.type = AirbyteStateType.GLOBAL
|
||||
first_record = [
|
||||
self._record(
|
||||
stream="mystream",
|
||||
str_value=f"Dogs are number {i}",
|
||||
int_value=i,
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
# initial sync with replace
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
_ = list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=catalog,
|
||||
input_messages=[*first_record, first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 5
|
||||
|
||||
# subsequent sync with append
|
||||
append_catalog = self._get_configured_catalog(DestinationSyncMode.append)
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=append_catalog,
|
||||
input_messages=[self._record("mystream", "Cats are nice", 6), first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 6
|
||||
|
||||
def test_write_and_replace(self):
|
||||
self._delete_table("mystream")
|
||||
catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
first_state_message = self._state({"state": "1"})
|
||||
first_state_message.state.type = AirbyteStateType.GLOBAL
|
||||
first_five_records = [
|
||||
self._record(
|
||||
stream="mystream",
|
||||
str_value=f"Dogs are number {i}",
|
||||
int_value=i,
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
# initial sync with replace
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=catalog,
|
||||
input_messages=[*first_five_records, first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 5
|
||||
|
||||
# subsequent sync with append
|
||||
append_catalog = self._get_configured_catalog(DestinationSyncMode.append)
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=append_catalog,
|
||||
input_messages=[self._record("mystream", "Cats are nice", 6), first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 6
|
||||
|
||||
# subsequent sync with append_dedup
|
||||
append_dedup_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup)
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=append_dedup_catalog,
|
||||
input_messages=[
|
||||
self._record("mystream", "Cats are nice too", 4),
|
||||
first_state_message,
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
# TODO: FIXME: This should be 6, but it's 7 because the deduplication is not working
|
||||
assert self._get_record_count("mystream") == 6
|
||||
|
||||
# comment the following so we can use fake for testing
|
||||
# embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"])
|
||||
# result = self._run_cosine_similarity(embeddings.embed_query("feline animals"), "mystream")
|
||||
# assert(len(result) == 1)
|
||||
# result[0] == "str_col: Cats are nice"
|
||||
|
||||
def test_overwrite_mode_deletes_records(self):
|
||||
self._delete_table("mystream")
|
||||
catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
first_state_message = self._state({"state": "1"})
|
||||
first_state_message.state.type = AirbyteStateType.GLOBAL
|
||||
first_four_records = [
|
||||
self._record(
|
||||
stream="mystream",
|
||||
str_value=f"Dogs are number {i}",
|
||||
int_value=i,
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
|
||||
# initial sync with replace
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
list(destination.write(self.config, catalog, [*first_four_records, first_state_message]))
|
||||
assert self._get_record_count("mystream") == 4
|
||||
|
||||
# following should replace existing records
|
||||
append_catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=append_catalog,
|
||||
input_messages=[self._record("mystream", "Cats are nice", 6), first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 1
|
||||
|
||||
def test_record_write_fidelity(self):
|
||||
self._delete_table("mystream")
|
||||
catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
first_state_message = self._state({"state": "1"})
|
||||
first_state_message.state.type = AirbyteStateType.GLOBAL
|
||||
records = [
|
||||
self._record(
|
||||
stream="mystream",
|
||||
str_value=f"Dogs are number {i}",
|
||||
int_value=i,
|
||||
)
|
||||
for i in range(1)
|
||||
]
|
||||
|
||||
# initial sync with replace
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
list(destination.write(self.config, catalog, [*records, first_state_message]))
|
||||
assert self._get_record_count("mystream") == 1
|
||||
first_written_record = self._get_all_records("mystream")[0]
|
||||
assert list(first_written_record.keys()) == [
|
||||
"document_id",
|
||||
"chunk_id",
|
||||
"metadata",
|
||||
"document_content",
|
||||
"embedding",
|
||||
]
|
||||
assert first_written_record.pop("embedding")
|
||||
assert first_written_record.pop("chunk_id")
|
||||
metadata = first_written_record.pop("metadata")
|
||||
_ = metadata
|
||||
|
||||
# TODO: Fix the data type issue here (currently stringified):
|
||||
# assert isinstance(metadata, dict), f"METADATA should be a dict: {metadata}"
|
||||
# assert metadata["int_col"] == 0
|
||||
|
||||
assert first_written_record == {
|
||||
"document_id": "Stream_mystream_Key_0",
|
||||
"document_content": "str_col: Dogs are number 0",
|
||||
}
|
||||
|
||||
def test_write_with_chunk_size_5(self):
|
||||
self._delete_table("mystream")
|
||||
self.config["processing"]["chunk_size"] = 5
|
||||
catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
first_state_message = self._state({"state": "1"})
|
||||
first_state_message.state.type = AirbyteStateType.GLOBAL
|
||||
first_record = [
|
||||
self._record(
|
||||
stream="mystream",
|
||||
str_value=f"Dogs are number {i}",
|
||||
int_value=i,
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
# initial sync with replace
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
_ = list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=catalog,
|
||||
input_messages=[*first_record, first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 15
|
||||
|
||||
# subsequent sync with append
|
||||
append_catalog = self._get_configured_catalog(DestinationSyncMode.append)
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=append_catalog,
|
||||
input_messages=[self._record("mystream", "Cats are nice", 6), first_state_message],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 18
|
||||
|
||||
# subsequent sync with append_dedup
|
||||
append_dedup_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup)
|
||||
list(
|
||||
destination.write(
|
||||
config=self.config,
|
||||
configured_catalog=append_dedup_catalog,
|
||||
input_messages=[
|
||||
self._record("mystream", "Cats are nice too", 4),
|
||||
first_state_message,
|
||||
],
|
||||
)
|
||||
)
|
||||
assert self._get_record_count("mystream") == 18
|
||||
|
||||
def test_write_fidelity_with_chunk_size_5(self):
|
||||
self._delete_table("mystream")
|
||||
self.config["processing"]["chunk_size"] = 5
|
||||
catalog = self._get_configured_catalog(DestinationSyncMode.overwrite)
|
||||
first_state_message = self._state({"state": "1"})
|
||||
first_state_message.state.type = AirbyteStateType.GLOBAL
|
||||
records = [
|
||||
self._record(
|
||||
stream="mystream",
|
||||
str_value=f"Dogs are number {i}",
|
||||
int_value=i,
|
||||
)
|
||||
for i in range(1)
|
||||
]
|
||||
|
||||
# initial sync with replace
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
list(destination.write(self.config, catalog, [*records, first_state_message]))
|
||||
assert self._get_record_count("mystream") == 3
|
||||
first_written_record = self._get_all_records("mystream")[0]
|
||||
second_written_record = self._get_all_records("mystream")[1]
|
||||
third_written_record = self._get_all_records("mystream")[2]
|
||||
assert list(first_written_record.keys()) == [
|
||||
"document_id",
|
||||
"chunk_id",
|
||||
"metadata",
|
||||
"document_content",
|
||||
"embedding",
|
||||
]
|
||||
assert first_written_record.pop("embedding")
|
||||
assert first_written_record.pop("chunk_id")
|
||||
metadata = first_written_record.pop("metadata")
|
||||
_ = metadata
|
||||
|
||||
assert first_written_record == {
|
||||
"document_id": "Stream_mystream_Key_0",
|
||||
"document_content": "str_col:",
|
||||
}
|
||||
assert second_written_record["document_id"] == "Stream_mystream_Key_0"
|
||||
assert second_written_record["document_content"] == "Dogs are"
|
||||
assert third_written_record["document_id"] == "Stream_mystream_Key_0"
|
||||
assert third_written_record["document_content"] == "number 0"
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
{
|
||||
"processing": {
|
||||
"text_fields": ["str_col"],
|
||||
"metadata_fields": ["int_col"],
|
||||
"chunk_size": 1000
|
||||
},
|
||||
"embedding": {
|
||||
"mode": "openai",
|
||||
"openai_key": "mykey"
|
||||
},
|
||||
"indexing": {
|
||||
"host": "MYACCOUNT",
|
||||
"port": 5432,
|
||||
"database": "MYDATABASE",
|
||||
"default_schema": "MYSCHEMA",
|
||||
"username": "MYUSERNAME",
|
||||
"password": "xxxxxxx"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,389 +0,0 @@
|
|||
{
|
||||
"documentationUrl": "https://docs.airbyte.com/integrations/destinations/opengauss-datavec",
|
||||
"connectionSpecification": {
|
||||
"title": "Destination Config",
|
||||
"description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"embedding": {
|
||||
"title": "Embedding",
|
||||
"description": "Embedding configuration",
|
||||
"group": "embedding",
|
||||
"type": "object",
|
||||
"oneOf": [
|
||||
{
|
||||
"title": "OpenAI",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "openai",
|
||||
"const": "openai",
|
||||
"enum": ["openai"],
|
||||
"type": "string"
|
||||
},
|
||||
"openai_key": {
|
||||
"title": "OpenAI API key",
|
||||
"airbyte_secret": true,
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["openai_key", "mode"],
|
||||
"description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions."
|
||||
},
|
||||
{
|
||||
"title": "Cohere",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "cohere",
|
||||
"const": "cohere",
|
||||
"enum": ["cohere"],
|
||||
"type": "string"
|
||||
},
|
||||
"cohere_key": {
|
||||
"title": "Cohere API key",
|
||||
"airbyte_secret": true,
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["cohere_key", "mode"],
|
||||
"description": "Use the Cohere API to embed text."
|
||||
},
|
||||
{
|
||||
"title": "Fake",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "fake",
|
||||
"const": "fake",
|
||||
"enum": ["fake"],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.",
|
||||
"required": ["mode"]
|
||||
},
|
||||
{
|
||||
"title": "Azure OpenAI",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "azure_openai",
|
||||
"const": "azure_openai",
|
||||
"enum": ["azure_openai"],
|
||||
"type": "string"
|
||||
},
|
||||
"openai_key": {
|
||||
"title": "Azure OpenAI API key",
|
||||
"description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource",
|
||||
"airbyte_secret": true,
|
||||
"type": "string"
|
||||
},
|
||||
"api_base": {
|
||||
"title": "Resource base URL",
|
||||
"description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource",
|
||||
"examples": ["https://your-resource-name.openai.azure.com"],
|
||||
"type": "string"
|
||||
},
|
||||
"deployment": {
|
||||
"title": "Deployment",
|
||||
"description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource",
|
||||
"examples": ["your-resource-name"],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["openai_key", "api_base", "deployment", "mode"],
|
||||
"description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions."
|
||||
},
|
||||
{
|
||||
"title": "OpenAI-compatible",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "openai_compatible",
|
||||
"const": "openai_compatible",
|
||||
"enum": ["openai_compatible"],
|
||||
"type": "string"
|
||||
},
|
||||
"api_key": {
|
||||
"title": "API key",
|
||||
"default": "",
|
||||
"airbyte_secret": true,
|
||||
"type": "string"
|
||||
},
|
||||
"base_url": {
|
||||
"title": "Base URL",
|
||||
"description": "The base URL for your OpenAI-compatible service",
|
||||
"examples": ["https://your-service-name.com"],
|
||||
"type": "string"
|
||||
},
|
||||
"model_name": {
|
||||
"title": "Model name",
|
||||
"description": "The name of the model to use for embedding",
|
||||
"default": "text-embedding-ada-002",
|
||||
"examples": ["text-embedding-ada-002"],
|
||||
"type": "string"
|
||||
},
|
||||
"dimensions": {
|
||||
"title": "Embedding dimensions",
|
||||
"description": "The number of dimensions the embedding model is generating",
|
||||
"examples": [1536, 384],
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": ["base_url", "dimensions", "mode"],
|
||||
"description": "Use a service that's compatible with the OpenAI API to embed text."
|
||||
}
|
||||
]
|
||||
},
|
||||
"processing": {
|
||||
"title": "ProcessingConfigModel",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"chunk_size": {
|
||||
"title": "Chunk size",
|
||||
"description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)",
|
||||
"maximum": 8191,
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
"chunk_overlap": {
|
||||
"title": "Chunk overlap",
|
||||
"description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context",
|
||||
"default": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
"text_fields": {
|
||||
"title": "Text fields to embed",
|
||||
"description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.",
|
||||
"default": [],
|
||||
"always_show": true,
|
||||
"examples": ["text", "user.name", "users.*.name"],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"metadata_fields": {
|
||||
"title": "Fields to store as metadata",
|
||||
"description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.",
|
||||
"default": [],
|
||||
"always_show": true,
|
||||
"examples": ["age", "user", "user.name"],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"text_splitter": {
|
||||
"title": "Text splitter",
|
||||
"description": "Split text fields into chunks based on the specified method.",
|
||||
"type": "object",
|
||||
"oneOf": [
|
||||
{
|
||||
"title": "By Separator",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "separator",
|
||||
"const": "separator",
|
||||
"enum": ["separator"],
|
||||
"type": "string"
|
||||
},
|
||||
"separators": {
|
||||
"title": "Separators",
|
||||
"description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".",
|
||||
"default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"keep_separator": {
|
||||
"title": "Keep separator",
|
||||
"description": "Whether to keep the separator in the resulting chunks",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.",
|
||||
"required": ["mode"]
|
||||
},
|
||||
{
|
||||
"title": "By Markdown header",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "markdown",
|
||||
"const": "markdown",
|
||||
"enum": ["markdown"],
|
||||
"type": "string"
|
||||
},
|
||||
"split_level": {
|
||||
"title": "Split level",
|
||||
"description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": 6,
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.",
|
||||
"required": ["mode"]
|
||||
},
|
||||
{
|
||||
"title": "By Programming Language",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"title": "Mode",
|
||||
"default": "code",
|
||||
"const": "code",
|
||||
"enum": ["code"],
|
||||
"type": "string"
|
||||
},
|
||||
"language": {
|
||||
"title": "Language",
|
||||
"description": "Split code in suitable places based on the programming language",
|
||||
"enum": [
|
||||
"cpp",
|
||||
"go",
|
||||
"java",
|
||||
"js",
|
||||
"php",
|
||||
"proto",
|
||||
"python",
|
||||
"rst",
|
||||
"ruby",
|
||||
"rust",
|
||||
"scala",
|
||||
"swift",
|
||||
"markdown",
|
||||
"latex",
|
||||
"html",
|
||||
"sol"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["language", "mode"],
|
||||
"description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks."
|
||||
}
|
||||
]
|
||||
},
|
||||
"field_name_mappings": {
|
||||
"title": "Field name mappings",
|
||||
"description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.",
|
||||
"default": [],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"title": "FieldNameMappingConfigModel",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from_field": {
|
||||
"title": "From field name",
|
||||
"description": "The field name in the source",
|
||||
"type": "string"
|
||||
},
|
||||
"to_field": {
|
||||
"title": "To field name",
|
||||
"description": "The field name to use in the destination",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["from_field", "to_field"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["chunk_size"],
|
||||
"group": "processing"
|
||||
},
|
||||
"omit_raw_text": {
|
||||
"title": "Do not store raw text",
|
||||
"description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.",
|
||||
"default": false,
|
||||
"group": "advanced",
|
||||
"type": "boolean"
|
||||
},
|
||||
"indexing": {
|
||||
"title": "Indexing",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {
|
||||
"title": "Host",
|
||||
"description": "Hostname of the database.",
|
||||
"order": 1,
|
||||
"type": "string"
|
||||
},
|
||||
"database": {
|
||||
"title": "Database",
|
||||
"description": "Enter the name of the database that you want to sync data into",
|
||||
"order": 2,
|
||||
"examples": ["AIRBYTE_DATABASE", "postgres"],
|
||||
"type": "string"
|
||||
},
|
||||
"username": {
|
||||
"title": "Username",
|
||||
"description": "Enter the name of the user you want to use to access the database",
|
||||
"order": 3,
|
||||
"examples": ["root"],
|
||||
"type": "string"
|
||||
},
|
||||
"password": {
|
||||
"title": "Password",
|
||||
"description": "Enter the password of the user you want to use to access the database",
|
||||
"order": 4,
|
||||
"airbyte_secret": true,
|
||||
"type": "string"
|
||||
},
|
||||
"port": {
|
||||
"title": "Port",
|
||||
"description": "Enter the port you want to use to access the database",
|
||||
"default": 5432,
|
||||
"examples": ["5432"],
|
||||
"type": "integer"
|
||||
},
|
||||
"default_schema": {
|
||||
"title": "Default Schema",
|
||||
"description": "Enter the name of the default schema",
|
||||
"default": "public",
|
||||
"order": 6,
|
||||
"examples": ["AIRBYTE_SCHEMA"],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["host", "database", "username", "password"],
|
||||
"description": "Indexing configuration",
|
||||
"group": "indexing"
|
||||
}
|
||||
},
|
||||
"required": ["embedding", "processing", "indexing"],
|
||||
"groups": [
|
||||
{
|
||||
"id": "processing",
|
||||
"title": "Processing"
|
||||
},
|
||||
{
|
||||
"id": "embedding",
|
||||
"title": "Embedding"
|
||||
},
|
||||
{
|
||||
"id": "indexing",
|
||||
"title": "Indexing"
|
||||
},
|
||||
{
|
||||
"id": "advanced",
|
||||
"title": "Advanced"
|
||||
}
|
||||
]
|
||||
},
|
||||
"supportsIncremental": true,
|
||||
"supported_destination_sync_modes": ["overwrite", "append", "append_dedup"]
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import sys
|
||||
|
||||
from destination_opengauss_datavec import DestinationOpenGaussDataVec
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
DestinationOpenGaussDataVec().run(sys.argv[1:])
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
data:
|
||||
ab_internal:
|
||||
ql: 300
|
||||
sl: 200
|
||||
allowedHosts:
|
||||
hosts:
|
||||
- api.openai.com
|
||||
- api.cohere.ai
|
||||
- ${embedding.api_base}
|
||||
connectorBuildOptions:
|
||||
baseImage: docker.io/airbyte/python-connector-base:4.0.0@sha256:d9894b6895923b379f3006fa251147806919c62b7d9021b5cd125bb67d7bbe22
|
||||
connectorSubtype: vectorstore
|
||||
connectorType: destination
|
||||
definitionId: 3ea1b85a-564c-46f9-9eae-632b845fcebe
|
||||
dockerImageTag: dev
|
||||
dockerRepository: airbyte/destination-opengauss-datavec
|
||||
documentationUrl: https://docs.airbyte.com/integrations/destinations/opengauss-datavec
|
||||
githubIssueLabel: destination-opengauss-datavec
|
||||
icon: icon.svg
|
||||
license: MIT
|
||||
name: openGauss Datavec
|
||||
# 必须有，否则不能通过qa_checks
|
||||
remoteRegistries:
|
||||
pypi:
|
||||
enabled: true
|
||||
packageName: airbyte-destination-opengauss-datavec
|
||||
registryOverrides:
|
||||
cloud:
|
||||
enabled: false
|
||||
oss:
|
||||
enabled: true
|
||||
releaseDate: 2025-08-30
|
||||
releaseStage: alpha
|
||||
resourceRequirements:
|
||||
jobSpecific:
|
||||
- jobType: sync
|
||||
resourceRequirements:
|
||||
memory_limit: 2Gi
|
||||
memory_request: 2Gi
|
||||
supportLevel: community
|
||||
tags:
|
||||
- language:python
|
||||
- cdk:python
|
||||
connectorTestSuitesOptions:
|
||||
# used for Airbyte's official CI/CD processes during pr and airbyte-ci.
|
||||
- suite: unitTests
|
||||
- suite: integrationTests
|
||||
testSecrets:
|
||||
- name: SECRET_DESTINATION-OPENGAUSS-DATAVEC_CREDS
|
||||
fileName: config.json
|
||||
secretStore:
|
||||
type: GSM
|
||||
alias: airbyte-connector-testing-secret-store
|
||||
- suite: acceptanceTests
|
||||
testSecrets:
|
||||
- name: SECRET_DESTINATION-OPENGAUSS-DATAVEC_CREDS
|
||||
fileName: config.json
|
||||
secretStore:
|
||||
type: GSM
|
||||
alias: airbyte-connector-testing-secret-store
|
||||
metadataSpecVersion: "1.0"
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,80 +0,0 @@
|
|||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
name = "airbyte-destination-opengauss-datavec"
|
||||
version = "0.1.0"
|
||||
description = "Airbyte destination implementation for OpenGauss with vector support"
|
||||
authors = ["hly <liyanhu03@163.com>"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
|
||||
[[tool.poetry.packages]]
|
||||
include = "destination_opengauss_datavec"
|
||||
|
||||
[tool.poetry.dependencies.airbyte-cdk]
|
||||
# Version constrained by PyAirbyte (`airbyte`) version
|
||||
version = ">=1.0"
|
||||
extras = ["vector-db-based"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.10,<3.13"
|
||||
|
||||
airbyte = "^0.12.0"
|
||||
sqlalchemy = "<2.0"
|
||||
opengauss-sqlalchemy = { git = "https://gitcode.com/opengauss/openGauss-sqlalchemy.git", rev = "da80fc05ebc105968bdebfd4d0614776ad83d5fe" }
|
||||
psycopg2-binary = "2.9.9"
|
||||
jaraco-functools = "^4.1.0"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.0.0"
|
||||
ruff = "^0.3.2"
|
||||
mypy = "^1.9.0"
|
||||
coverage = "^7.5.3"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
destination-opengauss-datavec = "destination_opengauss_datavec.run:run"
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py310"
|
||||
preview = true
|
||||
line-length = 100
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"airbyte_protocol",
|
||||
"airbyte_protocol.*",
|
||||
"sqlalchemy",
|
||||
"sqlalchemy.*",
|
||||
"opengauss_sqlalchemy",
|
||||
"opengauss_sqlalchemy.*",
|
||||
"dpath",
|
||||
"dpath.util",
|
||||
]
|
||||
ignore_missing_imports = true # No stubs yet (😢)
|
||||
|
||||
[tool.coverage.run]
|
||||
# branch = true
|
||||
# parallel = true
|
||||
omit = [
|
||||
"unit_tests/*",
|
||||
"integration_tests/*",
|
||||
"destination_opengauss_datavec/common/*",
|
||||
]
|
||||
|
||||
[tool.coverage.report]
|
||||
precision = 1
|
||||
fail_under = 90.0
|
||||
exclude_also = [
|
||||
# # TYPE_CHECKING block is only executed while running mypy
|
||||
"if TYPE_CHECKING:",
|
||||
"\\.\\.\\."
|
||||
]
|
||||
|
||||
[tool.poe]
|
||||
include = [
|
||||
# Shared tasks definition file(s) can be imported here.
|
||||
# Run `poe` or `poe --help` to see the list of available tasks.
|
||||
"${POE_GIT_DIR}/poe-tasks/poetry-connector-tasks.toml",
|
||||
]
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
import logging
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from airbyte.strategies import WriteStrategy
|
||||
from airbyte_cdk.models import ConnectorSpecification, Status
|
||||
|
||||
from destination_opengauss_datavec.config import ConfigModel
|
||||
from destination_opengauss_datavec.destination import DestinationOpenGaussDataVec
|
||||
|
||||
|
||||
class TestDestinationOpenGaussDataVec(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.config = {
|
||||
"processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000},
|
||||
"embedding": {"mode": "fake"},
|
||||
"indexing": {
|
||||
"host": "localhost",
|
||||
"database": "postgres",
|
||||
"username": "hly",
|
||||
"password": "Hly12345",
|
||||
"port": 8888,
|
||||
"schema": "public",
|
||||
"default_schema": "public",
|
||||
},
|
||||
}
|
||||
self.config_model = ConfigModel.parse_obj(self.config)
|
||||
self.logger = logging.getLogger("airbyte")
|
||||
|
||||
def test_spec(self):
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
result = destination.spec()
|
||||
|
||||
self.assertIsInstance(result, ConnectorSpecification)
|
||||
|
||||
@patch("destination_opengauss_datavec.opengauss_processor.OpenGaussDataVecProcessor")
|
||||
def test_check(self, MockedOpenGaussDataVecProcessor):
|
||||
mock_processor = Mock()
|
||||
MockedOpenGaussDataVecProcessor.return_value = mock_processor
|
||||
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
result = destination.check(self.logger, self.config)
|
||||
|
||||
self.assertEqual(result.status, Status.SUCCEEDED)
|
||||
mock_processor.sql_config.get_sql_engine().connect.assert_called_once()
|
||||
|
||||
@patch("destination_opengauss_datavec.opengauss_processor.OpenGaussDataVecProcessor")
|
||||
def test_check_with_errors(self, MockedOpenGaussDataVecProcessor):
|
||||
mock_processor = Mock()
|
||||
MockedOpenGaussDataVecProcessor.return_value = mock_processor
|
||||
|
||||
indexer_error_message = "Indexer Error"
|
||||
mock_processor.sql_config.get_sql_engine().connect.side_effect = Exception(indexer_error_message)
|
||||
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
result = destination.check(self.logger, self.config)
|
||||
self.assertEqual(result.status, Status.FAILED)
|
||||
mock_processor.sql_config.get_sql_engine().connect.assert_called_once()
|
||||
|
||||
@patch("destination_opengauss_datavec.opengauss_processor.OpenGaussDataVecProcessor")
|
||||
def test_write(
|
||||
self,
|
||||
MockedOpenGaussDataVecProcessor,
|
||||
):
|
||||
mock_processor = Mock()
|
||||
MockedOpenGaussDataVecProcessor.return_value = mock_processor
|
||||
mock_processor.process_airbyte_messages_as_generator.return_value = []
|
||||
|
||||
configured_catalog = MagicMock()
|
||||
input_messages = []
|
||||
|
||||
destination = DestinationOpenGaussDataVec()
|
||||
list(destination.write(self.config, configured_catalog, input_messages))
|
||||
|
||||
mock_processor.process_airbyte_messages_as_generator.assert_called_once_with(
|
||||
messages=input_messages,
|
||||
write_strategy=WriteStrategy.AUTO,
|
||||
)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 177 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 177 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 149 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 149 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 326 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 131 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 98 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 113 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 132 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 213 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 131 KiB |
|
|
@ -1,221 +0,0 @@
|
|||
# OpenGauss DataVec Destination
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](openGauss_DataVec_Connector_Document.md)
|
||||
[](openGauss_DataVec_Connector_Document.zh.md)
|
||||
|
||||
**English** | [中文](openGauss_DataVec_Connector_Document.zh.md)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Step 1: Set up OpenGauss](#step-1-set-up-opengauss)
|
||||
- [Step 2: Set up the OpenGauss DataVec connector in Airbyte](#step-2-set-up-the-opengauss-datavec-connector-in-airbyte)
|
||||
- [Naming Conventions](#naming-conventions)
|
||||
- [Features](#features)
|
||||
- [Data type mapping](#data-type-mapping)
|
||||
- [Configuration](#configuration)
|
||||
- [Changelog](#changelog)
|
||||
|
||||
## Overview
|
||||
|
||||
This page guides you through the process of setting up the OpenGauss DataVec destination connector.
|
||||
|
||||
There are three parts to this:
|
||||
* Processing - split up individual records in chunks so they will fit the context window and decide which fields to use as context and which are supplementary metadata.
|
||||
* Embedding - convert the text into a vector representation using a pre-trained model. Currently supported:
|
||||
* OpenAI's `text-embedding-ada-002`
|
||||
* Cohere's `embed-english-light-v2.0`
|
||||
* Azure OpenAI
|
||||
* Fake `random vectors with 1536 embedding dimensions`
|
||||
* OpenAI-compatible
|
||||
* Coming soon: Hugging Face's `e5-base-v2`.
|
||||
* OpenGauss Connection - where to store the vectors. This configures a vector store using OpenGauss tables having the `VECTOR` data type which is achieved by installing the DataVec extension.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
To use the OpenGauss DataVec destination, you'll need:
|
||||
|
||||
- An account with API access depending on which embedding method you want to use.
|
||||
- An openGauss database with vector engine support. See the [OpenGauss DataVec documentation](https://docs.opengauss.org/zh/docs/latest/docs/DataVec/DataVec-Overview.html) for installation instructions.
|
||||
|
||||
You'll need the following information to configure the destination:
|
||||
|
||||
- **Embedding service API Key** - The API key for your embedding account and other params depending on your model.
|
||||
- **Port** - The port number the server is listening on. Defaults to the standard port
|
||||
number (5432).
|
||||
- **Username**
|
||||
- **Password**
|
||||
- **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in
|
||||
the search-path. These schemas will be used to resolve unqualified object names used in statements
|
||||
executed over this connection.
|
||||
- **Database** - The database name. The default is to connect to a database with the same name as
|
||||
the user name.
|
||||
|
||||
### Configure Network Access
|
||||
|
||||
Make sure your OpenGauss database can be accessed by Airbyte. If your database is within a VPC, you
|
||||
may need to allow access from the IP you're using to expose Airbyte.
|
||||
|
||||
## Step 1: Set up OpenGauss
|
||||
|
||||
### **Permissions**
|
||||
|
||||
You need an OpenGauss user with the following permissions:
|
||||
|
||||
- can create tables and write rows.
|
||||
- can create schemas e.g:
|
||||
|
||||
You can create such a user by running:
|
||||
|
||||
```sql
|
||||
CREATE USER airbyte_user WITH PASSWORD '<password>';
|
||||
GRANT CREATE, TEMPORARY ON DATABASE <database> TO airbyte_user;
|
||||
```
|
||||
|
||||
You can also use a pre-existing user but we highly recommend creating a dedicated user for Airbyte.
|
||||
|
||||
## Step 2: Set up the OpenGauss DataVec connector in Airbyte
|
||||
|
||||
### Target Database
|
||||
|
||||
You will need to choose an existing database or create a new database that will be used to store
|
||||
synced data from Airbyte.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
From
|
||||
[Postgres SQL Identifiers syntax](https://www.postgresql.org/docs/9.0/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS):
|
||||
|
||||
- SQL identifiers and key words must begin with a letter \(a-z, but also letters with diacritical
|
||||
marks and non-Latin letters\) or an underscore \(\_\).
|
||||
- Subsequent characters in an identifier or key word can be letters, underscores, digits \(0-9\), or
|
||||
dollar signs \($\).
|
||||
|
||||
Note that dollar signs are not allowed in identifiers according to the SQL standard, so their use
|
||||
might render applications less portable. The SQL standard will not define a key word that contains
|
||||
digits or starts or ends with an underscore, so identifiers of this form are safe against possible
|
||||
conflict with future extensions of the standard.
|
||||
|
||||
- The system uses no more than NAMEDATALEN-1 bytes of an identifier; longer names can be written in
|
||||
commands, but they will be truncated. By default, NAMEDATALEN is 64 so the maximum identifier
|
||||
length is 63 bytes
|
||||
- Quoted identifiers can contain any character, except the character with code zero. \(To include a
|
||||
double quote, write two double quotes.\) This allows constructing table or column names that would
|
||||
otherwise not be possible, such as ones containing spaces or ampersands. The length limitation
|
||||
still applies.
|
||||
- Quoting an identifier also makes it case-sensitive, whereas unquoted names are always folded to
|
||||
lower case.
|
||||
- In order to make your applications portable and less error-prone, use consistent quoting with each
|
||||
name (either always quote it or never quote it).
|
||||
|
||||
:::info
|
||||
|
||||
Airbyte OpenGauss DataVec destination will create raw tables and schemas using the Unquoted identifiers by
|
||||
replacing any special characters with an underscore. All final tables and their corresponding
|
||||
columns are created using Quoted identifiers preserving the case sensitivity. Special characters in final
|
||||
tables are replaced with underscores.
|
||||
|
||||
:::
|
||||
|
||||
1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
|
||||
2. In the left navigation bar, click **Destinations**. In the top-right corner, click **new
|
||||
destination**.
|
||||
3. On the Set up the destination page, enter the name for the OpenGauss DataVec connector and select
|
||||
**OpenGauss DataVec** from the Destination type dropdown.
|
||||
4. Enter a name for your source.
|
||||
5. Enter processing information.
|
||||
6. Enter embedding information.
|
||||
7. For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your
|
||||
OpenGauss database.
|
||||
8. Enter the **Default Schemas**.
|
||||
|
||||
:::note
|
||||
|
||||
The schema names are case sensitive. The 'public' schema is set by default.
|
||||
|
||||
:::
|
||||
|
||||
7. For **User** and **Password**, enter the username and password you created in
|
||||
[Step 1](#step-1-set-up-opengauss).
|
||||
|
||||
## Features
|
||||
|
||||
| Feature | Supported? | Notes |
|
||||
| :----------------------------- | :------------------- | :---- |
|
||||
| Full Refresh Sync | Yes | |
|
||||
| Incremental - Append Sync | Yes | |
|
||||
| Incremental - Append + Deduped | Yes | |
|
||||
|
||||
## Data type mapping
|
||||
|
||||
All fields specified as metadata fields will be stored in the metadata object of the document and can be used for filtering. The following data types are allowed for metadata fields:
|
||||
* String
|
||||
* Number (integer or floating point, gets converted to a 64 bit floating point)
|
||||
* Booleans (true, false)
|
||||
* List of String
|
||||
|
||||
All other fields are ignored.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Processing
|
||||
|
||||
Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Please note that meta data fields can only be used for filtering and not for retrieval and have to be of type string, number, boolean (all other values are ignored). Please note that there's a 40kb limit on the _total_ size of the metadata saved for each entry. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction).
|
||||
|
||||
When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.
|
||||
|
||||
The chunk length is measured in tokens produced by the `tiktoken` library. The maximum is 8191 tokens, which is the maximum length supported by the `text-embedding-ada-002` model.
|
||||
|
||||
The stream name gets added as a metadata field `_ab_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_ab_record_id` metadata field.
|
||||
|
||||
### Embedding
|
||||
|
||||
The connector can use one of the following embedding methods:
|
||||
|
||||
1. OpenAI - using [OpenAI API](https://beta.openai.com/docs/api-reference/text-embedding) , the connector will produce embeddings using the `text-embedding-ada-002` model with **1536 dimensions**. This integration will be constrained by the [speed of the OpenAI embedding API](https://platform.openai.com/docs/guides/rate-limits/overview).
|
||||
|
||||
2. Cohere - using the [Cohere API](https://docs.cohere.com/reference/embed), the connector will produce embeddings using the `embed-english-light-v2.0` model with **1024 dimensions**.
|
||||
|
||||
For testing purposes, it's also possible to use the [Fake embeddings](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) integration. It will generate random embeddings and is suitable to test a data pipeline without incurring embedding costs.
|
||||
|
||||
### Indexing/Data Storage
|
||||
|
||||
- For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your
|
||||
OpenGauss database.
|
||||
- List the **Default Schemas**.
|
||||
|
||||
All streams will be indexed/stored into a table with the same name. The table will be created if it doesn't exist. The table will have the following columns:
|
||||
- document_id (string) - the unique identifier of the document, creating from appending the primary keys in the stream schema
|
||||
- chunk_id (string) - the unique identifier of the chunk, created by appending the chunk number to the document_id
|
||||
- metadata (variant) - the metadata of the document, stored as key-value pairs
|
||||
- document_content (string) - the text content of the chunk
|
||||
- embedding (vector) - the embedding of the chunk, stored as a list of floats
|
||||
|
||||
|
||||
> Note: If the table name exceeds the maximum identifier limit of 63 bytes, it will be hashed and truncated.
|
||||
|
||||
## Changelog
|
||||
|
||||
<details>
|
||||
<summary>Expand to review</summary>
|
||||
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 0.1.0 | 2025-09-01 | TBD | Add support for openGauss DateVec as a Vector Destination. |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
[🔝 Back to Top](#opengauss-datavec-destination) | [🇨🇳 中文版本](openGauss_DataVec_Connector_Document.zh.md)
|
||||
|
||||
</div>
|
||||
|
|
@ -1,195 +0,0 @@
|
|||
# OpenGauss DataVec 目标连接器
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](openGauss_DataVec_Connector_Document.md)
|
||||
[](openGauss_DataVec_Connector_Document.zh.md)
|
||||
|
||||
[English](openGauss_DataVec_Connector_Document.md) | **中文**
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
- [概述](#概述)
|
||||
- [前提条件](#前提条件)
|
||||
- [步骤 1：设置 OpenGauss](#步骤-1设置-opengauss)
|
||||
- [步骤 2：在 Airbyte 中设置 OpenGauss DataVec 连接器](#步骤-2在-airbyte-中设置-opengauss-datavec-连接器)
|
||||
- [命名约定](#命名约定)
|
||||
- [功能特性](#功能特性)
|
||||
- [数据类型映射](#数据类型映射)
|
||||
- [配置](#配置)
|
||||
- [更新日志](#更新日志)
|
||||
|
||||
## 概述
|
||||
|
||||
本页面指导您完成 OpenGauss DataVec 目标连接器的设置过程。
|
||||
|
||||
此过程包含三个部分：
|
||||
* **处理** - 将单个记录拆分为块，以适应上下文窗口，并决定哪些字段用作上下文，哪些是补充元数据。
|
||||
* **嵌入** - 使用预训练模型将文本转换为向量表示。目前支持：
|
||||
* OpenAI 的 `text-embedding-ada-002`
|
||||
* Cohere 的 `embed-english-light-v2.0`
|
||||
* Azure OpenAI
|
||||
* 虚拟 `1536 维随机向量`
|
||||
* OpenAI 兼容接口
|
||||
* 即将支持：Hugging Face 的 `e5-base-v2`
|
||||
* **OpenGauss 连接** - 存储向量的位置。通过安装 DataVec 扩展，使用具有 `VECTOR` 数据类型的 OpenGauss 表配置向量存储。
|
||||
|
||||
## 前提条件
|
||||
|
||||
要使用 OpenGauss DataVec 目标连接器，您需要：
|
||||
|
||||
- 根据您要使用的嵌入方法拥有 API 访问权限的账户。
|
||||
- 支持向量引擎的 openGauss 数据库。请参阅 [OpenGauss DataVec 文档](https://docs.opengauss.org/zh/docs/latest/docs/DataVec/DataVec-Overview.html) 获取安装说明。
|
||||
|
||||
您需要以下信息来配置目标连接器：
|
||||
|
||||
- **嵌入服务 API 密钥** - 您的嵌入账户的 API 密钥和其他参数，具体取决于您的模型。
|
||||
- **端口** - 服务器正在监听的端口号。默认为标准端口号 (5432)。
|
||||
- **用户名**
|
||||
- **密码**
|
||||
- **默认模式名称** - 指定要在搜索路径中设置的模式（或用逗号分隔的多个模式）。这些模式将用于解析在此连接上执行的语句中使用的非限定对象名称。
|
||||
- **数据库** - 数据库名称。默认是连接到与用户名同名的数据库。
|
||||
|
||||
### 配置网络访问
|
||||
|
||||
确保 Airbyte 可以访问您的 OpenGauss 数据库。如果您的数据库在 VPC 内，您可能需要允许从用于暴露 Airbyte 的 IP 进行访问。
|
||||
|
||||
## 步骤 1：设置 OpenGauss
|
||||
|
||||
### **权限**
|
||||
|
||||
您需要一个具有以下权限的 OpenGauss 用户：
|
||||
|
||||
- 可以创建表和写入行
|
||||
- 可以创建模式，例如：
|
||||
|
||||
您可以通过运行以下命令创建这样的用户：
|
||||
|
||||
```sql
|
||||
CREATE USER airbyte_user WITH PASSWORD '<password>';
|
||||
GRANT CREATE, TEMPORARY ON DATABASE <database> TO airbyte_user;
|
||||
```
|
||||
|
||||
您也可以使用现有用户，但我们强烈建议为 Airbyte 创建专用用户。
|
||||
|
||||
## 步骤 2：在 Airbyte 中设置 OpenGauss DataVec 连接器
|
||||
|
||||
### 目标数据库
|
||||
|
||||
您需要选择现有数据库或创建新数据库，用于存储来自 Airbyte 的同步数据。
|
||||
|
||||
## 命名约定
|
||||
|
||||
来自 [Postgres SQL 标识符语法](https://www.postgresql.org/docs/9.0/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS)：
|
||||
|
||||
- SQL 标识符和关键字必须以字母（a-z，还包括带变音符号的字母和非拉丁字母）或下划线（_）开头。
|
||||
- 标识符或关键字中的后续字符可以是字母、下划线、数字（0-9）或美元符号（$）。
|
||||
|
||||
请注意，根据 SQL 标准，标识符中不允许使用美元符号，因此使用它们可能会降低应用程序的可移植性。SQL 标准不会定义包含数字或以下划线开头或结尾的关键字，因此这种形式的标识符可以安全地避免与标准的未来扩展产生可能的冲突。
|
||||
|
||||
- 系统使用标识符的不超过 NAMEDATALEN-1 字节；可以在命令中写入更长的名称，但它们将被截断。默认情况下，NAMEDATALEN 是 64，所以最大标识符长度是 63 字节。
|
||||
- 引用的标识符可以包含任何字符，除了代码为零的字符。（要包含双引号，请写两个双引号。）这允许构造原本不可能的表或列名，例如包含空格或 & 符号的名称。长度限制仍然适用。
|
||||
- 引用标识符也使其区分大小写，而未引用的名称总是折叠为小写。
|
||||
- 为了使您的应用程序具有可移植性且不易出错，请对每个名称使用一致的引用（要么总是引用它，要么从不引用它）。
|
||||
|
||||
:::info
|
||||
|
||||
Airbyte OpenGauss DataVec 目标连接器将通过用下划线替换任何特殊字符来创建原始表和模式，使用未引用的标识符。所有最终表及其对应的列都使用引用的标识符创建，保持大小写敏感性。最终表中的特殊字符被替换为下划线。
|
||||
|
||||
:::
|
||||
|
||||
1. [登录您的 Airbyte Cloud](https://cloud.airbyte.com/workspaces) 账户。
|
||||
2. 在左侧导航栏中，点击 **目标连接器**。在右上角，点击 **新建目标连接器**。
|
||||
3. 在设置目标连接器页面上，输入 OpenGauss DataVec 连接器的名称，并从目标连接器类型下拉菜单中选择 **OpenGauss DataVec**。
|
||||
4. 输入您的源的名称。
|
||||
5. 输入处理信息。
|
||||
6. 输入嵌入信息。
|
||||
7. 对于 **主机**、**端口** 和 **数据库名称**，输入您的 OpenGauss 数据库的主机名、端口号和名称。
|
||||
8. 输入 **默认模式**。
|
||||
|
||||
:::note
|
||||
|
||||
模式名称区分大小写。默认设置 'public' 模式。
|
||||
|
||||
:::
|
||||
|
||||
7. 对于 **用户** 和 **密码**，输入您在 [步骤 1](#步骤-1设置-opengauss) 中创建的用户名和密码。
|
||||
|
||||
## 功能特性
|
||||
|
||||
| 功能 | 是否支持？ | 备注 |
|
||||
| :----------------------------- | :------------------- | :---- |
|
||||
| 完全刷新同步 | 是 | |
|
||||
| 增量 - 追加同步 | 是 | |
|
||||
| 增量 - 追加 + 去重 | 是 | |
|
||||
|
||||
## 数据类型映射
|
||||
|
||||
所有指定为元数据字段的字段将存储在文档的元数据对象中，可用于过滤。元数据字段允许以下数据类型：
|
||||
* 字符串
|
||||
* 数字（整数或浮点数，转换为 64 位浮点数）
|
||||
* 布尔值（true、false）
|
||||
* 字符串列表
|
||||
|
||||
所有其他字段将被忽略。
|
||||
|
||||
## 配置
|
||||
|
||||
### 处理
|
||||
|
||||
每个记录将根据"处理"部分中的配置分为文本字段和元字段。所有文本字段连接成单个字符串，然后分割成配置长度的块。如果指定，元数据字段将与嵌入的文本块一起按原样存储。请注意，元数据字段只能用于过滤而不能用于检索，并且必须是字符串、数字、布尔值类型（忽略所有其他值）。请注意，为每个条目保存的元数据总大小有 40kb 的限制。有关配置分块过程的选项使用 [Langchain Python 库](https://python.langchain.com/docs/get_started/introduction)。
|
||||
|
||||
在指定文本字段时，您可以使用点符号访问记录中的嵌套字段，例如 `user.name` 将访问 `user` 对象中的 `name` 字段。还可以使用通配符访问对象中的所有字段，例如 `users.*.name` 将访问 `users` 数组所有条目中的所有 `names` 字段。
|
||||
|
||||
块长度以 `tiktoken` 库产生的令牌来衡量。最大值为 8191 个令牌，这是 `text-embedding-ada-002` 模型支持的最大长度。
|
||||
|
||||
流名称作为元数据字段 `_ab_stream` 添加到每个文档中。如果可用，记录的主键用于标识文档，以避免在索引记录的更新版本时出现重复。它作为 `_ab_record_id` 元数据字段添加。
|
||||
|
||||
### 嵌入
|
||||
|
||||
连接器可以使用以下嵌入方法之一：
|
||||
|
||||
1. **OpenAI** - 使用 [OpenAI API](https://beta.openai.com/docs/api-reference/text-embedding)，连接器将使用具有 **1536 维** 的 `text-embedding-ada-002` 模型生成嵌入。此集成将受到 [OpenAI 嵌入 API 速度](https://platform.openai.com/docs/guides/rate-limits/overview) 的限制。
|
||||
|
||||
2. **Cohere** - 使用 [Cohere API](https://docs.cohere.com/reference/embed)，连接器将使用具有 **1024 维** 的 `embed-english-light-v2.0` 模型生成嵌入。
|
||||
|
||||
为了测试目的，也可以使用 [虚拟嵌入](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) 集成。它将生成随机嵌入，适用于测试数据管道而不产生嵌入成本。
|
||||
|
||||
### 索引/数据存储
|
||||
|
||||
- 对于 **主机**、**端口** 和 **数据库名称**，输入您的 OpenGauss 数据库的主机名、端口号和名称。
|
||||
- 列出 **默认模式**。
|
||||
|
||||
所有流将被索引/存储到同名的表中。如果表不存在将创建表。表将具有以下列：
|
||||
- `document_id`（字符串）- 文档的唯一标识符，通过附加流模式中的主键创建
|
||||
- `chunk_id`（字符串）- 块的唯一标识符，通过将块编号附加到 document_id 创建
|
||||
- `metadata`（变体）- 文档的元数据，以键值对形式存储
|
||||
- `document_content`（字符串）- 块的文本内容
|
||||
- `embedding`（向量）- 块的嵌入，以浮点数列表形式存储
|
||||
|
||||
|
||||
|
||||
> 注意：若表名超过63字节的最大标识符限制，会hash+截断处理
|
||||
|
||||
## 更新日志
|
||||
|
||||
<details>
|
||||
<summary>展开查看</summary>
|
||||
|
||||
| 版本 | 日期 | Pull Request | 主题 |
|
||||
|:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 0.1.0 | 2025-09-01 | TBD | 添加对 openGauss DateVec 作为向量目标连接器的支持。 |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
[🔝 返回顶部](#opengauss-datavec-目标连接器) | [🇺🇸 English Version](openGauss_DataVec_Connector_Document.md)
|
||||
|
||||
</div>
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
# type: ignore
|
||||
|
||||
import os
|
||||
|
||||
from apify_client import ApifyClient
|
||||
from langchain_core.output_parsers import StrOutputParser
|
||||
from langchain_core.prompts import PromptTemplate
|
||||
from langchain_core.runnables import RunnablePassthrough
|
||||
from langchain_opengauss import OpenGauss, OpenGaussSettings
|
||||
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
||||
|
||||
APIFY_API_TOKEN = os.getenv("APIFY_API_TOKEN") or "YOUR-APIFY-TOKEN"
|
||||
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") or "YOUR-OPENAI-API-KEY"
|
||||
|
||||
OPENGAUSS_HOST = os.getenv("OPENGAUSS_HOST")
|
||||
OPENGAUSS_PORT = os.getenv("OPENGAUSS_PORT")
|
||||
OPENGAUSS_USER = os.getenv("OPENGAUSS_USER")
|
||||
OPENGAUSS_PASSWORD = os.getenv("OPENGAUSS_PASSWORD")
|
||||
OPENGAUSS_DBNAME = os.getenv("OPENGAUSS_DBNAME")
|
||||
OPENGAUSS_TABLE_NAME = os.getenv("OPENGAUSS_TABLE_NAME")
|
||||
|
||||
client = ApifyClient(APIFY_API_TOKEN)
|
||||
|
||||
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
||||
|
||||
print("Starting Apify's Website Content Crawler")
|
||||
print("Crawling will take some time ... you can check the progress in the Apify console")
|
||||
|
||||
actor_call = client.actor(actor_id="apify/website-content-crawler").call(
|
||||
run_input={"maxCrawlPages": 10, "startUrls": [{"url": "https://opengauss.org/"}]}
|
||||
)
|
||||
|
||||
print("Actor website content crawler has finished")
|
||||
print(actor_call)
|
||||
|
||||
opengauss_integration_inputs = {
|
||||
"opengaussHost": OPENGAUSS_HOST,
|
||||
"opengaussPort": OPENGAUSS_PORT,
|
||||
"opengaussUser": OPENGAUSS_USER,
|
||||
"opengaussPassword": OPENGAUSS_PASSWORD,
|
||||
"opengaussDBname": OPENGAUSS_DBNAME,
|
||||
"opengaussTableName": OPENGAUSS_TABLE_NAME,
|
||||
"datasetFields": ["text"],
|
||||
"datasetId": actor_call["defaultDatasetId"],
|
||||
"deltaUpdatesPrimaryDatasetFields": ["url"],
|
||||
"expiredObjectDeletionPeriodDays": 7,
|
||||
"embeddingsApiKey": OPENAI_API_KEY,
|
||||
"embeddingsConfig": {
|
||||
"model": "text-embedding-3-small",
|
||||
},
|
||||
"embeddingsProvider": "OpenAI",
|
||||
"performChunking": True,
|
||||
"chunkSize": 2000,
|
||||
"chunkOverlap": 200
|
||||
}
|
||||
|
||||
print("Starting Apify's OpenGauss Integration")
|
||||
actor_call = client.actor("wyswyz/opengauss-integration").call(run_input=opengauss_integration_inputs)
|
||||
print("Apify's OpenGauss Integration has finished")
|
||||
print(actor_call)
|
||||
|
||||
print("Question answering using OpenGauss database")
|
||||
|
||||
config = OpenGaussSettings(
|
||||
host=OPENGAUSS_HOST,
|
||||
port=OPENGAUSS_PORT,
|
||||
user=OPENGAUSS_USER,
|
||||
password=OPENGAUSS_PASSWORD,
|
||||
database=OPENGAUSS_DBNAME,
|
||||
table_name=OPENGAUSS_TABLE_NAME,
|
||||
embedding_dimension=384,
|
||||
index_type="HNSW",
|
||||
distance_strategy="cosine",
|
||||
)
|
||||
vector_store = OpenGauss(embedding=embeddings, config=config)
|
||||
|
||||
prompt = PromptTemplate(
|
||||
input_variables=["context", "question"],
|
||||
template="Use the following pieces of retrieved context to answer the question. If you don't know the answer, "
|
||||
"just say that you don't know. \nQuestion: {question} \nContext: {context} \nAnswer:",
|
||||
)
|
||||
|
||||
|
||||
def format_docs(docs):
|
||||
return "\n\n".join(doc.page_content for doc in docs)
|
||||
|
||||
|
||||
rag_chain = (
|
||||
{
|
||||
"context": vector_store.as_retriever() | format_docs,
|
||||
"question": RunnablePassthrough(),
|
||||
}
|
||||
| prompt
|
||||
| ChatOpenAI(model="gpt-5-mini", temperature=1)
|
||||
| StrOutputParser()
|
||||
)
|
||||
|
||||
question = "什么是openGauss？"
|
||||
|
||||
print("Question:", question)
|
||||
print("Answer:", rag_chain.invoke(question))
|
||||
|
|
@ -1,219 +0,0 @@
|
|||
# 使用 Apify 和 openGauss 构建企业级 RAG 问答系统：实战教程
|
||||
|
||||
本文将通过一个完整的实例，向您展示如何利用 Apify 强大的云端数据抓取与集成能力，结合 openGauss 数据库的向量存储功能，快速构建一个针对 openGauss 官方文档的 RAG 问答机器人。
|
||||
|
||||
**Apify 是一个云端 Web 抓取和自动化平台。** 它允许开发者轻松地从任何网站提取数据，或将任何网站的工作流程自动化。其核心是 **"Actors"**——可以执行任意任务的无服务器云程序。
|
||||
|
||||
Apify 平台的优势在于：
|
||||
* **简化数据采集**：您无需关心基础设施、IP 轮换、浏览器指纹等复杂问题，Apify 会为您处理好一切。
|
||||
* **丰富的 Actor 市场**：Apify Store 中有大量预构建好的 Actors，例如通用的网站内容爬虫（Website Content Crawler）、搜索引擎爬虫等，可以开箱即用。
|
||||
* **强大的集成能力**：Apify 可以轻松地将采集到的数据推送到各种数据库、API 或云存储中，实现了从数据源到目的地的无缝衔接。
|
||||
|
||||
在本教程中，我们将使用两个关键的 Apify Actors：
|
||||
1. **`apify/website-content-crawler`**: 用于抓取 openGauss 官网的网页内容。
|
||||
2. **`wyswyz/opengauss-integration`**: 用于将抓取到的内容进行处理（分块、生成向量），并自动存入 openGauss 数据库。
|
||||
|
||||
|
||||
## **前置准备工作**
|
||||
|
||||
1. **Apify 账户**：注册一个 [Apify 账户](https://apify.com/) 并获取您的 API Token。
|
||||
2. **OpenAI 账户**：获取您的 [OpenAI API Key](https://platform.openai.com/api-keys)，用于内容向量化和问答生成。
|
||||
3. **openGauss 数据库**：您需要一个的 openGauss 数据库实例，并且支持向量数据库功能
|
||||
|
||||
### 使用docker部署openGauss
|
||||
**获取镜像** ：
|
||||
```bash
|
||||
$ docker pull opengauss/opengauss-server:latest
|
||||
```
|
||||
**查看镜像状态**
|
||||
```bash
|
||||
$ docker images
|
||||
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
opengauss/opengauss-server latest 9763e8b26794 6 months ago 1.68GB
|
||||
```
|
||||
**运行容器**
|
||||
```bash
|
||||
$ docker run --name opengauss --privileged=true -d -e GS_PASSWORD=YourPassoword -p 8888:5432 opengauss/opengauss-server:latest
|
||||
```
|
||||
**验证容器运行状态**
|
||||
```bash
|
||||
$ docker ps
|
||||
|
||||
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
|
||||
c5f0b44adf9a opengauss/opengauss-server:latest "entrypoint.sh gauss…" 7 weeks ago Up 7 weeks 0.0.0.0:8888->5432/tcp opengauss
|
||||
```
|
||||
至此，已经成功用docker部署openGauss数据库
|
||||
|
||||
**配置环境变量**：为了安全和方便，建议将密钥和数据库连接信息配置为环境变量。
|
||||
```bash
|
||||
export APIFY_API_TOKEN="YOUR-APIFY-TOKEN"
|
||||
export OPENAI_API_KEY="YOUR-OPENAI-API-KEY"
|
||||
export OPENGAUSS_HOST="your-db-host"
|
||||
export OPENGAUSS_PORT="your-db-port"
|
||||
export OPENGAUSS_USER="your-db-user"
|
||||
export OPENGAUSS_PASSWORD="your-db-password"
|
||||
export OPENGAUSS_DBNAME="your-db-name"
|
||||
export OPENGAUSS_TABLE_NAME="opengauss_docs"
|
||||
```
|
||||
|
||||
**Python 环境**：安装必要的库。
|
||||
```bash
|
||||
pip install apify-client langchain-core langchain-opengauss langchain-openai
|
||||
```
|
||||
|
||||
## **数据采集 - 使用 Apify 网站内容爬虫**
|
||||
|
||||
这是 RAG 流程的第一步：获取知识。我们使用 Apify 预构建的 `website-content-crawler` Actor 来抓取 openGauss 官网的内容。
|
||||
|
||||
```python
|
||||
import os
|
||||
from apify_client import ApifyClient
|
||||
|
||||
# ... [环境变量加载] ...
|
||||
|
||||
client = ApifyClient(APIFY_API_TOKEN)
|
||||
|
||||
print("Starting Apify's Website Content Crawler")
|
||||
print("Crawling will take some time")
|
||||
|
||||
# 调用 Actor 开始抓取
|
||||
actor_call = client.actor(actor_id="apify/website-content-crawler").call(
|
||||
run_input={
|
||||
"maxCrawlPages": 10,
|
||||
"startUrls": [{"url": "https://opengauss.org/"}]
|
||||
}
|
||||
)
|
||||
|
||||
print("Actor website content crawler has finished")
|
||||
print(actor_call)
|
||||
```
|
||||
|
||||
website-content-crawler的爬取行为可以通过run_input进行定制，我们选择爬取openGauss的主页。为了方便演示，设置最大爬取页数为10。等待actor爬取完成。
|
||||
|
||||
```bash
|
||||
2025-09-22T13:16:41.429Z INFO PlaywrightCrawler: Finished! Total 9 requests: 9 succeeded, 0 failed.
|
||||
```
|
||||
|
||||
|
||||
## **数据处理与入库 - 使用 Apify openGauss 集成 Actor**
|
||||
|
||||
现在我们有了原始的网页数据，下一步是将其处理并存入 openGauss 向量数据库。这正是 `opengauss-integration` Actor 的用武之地。它会自动完成**分块 (Chunking)、向量化 (Embedding) 和存储 (Storing)** 这一系列繁琐的工作。
|
||||
|
||||
首先配置 openGauss 数据库的参数，并设置我们需要的字段以及分块、向量化等参数，所有参数信息可以参考[actor详情页](https://apify.com/wyswyz/opengauss-integration)
|
||||
|
||||
```python
|
||||
# 准备 openGauss 集成 Actor 的输入参数
|
||||
opengauss_integration_inputs = {
|
||||
# 数据库连接信息
|
||||
"opengaussHost": OPENGAUSS_HOST,
|
||||
"opengaussPort": OPENGAUSS_PORT,
|
||||
"opengaussUser": OPENGAUSS_USER,
|
||||
"opengaussPassword": OPENGAUSS_PASSWORD,
|
||||
"opengaussDBname": OPENGAUSS_DBNAME,
|
||||
"opengaussTableName": OPENGAUSS_TABLE_NAME,
|
||||
|
||||
# 数据源和处理配置
|
||||
"datasetId": actor_call["defaultDatasetId"],
|
||||
"datasetFields": ["text"],
|
||||
"performChunking": True,
|
||||
"chunkSize": 2000,
|
||||
"chunkOverlap": 200,
|
||||
|
||||
# 向量化配置
|
||||
"embeddingsProvider": "OpenAI",
|
||||
"embeddingsApiKey": OPENAI_API_KEY,
|
||||
"embeddingsConfig": {
|
||||
"model": "text-embedding-3-small",
|
||||
},
|
||||
|
||||
# 其他可选配置
|
||||
"deltaUpdatesPrimaryDatasetFields": ["url"],
|
||||
"expiredObjectDeletionPeriodDays": 7,
|
||||
}
|
||||
```
|
||||
|
||||
接下来我们调用`wyswyz/opengauss-integration`将数据存储到openGauss数据库中
|
||||
|
||||
```python
|
||||
print("Starting Apify's OpenGauss Integration")
|
||||
# 调用 openGauss 集成 Actor
|
||||
actor_call = client.actor("wyswyz/opengauss-integration").call(run_input=opengauss_integration_inputs)
|
||||
print("Apify's OpenGauss Integration has finished")
|
||||
print(actor_call)
|
||||
```
|
||||
当这个 Actor 运行完毕后，您的 openGauss 数据库中指定的表（`opengauss_docs`）就已经包含了处理好的知识数据和对应的向量。
|
||||
|
||||
## **构建 RAG 应用 - 使用 LangChain 进行问答**
|
||||
|
||||
现在我们的知识库已经准备就绪，可以使用 LangChain 来构建一个问答应用了。
|
||||
|
||||
```python
|
||||
from langchain_core.output_parsers import StrOutputParser
|
||||
from langchain_core.prompts import PromptTemplate
|
||||
from langchain_core.runnables import RunnablePassthrough
|
||||
from langchain_opengauss import OpenGauss, OpenGaussSettings
|
||||
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
||||
|
||||
# ... [接上文] ...
|
||||
|
||||
print("Question answering using OpenGauss database")
|
||||
|
||||
# 1. 配置 openGauss 连接
|
||||
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
||||
config = OpenGaussSettings(
|
||||
host=OPENGAUSS_HOST,
|
||||
port=OPENGAUSS_PORT,
|
||||
user=OPENGAUSS_USER,
|
||||
password=OPENGAUSS_PASSWORD,
|
||||
database=OPENGAUSS_DBNAME,
|
||||
table_name=OPENGAUSS_TABLE_NAME,
|
||||
embedding_dimension=384,
|
||||
index_type="HNSW",
|
||||
distance_strategy="cosine",
|
||||
)
|
||||
# 初始化 LangChain 的 openGauss 向量存储
|
||||
vector_store = OpenGauss(embedding=embeddings, config=config)
|
||||
|
||||
# 2. 定义 Prompt 模板
|
||||
prompt = PromptTemplate(
|
||||
input_variables=["context", "question"],
|
||||
template="Use the following pieces of retrieved context to answer the question. If you don't know the answer, "
|
||||
"just say that you don't know. \nQuestion: {question} \nContext: {context} \nAnswer:",
|
||||
)
|
||||
|
||||
# 3. 定义文档格式化函数
|
||||
def format_docs(docs):
|
||||
return "\n\n".join(doc.page_content for doc in docs)
|
||||
|
||||
# 4. 构建 RAG 链 (Chain)
|
||||
rag_chain = (
|
||||
{
|
||||
"context": vector_store.as_retriever() | format_docs,
|
||||
"question": RunnablePassthrough(),
|
||||
}
|
||||
| prompt
|
||||
| ChatOpenAI(model="gpt-5-mini", temperature=1)
|
||||
| StrOutputParser()
|
||||
)
|
||||
|
||||
# 5. 提问与回答
|
||||
question = "什么是openGauss？"
|
||||
|
||||
print("Question:", question)
|
||||
print("Answer:", rag_chain.invoke(question))
|
||||
```
|
||||
|
||||
结果如下
|
||||
```bash
|
||||
Question: 什么是openGauss？
|
||||
Answer: openGauss 是一个开源数据库及其社区（即 openGauss 社区）。该项目面向企业级应用，倡导开源开放、社区协作，提供完善的技术文档和实践案例（如资源池化、告警机制、容灾集群、一主两备部署、线程池与 RDMA 指导、在 openEuler 上安装、与 OpenStack 集成等），并通过 meetup 等线下/线上活动推动生态繁荣。
|
||||
```
|
||||
|
||||
# 总结
|
||||
|
||||
通过本教程，我们利用 Apify 和 openGauss 成功构建了一个从数据采集到智能问答的端到端 RAG 应用。
|
||||
|
||||
此集成支持增量更新，仅更新已更改的数据。这种方法减少了不必要的嵌入计算和存储操作，使其适用于搜索和检索增强生成。
|
||||
|
||||
更多有关 Apify-openGauss 集成的信息，请参考集成[README文件](https://apify.com/wyswyz/opengauss-integration)。
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Files and folders to ignore during Docker build
|
||||
.env/
|
||||
.env.example
|
||||
|
||||
# Git
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# Docker
|
||||
.docker
|
||||
|
||||
# Python
|
||||
.pytest_cache
|
||||
.idea
|
||||
.mypy_cache
|
||||
|
||||
# mono-repo
|
||||
code/.venv
|
||||
|
|
@ -1,207 +0,0 @@
|
|||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[codz]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
#poetry.toml
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||
#pdm.lock
|
||||
#pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# pixi
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||
#pixi.lock
|
||||
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||
.pixi
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.envrc
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# Abstra
|
||||
# Abstra is an AI-powered process automation framework.
|
||||
# Ignore directories containing user credentials, local state, and settings.
|
||||
# Learn more at https://abstra.io/docs
|
||||
.abstra/
|
||||
|
||||
# Visual Studio Code
|
||||
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||
# you could uncomment the following to ignore the entire vscode folder
|
||||
# .vscode/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Cursor
|
||||
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
||||
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
||||
# refer to https://docs.cursor.com/context/ignore-files
|
||||
.cursorignore
|
||||
.cursorindexingignore
|
||||
|
||||
# Marimo
|
||||
marimo/_static/
|
||||
marimo/_lsp/
|
||||
__marimo__/
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: lint
|
||||
name: Lint codebase
|
||||
entry: make lint
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
||||
- id: type-check
|
||||
name: Type-check codebase
|
||||
entry: make type-check
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
.PHONY: clean install-dev lint type-check check-code format
|
||||
|
||||
DIRS_WITH_CODE = code
|
||||
DIRS_WITH_ACTORS = actors
|
||||
|
||||
clean:
|
||||
rm -rf .venv .mypy_cache .pytest_cache .ruff_cache __pycache__
|
||||
|
||||
install-dev:
|
||||
cd $(DIRS_WITH_CODE) && pip install --upgrade pip poetry && poetry install --with main,dev,opengauss && poetry run pre-commit install && cd ..
|
||||
|
||||
lint:
|
||||
poetry run -C $(DIRS_WITH_CODE) ruff check
|
||||
|
||||
type-check:
|
||||
poetry run -C $(DIRS_WITH_CODE) mypy
|
||||
|
||||
check-code: lint type-check
|
||||
|
||||
format:
|
||||
poetry run -C $(DIRS_WITH_CODE) ruff check --fix
|
||||
poetry run -C $(DIRS_WITH_CODE) ruff format
|
||||
|
||||
pydantic-model:
|
||||
datamodel-codegen --input $(DIRS_WITH_ACTORS)/opengauss/.actor/input_schema.json --output $(DIRS_WITH_CODE)/src/models/opengauss_input_model.py --input-file-type jsonschema --field-constraints --enum-field-as-literal all
|
||||
|
||||
|
||||
# Integration tests are marked with @pytest.mark.integration_test
|
||||
# You will require all databased running to run these tests.
|
||||
# Check docker-compose.yml for the list of databases.
|
||||
test-integration:
|
||||
poetry run -C $(DIRS_WITH_CODE) pytest --with-integration
|
||||
|
||||
test-unit:
|
||||
poetry run -C $(DIRS_WITH_CODE) pytest
|
||||
|
||||
test: test-unit test-integration
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
# Apify Actor for OpenGauss Integrations
|
||||
|
||||
This project was inspired by and derived from the official `apify/actor-vector-database-integrations` repository. Special thanks to the Apify team for their foundational work.
|
||||
|
||||
| Actor | Actor badge |
|
||||
|-----------------------------|---------------------|
|
||||
| [OpenGauss](https://opengauss.org/) | [](https://apify.com/wyswyz/opengauss-integration) |
|
||||
|
||||
#### Vector database integrations (Actors)
|
||||
|
||||
The Apify Vector Database Integrations facilitate the transfer of data from Apify Actors to openGauss vector database.
|
||||
This process includes data processing, optional splitting into chunks, embedding computation, and data storage
|
||||
|
||||
These integrations support incremental updates, ensuring that only changed data is updated.
|
||||
This reduces unnecessary embedding computation and storage operations, making it ideal for search and retrieval augmented generation (RAG) use cases.
|
||||
|
||||
## How does it work?
|
||||
|
||||
1. Retrieve a dataset as output from an Actor.
|
||||
2. _[Optional]_ Split text data into chunks using [langchain](https://python.langchain.com).
|
||||
3. _[Optional]_ Update only changed data.
|
||||
4. Compute embeddings, e.g. using [OpenAI](https://platform.openai.com/docs/guides/embeddings) or [Cohere](https://cohere.com/embeddings).
|
||||
5. Save data into the database.
|
||||
|
||||
## Supported Vector Embeddings
|
||||
|
||||
- [OpenAI](https://platform.openai.com/docs/guides/embeddings)
|
||||
- [Cohere](https://cohere.com/embeddings)
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
{
|
||||
"actorSpecification": 1,
|
||||
"name": "opengauss-integration",
|
||||
"title": "openGauss Integration",
|
||||
"description": "Upload a dataset to openGauss",
|
||||
"version": "0.0",
|
||||
"input": "./input_schema.json",
|
||||
"dockerfile": "../../../shared/Dockerfile",
|
||||
"readme": "./README.md",
|
||||
"changelog":"../../../shared/CHANGELOG.md",
|
||||
"storages": {
|
||||
"dataset": "../../../shared/dataset_schema.json"
|
||||
},
|
||||
"dockerContextDir": "../../.."
|
||||
}
|
||||
|
|
@ -1,180 +0,0 @@
|
|||
{
|
||||
"title": "openGauss-integration",
|
||||
"type": "object",
|
||||
"schemaVersion": 1,
|
||||
"properties": {
|
||||
"opengaussHost": {
|
||||
"title": "openGauss Host",
|
||||
"type": "string",
|
||||
"description": "The Host of openGauss",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"opengaussPort": {
|
||||
"title": "openGauss Port",
|
||||
"type": "string",
|
||||
"description": "The Port of openGauss",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"opengaussUser": {
|
||||
"title": "openGauss User",
|
||||
"type": "string",
|
||||
"description": "The User of openGauss",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"opengaussPassword": {
|
||||
"title": "openGauss Password",
|
||||
"type": "string",
|
||||
"description": "The Password of openGauss",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"opengaussDBname": {
|
||||
"title": "openGauss DBname",
|
||||
"type": "string",
|
||||
"description": "The DBname of openGauss",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"opengaussTableName": {
|
||||
"title": "openGauss SQL table name",
|
||||
"type": "string",
|
||||
"description": "The name of the table to use",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"embeddingsProvider": {
|
||||
"title": "Embeddings provider (as defined in the langchain API)",
|
||||
"description": "Choose the embeddings provider to use for generating embeddings",
|
||||
"type": "string",
|
||||
"editor": "select",
|
||||
"enum": ["OpenAI", "Cohere"],
|
||||
"default": "OpenAI",
|
||||
"sectionCaption": "Embeddings settings"
|
||||
},
|
||||
"embeddingsConfig": {
|
||||
"title": "Configuration for embeddings provider",
|
||||
"description": "Configure the parameters for the LangChain embedding class. Key points to consider:\n\n1. Typically, you only need to specify the model name. For example, for OpenAI, set the model name as {\"model\": \"text-embedding-3-small\"}.\n\n2. It's required to ensure that the vector size of your embeddings matches the size of embeddings in the database.\n\n3. Here are examples of embedding models:\n - [OpenAI](https://platform.openai.com/docs/guides/embeddings): `text-embedding-3-small`, `text-embedding-3-large`, etc.\n - [Cohere](https://docs.cohere.com/docs/cohere-embed): `embed-english-v3.0`, `embed-multilingual-light-v3.0`, etc.\n\n4. For more details about other parameters, refer to the [LangChain documentation](https://python.langchain.com/docs/integrations/text_embedding/).",
|
||||
"type": "object",
|
||||
"editor": "json"
|
||||
},
|
||||
"embeddingsApiKey": {
|
||||
"title": "Embeddings API KEY (whenever applicable, depends on provider)",
|
||||
"description": "Value of the API KEY for the embeddings provider (if required).\n\n For example for OpenAI it is OPENAI_API_KEY, for Cohere it is COHERE_API_KEY)",
|
||||
"type": "string",
|
||||
"editor": "textfield",
|
||||
"isSecret": true
|
||||
},
|
||||
"datasetFields": {
|
||||
"title": "Dataset fields to select from the dataset results and store in the database",
|
||||
"type": "array",
|
||||
"description": "This array specifies the dataset fields to be selected and stored in the vector store. Only the fields listed here will be included in the vector store.\n\nFor instance, when using the Website Content Crawler, you might choose to include fields such as `text`, `url`, and `metadata.title` in the vector store.",
|
||||
"default": ["text"],
|
||||
"prefill": ["text"],
|
||||
"editor": "stringList",
|
||||
"sectionCaption": "Dataset settings"
|
||||
},
|
||||
"metadataDatasetFields": {
|
||||
"title": "Dataset fields to select from the dataset and store as metadata in the database",
|
||||
"type": "object",
|
||||
"description": "A list of dataset fields which should be selected from the dataset and stored as metadata in the vector stores.\n\nFor example, when using the Website Content Crawler, you might want to store `url` in metadata. In this case, use `metadataDatasetFields parameter as follows {\"url\": \"url\"}`",
|
||||
"editor": "json"
|
||||
},
|
||||
"metadataObject": {
|
||||
"title": "Custom object to be stored as metadata in the vector store database",
|
||||
"type": "object",
|
||||
"description": "This object allows you to store custom metadata for every item in the vector store.\n\nFor example, if you want to store the `domain` as metadata, use the `metadataObject` like this: {\"domain\": \"apify.com\"}.",
|
||||
"editor": "json"
|
||||
},
|
||||
"datasetId": {
|
||||
"title": "Dataset ID",
|
||||
"type": "string",
|
||||
"description": "Dataset ID (when running standalone without integration)",
|
||||
"editor": "textfield"
|
||||
},
|
||||
"dataUpdatesStrategy": {
|
||||
"title": "Update strategy (add, upsert, deltaUpdates (default))",
|
||||
"description": "Choose the update strategy for the integration. The update strategy determines how the integration updates the data in the database.\n\nThe available options are:\n\n- **Add data** (`add`):\n - Always adds new records to the database.\n - No checks for existing records or updates are performed.\n - Useful when appending data without concern for duplicates.\n\n- **Upsert data** (`upsert`):\n - Updates existing records if they match a key or identifier.\n - Inserts new records into the database if they don't already exist.\n - Ideal for ensuring the database contains the most up-to-date data, avoiding duplicates.\n\n- **Update changed data based on deltas** (`deltaUpdates`):\n - Performs incremental updates by identifying differences (deltas) between the new dataset and the existing records.\n - Only adds new records and updates those that have changed.\n - Unchanged records are left untouched.\n - Maximizes efficiency by reducing unnecessary updates.\n\nSelect the strategy that best fits your use case.",
|
||||
"type": "string",
|
||||
"editor": "select",
|
||||
"enum": ["add", "upsert", "deltaUpdates"],
|
||||
"default": "deltaUpdates",
|
||||
"prefill": "deltaUpdates",
|
||||
"sectionCaption": "Data updates settings"
|
||||
},
|
||||
"dataUpdatesPrimaryDatasetFields": {
|
||||
"title": "Dataset fields to uniquely identify dataset items (only relevant when dataUpdatesStrategy is `upsert` or `deltaUpdates`)",
|
||||
"type": "array",
|
||||
"description": "This array contains fields that are used to uniquely identify dataset items, which helps to handle content changes across different runs.\n\nFor instance, in a web content crawling scenario, the `url` field could serve as a unique identifier for each item.",
|
||||
"editor": "stringList",
|
||||
"default": [
|
||||
"url"
|
||||
],
|
||||
"prefill": [
|
||||
"url"
|
||||
]
|
||||
},
|
||||
"enableDeltaUpdates": {
|
||||
"title": "Enable incremental updates for objects based on deltas (deprecated)",
|
||||
"type": "boolean",
|
||||
"description": "When set to true, this setting enables incremental updates for objects in the database by comparing the changes (deltas) between the crawled dataset items and the existing objects, uniquely identified by the `datasetKeysToItemId` field.\n\n The integration will only add new objects and update those that have changed, reducing unnecessary updates. The `datasetFields`, `metadataDatasetFields`, and `metadataObject` fields are used to determine the changes.",
|
||||
"default": true,
|
||||
"editor": "hidden"
|
||||
},
|
||||
"deltaUpdatesPrimaryDatasetFields": {
|
||||
"title": "Dataset fields to uniquely identify dataset items (only relevant when `enableDeltaUpdates` is enabled) (deprecated)",
|
||||
"type": "array",
|
||||
"description": "This array contains fields that are used to uniquely identify dataset items, which helps to handle content changes across different runs.\n\nFor instance, in a web content crawling scenario, the `url` field could serve as a unique identifier for each item.",
|
||||
"editor": "hidden",
|
||||
"default": [
|
||||
"url"
|
||||
],
|
||||
"prefill": [
|
||||
"url"
|
||||
]
|
||||
},
|
||||
"deleteExpiredObjects": {
|
||||
"title": "Delete expired objects from the database",
|
||||
"type": "boolean",
|
||||
"description": "When set to true, delete objects from the database that have not been crawled for a specified period.",
|
||||
"default": true
|
||||
},
|
||||
"expiredObjectDeletionPeriodDays": {
|
||||
"title": "Delete expired objects from the database after a specified number of days",
|
||||
"type": "integer",
|
||||
"description": "This setting allows the integration to manage the deletion of objects from the database that have not been crawled for a specified period. It is typically used in subsequent runs after the initial crawl.\n\nWhen the value is greater than 0, the integration checks if objects have been seen within the last X days (determined by the expiration period). If the objects are expired, they are deleted from the database. The specific value for `deletedExpiredObjectsDays` depends on your use case and how frequently you crawl data.\n\nFor example, if you crawl data daily, you can set `deletedExpiredObjectsDays` to 7 days. If you crawl data weekly, you can set `deletedExpiredObjectsDays` to 30 days.",
|
||||
"default": 30,
|
||||
"minimum": 0,
|
||||
"unit": "days",
|
||||
"editor": "number"
|
||||
},
|
||||
"performChunking": {
|
||||
"title": "Enable text chunking",
|
||||
"description": "When set to true, the text will be divided into smaller chunks based on the settings provided below. Proper chunking helps optimize retrieval and ensures accurate and efficient responses.",
|
||||
"default": true,
|
||||
"type": "boolean",
|
||||
"sectionCaption": "Text chunking settings"
|
||||
},
|
||||
"chunkSize": {
|
||||
"title": "Maximum chunk size",
|
||||
"type": "integer",
|
||||
"description": "Defines the maximum number of characters in each text chunk. Choosing the right size balances between detailed context and system performance. Optimal sizes ensure high relevancy and minimal response time.",
|
||||
"default": 2000,
|
||||
"minimum": 1
|
||||
},
|
||||
"chunkOverlap": {
|
||||
"title": "Chunk overlap",
|
||||
"type": "integer",
|
||||
"description": "Specifies the number of overlapping characters between consecutive text chunks. Adjusting this helps maintain context across chunks, which is crucial for accuracy in retrieval-augmented generation systems.",
|
||||
"default": 0,
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"opengaussHost",
|
||||
"opengaussPort",
|
||||
"opengaussUser",
|
||||
"opengaussPassword",
|
||||
"opengaussDBname",
|
||||
"opengaussTableName",
|
||||
"embeddingsProvider",
|
||||
"embeddingsApiKey",
|
||||
"datasetFields"
|
||||
]
|
||||
}
|
||||
|
|
@ -1,248 +0,0 @@
|
|||
# openGauss integration
|
||||
|
||||
The Apify openGauss integration transfers selected data from Apify Actors to a [openGauss](https://opengauss.org/) database.
|
||||
It processes the data, optionally splits it into chunks, computes embeddings, and saves them to openGauss.
|
||||
|
||||
This integration supports incremental updates, updating only the data that has changed.
|
||||
This approach reduces unnecessary embedding computation and storage operations, making it suitable for search and retrieval augmented generation (RAG) use cases.
|
||||
|
||||
💡 **Note**: This Actor is meant to be used together with other Actors' integration sections.
|
||||
For instance, if you are using the [Website Content Crawler](https://apify.com/apify/website-content-crawler), you can activate openGauss integration to save web data as vectors to openGauss.
|
||||
|
||||
## 📋 How does Apify-openGauss integration work?
|
||||
|
||||
Apify openGauss integration computes text embeddings and store them in openGauss.
|
||||
It uses [LangChain](https://www.langchain.com/) to compute embeddings and interact with [openGauss](https://opengauss.org/).
|
||||
|
||||
1. Retrieve a dataset as output from an Actor
|
||||
2. _[Optional]_ Split text data into chunks using `langchain`'s `RecursiveCharacterTextSplitter`
|
||||
(enable/disable using `performChunking` and specify `chunkSize`, `chunkOverlap`)
|
||||
3. _[Optional]_ Update only changed data (select `dataUpdatesStrategy`)
|
||||
4. Compute embeddings, e.g. using `OpenAI` or `Cohere` (specify `embeddings` and `embeddingsConfig`)
|
||||
5. Save data into the database
|
||||
|
||||
## ✅ Before you start
|
||||
|
||||
To utilize this integration, ensure you have:
|
||||
|
||||
- Created or existing `openGauss` database. You need to know `opengaussHost`, `opengaussPort`, `opengaussUser`, `opengaussPassword`, `opengaussDBName`and `opengaussTableName`.
|
||||
- An account to compute embeddings using one of the providers, e.g., [OpenAI](https://platform.openai.com/docs/guides/embeddings) or [Cohere](https://docs.cohere.com/docs/cohere-embed).
|
||||
|
||||
## 👉 Examples
|
||||
|
||||
The configuration consists of three parts: openGauss, embeddings provider, and data.
|
||||
|
||||
Ensure that the vector size of your embeddings aligns with the configuration of your openGauss.
|
||||
For instance, if you're using the `text-embedding-3-small` model from `OpenAI`, it generates vectors of size `1536`.
|
||||
This means your openGauss vector should also be configured to accommodate vectors of the same size, `1536` in this case.
|
||||
|
||||
For detailed input information refer to the [Input page]().
|
||||
|
||||
#### Database: openGauss
|
||||
|
||||
```json
|
||||
{
|
||||
"opengaussHost": "YOUR-opengaussHost",
|
||||
"opengaussPort": "YOUR-opengaussPort",
|
||||
"opengaussUser": "YOUR-opengaussUser",
|
||||
"opengaussPassword": "YOUR-opengaussPassword",
|
||||
"opengaussDBname": "YOUR-opengaussDBname",
|
||||
"opengaussTableName": "apif_collection"
|
||||
}
|
||||
```
|
||||
|
||||
#### Embeddings provider: OpenAI
|
||||
|
||||
```json
|
||||
{
|
||||
"embeddingsProvider": "OpenAIEmbeddings",
|
||||
"embeddingsApiKey": "YOUR-OPENAI-API-KEY",
|
||||
"embeddingsConfig": {"model": "text-embedding-3-large"}
|
||||
}
|
||||
```
|
||||
|
||||
### Save data from Website Content Crawler to openGauss
|
||||
|
||||
Data is transferred in the form of a dataset from [Website Content Crawler](https://apify.com/apify/website-content-crawler), which provides a dataset with the following output fields (truncated for brevity):
|
||||
|
||||
```json
|
||||
{
|
||||
"url": "https://www.apify.com",
|
||||
"text": "Apify is a platform that enables developers to build, run, and share automation tasks.",
|
||||
"metadata": {"title": "Apify"}
|
||||
}
|
||||
```
|
||||
|
||||
This dataset is then processed by the openGauss integration.
|
||||
In the integration settings you need to specify which fields you want to save to openGauss, e.g., `["text"]` and which of them should be used as metadata, e.g., `{"title": "metadata.title"}`.
|
||||
Without any other configuration, the data is saved to openGauss as is.
|
||||
|
||||
|
||||
```json
|
||||
{
|
||||
"datasetFields": ["text"],
|
||||
"metadataDatasetFields": {"title": "metadata.title"}
|
||||
}
|
||||
```
|
||||
|
||||
### Create chunks from Website Content Crawler data and save them to the database
|
||||
|
||||
Assume that the text data from the [Website Content Crawler](https://apify.com/apify/website-content-crawler) is too long to compute embeddings.
|
||||
Therefore, we need to divide the data into smaller pieces called chunks.
|
||||
We can leverage LangChain's `RecursiveCharacterTextSplitter` to split the text into chunks and save them into a database.
|
||||
The parameters `chunkSize` and `chunkOverlap` are important.
|
||||
The settings depend on your use case where a proper chunking helps optimize retrieval and ensures accurate responses.
|
||||
|
||||
```json
|
||||
{
|
||||
"datasetFields": ["text"],
|
||||
"metadataDatasetFields": {"title": "metadata.title"},
|
||||
"performChunking": true,
|
||||
"chunkSize": 1000,
|
||||
"chunkOverlap": 0
|
||||
}
|
||||
```
|
||||
|
||||
### Configure update strategy
|
||||
|
||||
To control how the integration updates data in the database, use the `dataUpdatesStrategy` parameter. This parameter allows you to choose between different update strategies based on your use case, such as adding new data, upserting records, or incrementally updating records based on changes (deltas). Below are the available strategies and explanations for when to use each:
|
||||
|
||||
- **Add data (`add`)**:
|
||||
- Appends new data to the database without checking for duplicates or updating existing records.
|
||||
- Suitable for cases where deduplication or updates are unnecessary, and the data simply needs to be added.
|
||||
- For example, you might use this strategy to continually append data from independent crawls without regard for overlaps.
|
||||
|
||||
- **Upsert data (`upsert`)**:
|
||||
- Delete existing records in the database if they match a key or identifier and inserts new records.
|
||||
- Ideal when you want to maintain accurate and up-to-date data while avoiding duplication.
|
||||
- For instance, this is useful in cases where unique items (such as user profiles or documents) need to be managed, ensuring the database reflects the latest changes.
|
||||
- Check the `dataUpdatesPrimaryDatasetFields` parameter to specify which fields are used to uniquely identify each dataset item.
|
||||
|
||||
- **Delta updates (`deltaUpdates`)**:
|
||||
- Incrementally updates records by identifying differences (deltas) between the new dataset and the existing database records.
|
||||
- Ensures only new or modified records are processed, leaving unchanged records untouched. This minimizes unnecessary database operations and improves efficiency.
|
||||
- This is the most efficient strategy when integrating data that evolves over time, such as website content or recurring crawls.
|
||||
- Check the `dataUpdatesPrimaryDatasetFields` parameter to specify which fields are used to uniquely identify each dataset item.
|
||||
|
||||
|
||||
### Incrementally update database from the Website Content Crawler
|
||||
|
||||
To incrementally update data from the [Website Content Crawler](https://apify.com/apify/website-content-crawler) to database, configure the integration to update only the changed or new data.
|
||||
This is controlled by the `dataUpdatesStrategy` setting.
|
||||
This way, the integration minimizes unnecessary updates and ensures that only new or modified data is processed.
|
||||
|
||||
A checksum is computed for each dataset item (together with all metadata) and stored in the database alongside the vectors.
|
||||
When the data is re-crawled, the checksum is recomputed and compared with the stored checksum.
|
||||
If the checksum is different, the old data (including vectors) is deleted and new data is saved.
|
||||
Otherwise, only the `last_seen_at` metadata field is updated to indicate when the data was last seen.
|
||||
|
||||
#### Provide unique identifier for each dataset item
|
||||
|
||||
To incrementally update the data, you need to be able to uniquely identify each dataset item.
|
||||
The variable `dataUpdatesPrimaryDatasetFields` specifies which fields are used to uniquely identify each dataset item and helps track content changes across different crawls.
|
||||
For instance, when working with the Website Content Crawler, you can use the URL as a unique identifier.
|
||||
|
||||
```json
|
||||
{
|
||||
"dataUpdatesStrategy": "deltaUpdates",
|
||||
"dataUpdatePrimaryDatasetFields": ["url"]
|
||||
}
|
||||
```
|
||||
|
||||
To fully maximize the potential of incremental data updates, it is recommended to start with an empty database.
|
||||
While it is possible to use this feature with an existing database, records that were not originally saved using a prefix or metadata will not be updated.
|
||||
|
||||
### Delete outdated (expired) data
|
||||
|
||||
The integration can delete data from the database that hasn't been crawled for a specified period, which is useful when data becomes outdated, such as when a page is removed from a website.
|
||||
|
||||
The deletion feature can be enabled or disabled using the `deleteExpiredObjects` setting.
|
||||
|
||||
For each crawl, the `last_seen_at` metadata field is created or updated.
|
||||
This field records the most recent time the data object was crawled.
|
||||
The `expiredObjectDeletionPeriodDays` setting is used to control number of days since the last crawl, after which the data object is considered expired.
|
||||
If a database object has not been seen for more than the `expiredObjectDeletionPeriodDays`, it will be deleted automatically.
|
||||
|
||||
The specific value of `expiredObjectDeletionPeriodDays` depends on your use case.
|
||||
|
||||
- If a website is crawled daily, `expiredObjectDeletionPeriodDays` can be set to 7.
|
||||
- If you crawl weekly, it can be set to 30.
|
||||
|
||||
To disable this feature, set `deleteExpiredObjects` to `false`.
|
||||
|
||||
```json
|
||||
{
|
||||
"deleteExpiredObjects": true,
|
||||
"expiredObjectDeletionPeriodDays": 30
|
||||
}
|
||||
```
|
||||
|
||||
💡 If you are using multiple Actors to update the same database, ensure that all Actors crawl the data at the same frequency.
|
||||
Otherwise, data crawled by one Actor might expire due to inconsistent crawling schedules.
|
||||
|
||||
|
||||
## 💾 Outputs
|
||||
|
||||
This integration will save the selected fields from your Actor to openGauss.
|
||||
|
||||
## 🔢 Example configuration
|
||||
|
||||
#### Full Input Example for Website Content Crawler Actor with openGauss integration
|
||||
|
||||
```json
|
||||
{
|
||||
"opengaussHost": "YOUR-opengaussHost",
|
||||
"opengaussPort": "YOUR-opengaussPort",
|
||||
"opengaussUser": "YOUR-opengaussUser",
|
||||
"opengaussPassword": "YOUR-opengaussPassword",
|
||||
"opengaussDBname": "YOUR-opengaussDBname",
|
||||
"opengaussTableName": "apif_collection",
|
||||
"embeddingsApiKey": "YOUR-OPENAI-API-KEY",
|
||||
"embeddingsConfig": {
|
||||
"model": "text-embedding-3-small"
|
||||
},
|
||||
"embeddingsProvider": "OpenAI",
|
||||
"datasetFields": [
|
||||
"text"
|
||||
],
|
||||
"dataUpdatesStrategy": "deltaUpdates",
|
||||
"dataUpdatePrimaryDatasetFields": ["url"],
|
||||
"expiredObjectDeletionPeriodDays": 7,
|
||||
"performChunking": true,
|
||||
"chunkSize": 2000,
|
||||
"chunkOverlap": 200
|
||||
}
|
||||
```
|
||||
|
||||
#### openGauss
|
||||
|
||||
```json
|
||||
{
|
||||
"opengaussHost": "YOUR-opengaussHost",
|
||||
"opengaussPort": "YOUR-opengaussPort",
|
||||
"opengaussUser": "YOUR-opengaussUser",
|
||||
"opengaussPassword": "YOUR-opengaussPassword",
|
||||
"opengaussDBname": "YOUR-opengaussDBname",
|
||||
"opengaussTableName": "apif_collection"
|
||||
}
|
||||
```
|
||||
|
||||
#### OpenAI embeddings
|
||||
|
||||
```json
|
||||
{
|
||||
"embeddingsApiKey": "YOUR-OPENAI-API-KEY",
|
||||
"embeddings": "OpenAI",
|
||||
"embeddingsConfig": {"model": "text-embedding-3-large"}
|
||||
}
|
||||
```
|
||||
|
||||
#### Cohere embeddings
|
||||
|
||||
```json
|
||||
{
|
||||
"embeddingsApiKey": "YOUR-COHERE-API-KEY",
|
||||
"embeddings": "Cohere",
|
||||
"embeddingsConfig": {"model": "embed-multilingual-v3.0"}
|
||||
}
|
||||
```
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# IDE configurations
|
||||
.idea
|
||||
.vscode
|
||||
|
||||
# crawlee and apify storage folders
|
||||
apify_storage
|
||||
crawlee_storage
|
||||
storage
|
||||
|
||||
# python related
|
||||
__pycache__
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.coverage
|
||||
.venv
|
||||
|
||||
# git folder
|
||||
.git
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
end_of_line = lf
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[{*.yaml, *.yml, *.toml, *.ini, *.cfg}]
|
||||
indent_size = 2
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
APIFY_API_TOKEN=
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# openGauss
|
||||
OPENGAUSS_HOST=
|
||||
OPENGAUSS_PORT=
|
||||
OPENGAUSS_USER=
|
||||
OPENGAUSS_PASSWORD=
|
||||
OPENGAUSS_DBNAME=
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
# This file tells Git which files shouldn't be added to source control
|
||||
|
||||
.idea
|
||||
.DS_Store
|
||||
.vscode
|
||||
|
||||
apify_storage
|
||||
storage
|
||||
|
||||
.venv/
|
||||
.env/
|
||||
__pypackages__
|
||||
dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
__pycache__
|
||||
|
||||
.mypy_cache
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
|
||||
.scrapy
|
||||
*.log
|
||||
|
||||
# Added by Apify CLI
|
||||
node_modules
|
||||
.venv
|
||||
.env
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,4 +0,0 @@
|
|||
cache-dir = "/tmp/.poetry-cache"
|
||||
|
||||
[virtualenvs]
|
||||
in-project = true
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
[tool.poetry]
|
||||
# These fields are not used by Apify, fill configs in .actor/ instead
|
||||
authors = ["jiri.spilka@apify.com"]
|
||||
description = ""
|
||||
name = "store-vector-db"
|
||||
readme = "README.md"
|
||||
version = "0.1.5"
|
||||
package-mode = false
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
apify = "^2.7.3"
|
||||
apify-client = "^1.12.2"
|
||||
backoff = "^2.2.1"
|
||||
langchain-cohere = "^0.3.0"
|
||||
langchain-community = "^0.3.0"
|
||||
langchain-core = "0.3.70"
|
||||
langchain-openai = "^0.2.0"
|
||||
openai = "^1.17.0"
|
||||
python = ">=3.11,<3.12"
|
||||
python-dotenv = "^1.0.1"
|
||||
langchain-apify = "^0.1.4"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
coverage = "^7.5.4"
|
||||
datamodel-code-generator = "^0.25.5"
|
||||
ipython = "^8.23.0"
|
||||
mypy = "^1.9.0"
|
||||
pandas = "^2.2.2"
|
||||
pre-commit = "^3.7.0"
|
||||
pytest = "^8.2.0"
|
||||
pytest-asyncio = "^0.23.6"
|
||||
pytest-integration-mark = "^0.2.0"
|
||||
ruff = "^0.3.5"
|
||||
|
||||
[tool.poetry.group.opengauss]
|
||||
optional = true
|
||||
|
||||
[tool.poetry.group.opengauss.dependencies]
|
||||
langchain-opengauss = "^0.1.4"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 150
|
||||
exclude = ["src/models/**", "src/examples/**"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["ALL"]
|
||||
ignore = [
|
||||
"ANN101", # Missing type annotation for `{name}` in method
|
||||
"ANN102", # Missing type annotation for `{name}` in classmethod
|
||||
"ANN401", # Dynamically typed expressions (typing.Any) are disallowed in {filename}
|
||||
"BLE001", # Do not catch blind exception
|
||||
"C901", # `{name}` is too complex
|
||||
"COM812", # This rule may cause conflicts when used with the formatter
|
||||
"D100", # Missing docstring in public module
|
||||
"D104", # Missing docstring in public package
|
||||
"D107", # Missing docstring in `__init__`
|
||||
"EM", # flake8-errmsg
|
||||
"G004", # Logging statement uses f-string
|
||||
"ISC001", # This rule may cause conflicts when used with the formatter
|
||||
"FIX", # flake8-fixme
|
||||
"PGH003", # Use specific rule codes when ignoring type issues
|
||||
"PLR0911", # Too many return statements
|
||||
"PLR0913", # Too many arguments in function definition
|
||||
"PLR0915", # Too many statements
|
||||
"PTH", # flake8-use-pathlib
|
||||
"PYI034", # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
|
||||
"PYI036", # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
|
||||
"S102", # Use of `exec` detected
|
||||
"S105", # Possible hardcoded password assigned to
|
||||
"S106", # Possible hardcoded password assigned to argument: "{name}"
|
||||
"S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
|
||||
"S303", # Use of insecure MD2, MD4, MD5, or SHA1 hash function
|
||||
"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
|
||||
"TD002", # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
|
||||
"TID252", # Prefer absolute imports over relative imports from parent modules
|
||||
"TRY003", # Avoid specifying long messages outside the exception class
|
||||
#
|
||||
"D",
|
||||
]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"**/__init__.py" = [
|
||||
"F401", # Unused imports
|
||||
]
|
||||
"**/{tests}/*" = [
|
||||
"D", # Everything from the pydocstyle
|
||||
"INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
|
||||
"PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
|
||||
"S101", # Use of assert detected
|
||||
"SLF001", # Private member accessed: `{name}`
|
||||
"T20", # flake8-print
|
||||
"TRY301", # Abstract `raise` to an inner function
|
||||
]
|
||||
|
||||
[tool.ruff.lint.flake8-quotes]
|
||||
docstring-quotes = "double"
|
||||
inline-quotes = "double"
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
#[tool.ruff.lint.isort]
|
||||
#known-first-party = ["apify"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "-ra"
|
||||
asyncio_mode = "auto"
|
||||
timeout = 1200
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.11"
|
||||
files = ["src", "tests"]
|
||||
check_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
disallow_untyped_calls = true
|
||||
disallow_untyped_decorators = true
|
||||
disallow_untyped_defs = true
|
||||
no_implicit_optional = true
|
||||
warn_redundant_casts = true
|
||||
warn_return_any = true
|
||||
warn_unreachable = true
|
||||
warn_unused_ignores = true
|
||||
|
||||
[tool.mypy-sortedcollections]
|
||||
ignore_missing_imports = true
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
import asyncio
|
||||
import logging
|
||||
|
||||
from apify.log import ActorLogFormatter
|
||||
|
||||
from .entrypoint import main
|
||||
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(ActorLogFormatter())
|
||||
|
||||
apify_client_logger = logging.getLogger("apify_client")
|
||||
apify_client_logger.setLevel(logging.INFO)
|
||||
apify_client_logger.addHandler(handler)
|
||||
|
||||
apify_logger = logging.getLogger("apify")
|
||||
apify_logger.setLevel(logging.DEBUG)
|
||||
apify_logger.addHandler(handler)
|
||||
|
||||
asyncio.run(main())
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, TypeAlias
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .models import OpengaussIntegration
|
||||
from .vector_stores import OpenGaussDatabase
|
||||
|
||||
ActorInputsDb: TypeAlias = OpengaussIntegration
|
||||
VectorDb: TypeAlias = OpenGaussDatabase
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
import enum
|
||||
|
||||
VCR_HEADERS_EXCLUDE = ["Authorization", "Api-Key"]
|
||||
|
||||
DAY_IN_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
class SupportedVectorStores(str, enum.Enum):
|
||||
opengauss = "opengauss"
|
||||
|
||||
|
||||
class SupportedEmbeddings(str, enum.Enum):
|
||||
openai = "OpenAI"
|
||||
cohere = "Cohere"
|
||||
fake = "Fake"
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from apify import Actor
|
||||
|
||||
from .constants import SupportedEmbeddings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langchain_core.embeddings import Embeddings
|
||||
|
||||
|
||||
async def get_embedding_provider(embeddings_name: str, api_key: str | None = None, config: dict | None = None) -> Embeddings:
|
||||
"""Return the embeddings based on the user preference."""
|
||||
|
||||
if embeddings_name == SupportedEmbeddings.openai:
|
||||
from langchain_openai.embeddings import OpenAIEmbeddings
|
||||
|
||||
config = config or {}
|
||||
config["openai_api_key"] = api_key
|
||||
return config and OpenAIEmbeddings(**config) or OpenAIEmbeddings()
|
||||
|
||||
if embeddings_name == SupportedEmbeddings.cohere:
|
||||
from langchain_cohere import CohereEmbeddings
|
||||
|
||||
config = config or {}
|
||||
config["cohere_api_key"] = api_key
|
||||
return CohereEmbeddings(**config)
|
||||
|
||||
if embeddings_name == SupportedEmbeddings.fake:
|
||||
from langchain_core.embeddings import FakeEmbeddings
|
||||
|
||||
config = config or {}
|
||||
return FakeEmbeddings(**config)
|
||||
|
||||
await Actor.fail(status_message=f"Failed to get embeddings for embeddings: {embeddings_name} and config: {config}")
|
||||
raise ValueError("Failed to get embeddings")
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
import os
|
||||
|
||||
from apify import Actor
|
||||
|
||||
from .constants import SupportedVectorStores
|
||||
from .main import run_actor
|
||||
from .models import OpengaussIntegration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
Actor.log.info("Starting the Vector Store Actor")
|
||||
|
||||
if not (actor_input := await Actor.get_input() or {}):
|
||||
await Actor.fail(status_message="No input provided", exit_code=1)
|
||||
|
||||
# Set the Apify API token if it is available in the environment variables.
|
||||
if apify_api_token := os.getenv("APIFY_TOKEN"):
|
||||
os.environ["APIFY_API_TOKEN"] = apify_api_token
|
||||
|
||||
if not (arg := os.getenv("ACTOR_PATH_IN_DOCKER_CONTEXT")):
|
||||
if Actor.is_at_home():
|
||||
await Actor.exit(
|
||||
exit_code=100,
|
||||
status_message="This Actor was built incorrectly; no environment variable specifies which Actor "
|
||||
"to start. If you encounter this issue, please contact the Actor developer.",
|
||||
)
|
||||
|
||||
arg = f"actors/{SupportedVectorStores.opengauss.value}"
|
||||
Actor.log.warning(
|
||||
f"The environment variable ACTOR_PATH_IN_DOCKER_CONTEXT was not specified. " f"Using default for local development: {arg}"
|
||||
)
|
||||
|
||||
actor_type = arg.split("/")[-1]
|
||||
Actor.log.info("Received start argument (vector database name): %s", actor_type)
|
||||
|
||||
actor_input_ensure_backward_compatibility(actor_input)
|
||||
if actor_type == SupportedVectorStores.opengauss.value:
|
||||
await run_actor(OpengaussIntegration(**actor_input), actor_input)
|
||||
else:
|
||||
await Actor.exit(
|
||||
exit_code=10,
|
||||
status_message=f"This Actor was built incorrectly; an unknown Actor was selected "
|
||||
f"to start ({actor_type}). If you encounter this issue, please contact the Actor developer.",
|
||||
)
|
||||
|
||||
|
||||
def actor_input_ensure_backward_compatibility(actor_input: dict) -> None:
|
||||
"""Ensure backward compatibility for the actor input."""
|
||||
if not actor_input.get("dataUpdatesStrategy"):
|
||||
# legacy update mechanism
|
||||
if actor_input.get("enableDeltaUpdates") is False:
|
||||
actor_input["dataUpdatesStrategy"] = "add"
|
||||
else:
|
||||
actor_input["dataUpdatesStrategy"] = "deltaUpdates"
|
||||
else:
|
||||
# for integrations that do not have updateStrategy implemented
|
||||
actor_input["enableDeltaUpdates"] = actor_input["dataUpdatesStrategy"] == "deltaUpdates"
|
||||
|
||||
if not actor_input.get("dataUpdatesPrimaryDatasetFields"):
|
||||
actor_input["dataUpdatesPrimaryDatasetFields"] = actor_input.get("deltaUpdatesPrimaryDatasetFields", [])
|
||||
else:
|
||||
# for integrations that do not have updateStrategy implemented
|
||||
actor_input["deltaUpdatesPrimaryDatasetFields"] = actor_input.get("dataUpdatesPrimaryDatasetFields")
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
# type: ignore
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from langchain_openai.embeddings import OpenAIEmbeddings
|
||||
|
||||
from ..models import OpengaussIntegration
|
||||
from .data_examples_uuid import (
|
||||
ID1, ID3, ID4A, ID4B, ID4C, ID5A, ID5B, ID5C, ID6,
|
||||
crawl_1, crawl_2, expected_results,
|
||||
)
|
||||
from ..vcs import compare_crawled_data_with_db
|
||||
from ..vector_stores.opengauss import OpenGaussDatabase
|
||||
|
||||
load_dotenv()
|
||||
OPENGAUSS_TABLE_NAME = os.getenv("OPENGAUSS_TABLE_NAME", "apify")
|
||||
|
||||
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
||||
|
||||
DROP_AND_INSERT = True
|
||||
|
||||
db = OpenGaussDatabase(
|
||||
actor_input=OpengaussIntegration(
|
||||
opengaussHost=os.getenv("OPENGAUSS_HOST"),
|
||||
opengaussPort=os.getenv("OPENGAUSS_PORT"),
|
||||
opengaussUser=os.getenv("OPENGAUSS_USER"),
|
||||
opengaussPassword=os.getenv("OPENGAUSS_PASSWORD"),
|
||||
opengaussDBname=os.getenv("OPENGAUSS_DBNAME"),
|
||||
opengaussTableName=OPENGAUSS_TABLE_NAME,
|
||||
embeddingsProvider="OpenAI",
|
||||
embeddingsApiKey=os.getenv("OPENAI_API_KEY"),
|
||||
datasetFields=["text"],
|
||||
),
|
||||
embeddings=embeddings,
|
||||
)
|
||||
|
||||
|
||||
def wait_for_index(sec: float = 1.0):
|
||||
time.sleep(sec)
|
||||
|
||||
|
||||
if DROP_AND_INSERT:
|
||||
db.delete_all()
|
||||
r = db.similarity_search("text", k=100)
|
||||
print("Initial results count:", len(r))
|
||||
|
||||
inserted = db.add_documents(documents=crawl_1, ids=[d.metadata["chunk_id"] for d in crawl_1])
|
||||
print("Inserted ids:", inserted)
|
||||
wait_for_index()
|
||||
|
||||
r = db.similarity_search("text", k=100)
|
||||
print("Search results:", r)
|
||||
print("Search results count:", len(r))
|
||||
|
||||
res = db.search_by_vector(db.dummy_vector, k=10)
|
||||
print("Objects in the database:", len(res), res)
|
||||
assert len(res) == 6, "Expected 6 objects in the database"
|
||||
|
||||
data_add, ids_update_last_seen, ids_del = compare_crawled_data_with_db(db, crawl_2)
|
||||
|
||||
print("Data to add", data_add)
|
||||
print("Ids to update", ids_update_last_seen)
|
||||
print("Ids to delete", ids_del)
|
||||
|
||||
assert len(data_add) == 4, "Expected 4 objects to add"
|
||||
assert data_add[0].metadata["chunk_id"] == ID4C
|
||||
assert data_add[1].metadata["chunk_id"] == ID5B
|
||||
assert data_add[2].metadata["chunk_id"] == ID5C
|
||||
assert data_add[3].metadata["chunk_id"] == ID6
|
||||
|
||||
assert len(ids_update_last_seen) == 1, "Expected 1 object to update"
|
||||
assert ID3 in ids_update_last_seen, f"Expected {ID3} to be updated"
|
||||
|
||||
assert len(ids_del) == 3, "Expected 3 objects to delete"
|
||||
assert ID4A in ids_del, f"Expected {ID4A} to be deleted"
|
||||
assert ID4B in ids_del, f"Expected {ID4B} to be deleted"
|
||||
assert ID5A in ids_del, f"Expected {ID5A} to be deleted"
|
||||
|
||||
# Delete data that were removed
|
||||
db.delete(ids_del)
|
||||
wait_for_index()
|
||||
res = db.search_by_vector(db.dummy_vector, k=10)
|
||||
print("Database objects after delete: ", len(res), res)
|
||||
assert len(res) == 3, "Expected 3 objects in the database after deletion"
|
||||
|
||||
# Add new data
|
||||
r = db.add_documents(data_add, ids=[d.metadata["chunk_id"] for d in data_add])
|
||||
wait_for_index()
|
||||
res = db.search_by_vector(db.dummy_vector, k=10)
|
||||
print("Database objects after adding new", len(res), res)
|
||||
ids = [r.metadata["chunk_id"] for r in res]
|
||||
assert len(res) == 7, "Expected 7 objects in the database after addition"
|
||||
assert ID4C in ids and ID5B in ids and ID5C in ids, "Expected new chunk_ids to be present"
|
||||
|
||||
# Update metadata (last_seen_at)
|
||||
ts = int(datetime.now(timezone.utc).timestamp())
|
||||
res = db.search_by_vector(db.dummy_vector, k=10)
|
||||
# precondition in examples: ID3 initially has last_seen_at == 1
|
||||
assert next(r for r in res if r.metadata["chunk_id"] == ID3).metadata["last_seen_at"] == 1
|
||||
|
||||
db.update_last_seen_at(ids_update_last_seen)
|
||||
wait_for_index()
|
||||
|
||||
res = db.search_by_vector(db.dummy_vector, k=10)
|
||||
assert len(res) == 7, "Expected 7 objects after metadata update"
|
||||
assert next(r for r in res if r.metadata["chunk_id"] == ID3).metadata["last_seen_at"] >= ts, f"Expected {ID3} to be updated"
|
||||
|
||||
# delete expired objects
|
||||
db.delete_expired(expired_ts=1)
|
||||
wait_for_index()
|
||||
|
||||
res = db.search_by_vector(db.dummy_vector, k=10)
|
||||
res = [r for r in res]
|
||||
print("Database objects after all updates", len(res), res)
|
||||
assert len(res) == 6, "Expected 6 objects after all updates"
|
||||
assert next((r for r in res if r.metadata["chunk_id"] == ID1), None) is None, f"Expected {ID1} to be deleted"
|
||||
|
||||
# compare results with expected results
|
||||
for r in expected_results:
|
||||
d = db.get_by_id(r.metadata["chunk_id"])
|
||||
assert d is not None, f"Expected document {r.metadata['chunk_id']} to exist"
|
||||
metadata = d.metadata
|
||||
assert metadata["item_id"] == r.metadata["item_id"], f"Expected item_id {r.metadata['item_id']}"
|
||||
assert metadata["checksum"] == r.metadata["checksum"], f"Expected checksum {r.metadata['checksum']}"
|
||||
|
||||
print("DONE")
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
"""
|
||||
Define crawled data for the database playground files
|
||||
"""
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
d1 = Document(page_content="Expired->del", metadata={"item_id": "id1", "id": "id1#1", "checksum": "1", "last_seen_at": 0})
|
||||
d2 = Document(page_content="Old->not-del", metadata={"item_id": "id2", "id": "id2#2", "checksum": "2", "last_seen_at": 1})
|
||||
d3a = Document(page_content="Unchanged->upt-meta", metadata={"item_id": "id3", "id": "id3#3", "checksum": "3", "last_seen_at": 1})
|
||||
d3b = Document(page_content="Unchanged->upt-meta", metadata={"item_id": "id3", "id": "id3#3", "checksum": "3", "last_seen_at": 2})
|
||||
d4a = Document(page_content="Changed->del", metadata={"item_id": "id4", "id": "id4#4a", "checksum": "4", "last_seen_at": 1})
|
||||
d4b = Document(page_content="Changed->del", metadata={"item_id": "id4", "id": "id4#4b", "checksum": "4", "last_seen_at": 1})
|
||||
d4c = Document(page_content="Changed->add-new", metadata={"item_id": "id4", "id": "id4#4c", "checksum": "0", "last_seen_at": 2})
|
||||
d5 = Document(page_content="New->add", metadata={"item_id": "id5", "id": "id5#5", "checksum": "5", "last_seen_at": 2})
|
||||
|
||||
|
||||
crawl_1 = [d1, d2, d3a, d4a, d4b]
|
||||
crawl_2 = [d3b, d4c, d5]
|
||||
expected_results = [d2, d3b, d4c, d5]
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
"""
|
||||
Define crawled data for the database playground files
|
||||
"""
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
UUID = "00000000-0000-0000-0000-0000000000"
|
||||
ID1 = f"{UUID}10"
|
||||
ID2 = f"{UUID}20"
|
||||
ID3 = f"{UUID}30"
|
||||
ID4A, ID4B, ID4C = f"{UUID}4a", f"{UUID}4b", f"{UUID}4c"
|
||||
ID5A, ID5B, ID5C = f"{UUID}5a", f"{UUID}5b", f"{UUID}5c"
|
||||
ID6 = f"{UUID}60"
|
||||
|
||||
ITEM_ID1 = "id1"
|
||||
ITEM_ID4 = "id4"
|
||||
|
||||
d1 = Document(page_content="Expired->del", metadata={"item_id": ITEM_ID1, "chunk_id": ID1, "checksum": "1", "last_seen_at": 0})
|
||||
d2 = Document(page_content="Old->not-del", metadata={"item_id": "id2", "chunk_id": ID2, "checksum": "2", "last_seen_at": 1})
|
||||
d3a = Document(page_content="Unchanged->upt-meta", metadata={"item_id": "id3", "chunk_id": ID3, "checksum": "3", "last_seen_at": 1})
|
||||
d3b = Document(page_content="Unchanged->upt-meta", metadata={"item_id": "id3", "chunk_id": ID3, "checksum": "3", "last_seen_at": 2})
|
||||
d4a = Document(page_content="Changed->del", metadata={"item_id": ITEM_ID4, "chunk_id": ID4A, "checksum": "4", "last_seen_at": 1})
|
||||
d4b = Document(page_content="Changed->del", metadata={"item_id": ITEM_ID4, "chunk_id": ID4B, "checksum": "4", "last_seen_at": 1})
|
||||
d4c = Document(page_content="Changed->add-new", metadata={"item_id": ITEM_ID4, "chunk_id": ID4C, "checksum": "4c", "last_seen_at": 2})
|
||||
d5a = Document(page_content="Changed->del", metadata={"item_id": "id5", "chunk_id": ID5A, "checksum": "5", "last_seen_at": 1})
|
||||
d5b = Document(page_content="Changed->add-new", metadata={"item_id": "id5", "chunk_id": ID5B, "checksum": "5bc", "last_seen_at": 2})
|
||||
d5c = Document(page_content="Changed->add-new", metadata={"item_id": "id5", "chunk_id": ID5C, "checksum": "5bc", "last_seen_at": 2})
|
||||
d6 = Document(page_content="New->add", metadata={"item_id": "id5", "chunk_id": ID6, "checksum": "6", "last_seen_at": 2})
|
||||
|
||||
|
||||
crawl_1 = [d1, d2, d3a, d4a, d4b, d5a]
|
||||
crawl_2 = [d3b, d4c, d5b, d5c, d6]
|
||||
expected_results = [d2, d3b, d4c, d5b, d5c, d6]
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
class FailedToConnectToDatabaseError(Exception):
|
||||
"""Failed to connect to a vector database."""
|
||||
|
|
@ -1,154 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from apify import Actor
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
from .constants import DAY_IN_SECONDS
|
||||
from .emb import get_embedding_provider
|
||||
from .utils import add_chunk_id, add_item_checksum, get_dataset_loader
|
||||
from .vcs import delete_expired_objects, get_vector_database, update_db_with_crawled_data, upsert_db_with_crawled_data
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langchain_core.documents import Document
|
||||
from langchain_core.embeddings import Embeddings
|
||||
|
||||
from ._types import ActorInputsDb, VectorDb
|
||||
|
||||
|
||||
async def run_actor(actor_input: ActorInputsDb, payload: dict) -> None:
|
||||
"""Main function to run the actor.
|
||||
|
||||
It loads the dataset, chunks the documents if necessary and updates the vector store with the new documents while removing the old ones.
|
||||
"""
|
||||
|
||||
payload = payload.get("payload", {})
|
||||
resource = payload.get("resource", {})
|
||||
if not (dataset_id := resource.get("defaultDatasetId") or actor_input.datasetId):
|
||||
msg = (
|
||||
"The `datasetId` is not provided. There are two ways to specify the datasetId:"
|
||||
"1. Automatic Input: If this integration is used with other Actors, such as the Website Content Crawler, the datasetId should be "
|
||||
"automatically passed in the 'payload'. Please check the `Input` payload to ensure the datasetId is included."
|
||||
"2. Manual Input: If you are running this Actor independently, you need to manually specify the 'datasetId'. "
|
||||
"You can do this by entering the dataset ID in the 'Dataset Settings' section of the Actor's input screen."
|
||||
"Please verify that one of these options is correctly configured to provide the datasetId."
|
||||
)
|
||||
|
||||
Actor.log.error(msg)
|
||||
await Actor.fail(status_message=msg)
|
||||
return
|
||||
|
||||
embeddings = await get_embeddings(actor_input)
|
||||
documents = await load_dataset(actor_input, dataset_id)
|
||||
documents = add_item_checksum(documents, actor_input.dataUpdatesPrimaryDatasetFields) # type: ignore[arg-type]
|
||||
|
||||
if actor_input.performChunking:
|
||||
text_splitter = RecursiveCharacterTextSplitter(chunk_size=actor_input.chunkSize, chunk_overlap=actor_input.chunkOverlap)
|
||||
documents = text_splitter.split_documents(documents)
|
||||
Actor.log.info("Documents chunked to %s chunks", len(documents))
|
||||
|
||||
documents = add_chunk_id(documents)
|
||||
|
||||
try:
|
||||
vcs_: VectorDb = await get_vector_database(actor_input, embeddings)
|
||||
except Exception as e:
|
||||
Actor.log.exception(e)
|
||||
await Actor.fail(
|
||||
status_message="Failed to connect/get database. Please ensure the following: "
|
||||
"1. Database credentials are correct and the database is configure properly. "
|
||||
"2. The vector dimension of your embedding model in the Actor input (Embedding settings -> model) matches the one set up in the database."
|
||||
f" Database error message: {e}"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
data_update_strategy = hasattr(actor_input, "dataUpdatesStrategy") and actor_input.dataUpdatesStrategy
|
||||
if data_update_strategy == "deltaUpdates":
|
||||
Actor.log.info("Update database with crawled data. Delta updates enabled")
|
||||
update_db_with_crawled_data(vcs_, documents)
|
||||
elif data_update_strategy == "add":
|
||||
vcs_.add_documents(documents)
|
||||
Actor.log.info("Added %s new objects to the vector store", len(documents))
|
||||
elif data_update_strategy == "upsert":
|
||||
upsert_db_with_crawled_data(vcs_, documents)
|
||||
else:
|
||||
await Actor.fail(
|
||||
status_message=f"Invalid dataUpdatesStrategy: {data_update_strategy}. "
|
||||
f"Please ensure that the configuration in the Database Settings is correct."
|
||||
)
|
||||
|
||||
if actor_input.deleteExpiredObjects:
|
||||
expired_days = actor_input.expiredObjectDeletionPeriodDays or 0
|
||||
ts_expired = expired_days and int(datetime.now(timezone.utc).timestamp() - expired_days * DAY_IN_SECONDS) or 0
|
||||
Actor.log.info("Delete expired objects in the database: expired_days: %s", expired_days)
|
||||
delete_expired_objects(vcs_, ts_expired)
|
||||
|
||||
await Actor.push_data([doc.dict() for doc in documents])
|
||||
|
||||
if hasattr(vcs_, "close"):
|
||||
vcs_.close()
|
||||
|
||||
except Exception as e:
|
||||
Actor.log.error(e)
|
||||
# I had to create a msg variable to avoid a ruff lint error S608 (SQL Injection)
|
||||
msg = (
|
||||
"Failed to update database. Please ensure the following:"
|
||||
"1. Database is configured properly."
|
||||
"2. The vector dimension of your embedding model in the Actor input (Embedding settings -> model) matches the one set up in the database."
|
||||
"Error message:"
|
||||
)
|
||||
await Actor.fail(status_message=f"{msg} {e}", exception=e)
|
||||
|
||||
|
||||
async def get_embeddings(actor_input: ActorInputsDb) -> Embeddings: # type: ignore[return]
|
||||
try:
|
||||
embed_provider_name = str(actor_input.embeddingsProvider)
|
||||
Actor.log.info("Get embeddings class: %s", embed_provider_name)
|
||||
embeddings = await get_embedding_provider(
|
||||
embed_provider_name,
|
||||
actor_input.embeddingsApiKey,
|
||||
actor_input.embeddingsConfig,
|
||||
)
|
||||
except Exception as e:
|
||||
Actor.log.error(e)
|
||||
await Actor.fail(status_message=f"Failed to get embeddings: {e}. Ensure that the configuration in the Embeddings Settings is correct.")
|
||||
else:
|
||||
return embeddings
|
||||
|
||||
|
||||
async def load_dataset(actor_input: ActorInputsDb, dataset_id: str) -> list[Document]: # type: ignore[return]
|
||||
"""Load dataset from the datasetId and extract fields from the dataset."""
|
||||
|
||||
# Add parameters related to chunking to every dataset item to be able to update DB when chunkSize, chunkOverlap or performChunking changes
|
||||
meta_object = actor_input.metadataObject or {}
|
||||
meta_object.update({"chunkSize": actor_input.chunkSize, "chunkOverlap": actor_input.chunkOverlap, "performChunking": actor_input.performChunking})
|
||||
|
||||
# Required for checksum calculation
|
||||
# Update metadata fields with datasetFieldsToItemId for dataset loading
|
||||
meta_fields = actor_input.metadataDatasetFields or {}
|
||||
meta_fields.update({k: k for k in actor_input.dataUpdatesPrimaryDatasetFields or []})
|
||||
Actor.log.info("Load Dataset ID %s and extract fields %s", dataset_id, actor_input.datasetFields)
|
||||
|
||||
try:
|
||||
dataset_loader = get_dataset_loader(
|
||||
str(dataset_id),
|
||||
fields=actor_input.datasetFields,
|
||||
meta_object=meta_object,
|
||||
meta_fields=meta_fields,
|
||||
)
|
||||
documents = dataset_loader.load()
|
||||
documents = [doc for doc in documents if doc.page_content]
|
||||
Actor.log.info("Dataset loaded, number of documents: %s", len(documents))
|
||||
|
||||
except Exception as e:
|
||||
Actor.log.error(e)
|
||||
await Actor.fail(
|
||||
status_message=f"Failed to load datasetId {dataset_id} due to error: {e}. Ensure the following: "
|
||||
f"1. If running this Actor standalone, the dataset should exist. "
|
||||
f"2. If this Actor is configured with another Actor (in the integration section), the `datasetId` should be correctly passed. "
|
||||
f"3. If the problem persists, consider creating an issue."
|
||||
)
|
||||
else:
|
||||
return documents
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
# __init__.py
|
||||
from .opengauss_input_model import OpengaussIntegration
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
# generated by datamodel-codegen:
|
||||
# filename: input_schema.json
|
||||
# timestamp: 2025-09-19T03:51:44+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing_extensions import Literal
|
||||
|
||||
|
||||
class OpengaussIntegration(BaseModel):
|
||||
opengaussHost: str = Field(
|
||||
..., description='The Host of openGauss', title='openGauss Host'
|
||||
)
|
||||
opengaussPort: str = Field(
|
||||
..., description='The Port of openGauss', title='openGauss Port'
|
||||
)
|
||||
opengaussUser: str = Field(
|
||||
..., description='The User of openGauss', title='openGauss User'
|
||||
)
|
||||
opengaussPassword: str = Field(
|
||||
..., description='The Password of openGauss', title='openGauss Password'
|
||||
)
|
||||
opengaussDBname: str = Field(
|
||||
..., description='The DBname of openGauss', title='openGauss DBname'
|
||||
)
|
||||
opengaussTableName: str = Field(
|
||||
...,
|
||||
description='The name of the table to use',
|
||||
title='openGauss SQL table name',
|
||||
)
|
||||
embeddingsProvider: Literal['OpenAI', 'Cohere'] = Field(
|
||||
...,
|
||||
description='Choose the embeddings provider to use for generating embeddings',
|
||||
title='Embeddings provider (as defined in the langchain API)',
|
||||
)
|
||||
embeddingsConfig: Optional[Dict[str, Any]] = Field(
|
||||
None,
|
||||
description='Configure the parameters for the LangChain embedding class. Key points to consider:\n\n1. Typically, you only need to specify the model name. For example, for OpenAI, set the model name as {"model": "text-embedding-3-small"}.\n\n2. It\'s required to ensure that the vector size of your embeddings matches the size of embeddings in the database.\n\n3. Here are examples of embedding models:\n - [OpenAI](https://platform.openai.com/docs/guides/embeddings): `text-embedding-3-small`, `text-embedding-3-large`, etc.\n - [Cohere](https://docs.cohere.com/docs/cohere-embed): `embed-english-v3.0`, `embed-multilingual-light-v3.0`, etc.\n\n4. For more details about other parameters, refer to the [LangChain documentation](https://python.langchain.com/docs/integrations/text_embedding/).',
|
||||
title='Configuration for embeddings provider',
|
||||
)
|
||||
embeddingsApiKey: str = Field(
|
||||
...,
|
||||
description='Value of the API KEY for the embeddings provider (if required).\n\n For example for OpenAI it is OPENAI_API_KEY, for Cohere it is COHERE_API_KEY)',
|
||||
title='Embeddings API KEY (whenever applicable, depends on provider)',
|
||||
)
|
||||
datasetFields: List = Field(
|
||||
...,
|
||||
description='This array specifies the dataset fields to be selected and stored in the vector store. Only the fields listed here will be included in the vector store.\n\nFor instance, when using the Website Content Crawler, you might choose to include fields such as `text`, `url`, and `metadata.title` in the vector store.',
|
||||
title='Dataset fields to select from the dataset results and store in the database',
|
||||
)
|
||||
metadataDatasetFields: Optional[Dict[str, Any]] = Field(
|
||||
None,
|
||||
description='A list of dataset fields which should be selected from the dataset and stored as metadata in the vector stores.\n\nFor example, when using the Website Content Crawler, you might want to store `url` in metadata. In this case, use `metadataDatasetFields parameter as follows {"url": "url"}`',
|
||||
title='Dataset fields to select from the dataset and store as metadata in the database',
|
||||
)
|
||||
metadataObject: Optional[Dict[str, Any]] = Field(
|
||||
None,
|
||||
description='This object allows you to store custom metadata for every item in the vector store.\n\nFor example, if you want to store the `domain` as metadata, use the `metadataObject` like this: {"domain": "apify.com"}.',
|
||||
title='Custom object to be stored as metadata in the vector store database',
|
||||
)
|
||||
datasetId: Optional[str] = Field(
|
||||
None,
|
||||
description='Dataset ID (when running standalone without integration)',
|
||||
title='Dataset ID',
|
||||
)
|
||||
dataUpdatesStrategy: Optional[Literal['add', 'upsert', 'deltaUpdates']] = Field(
|
||||
'deltaUpdates',
|
||||
description="Choose the update strategy for the integration. The update strategy determines how the integration updates the data in the database.\n\nThe available options are:\n\n- **Add data** (`add`):\n - Always adds new records to the database.\n - No checks for existing records or updates are performed.\n - Useful when appending data without concern for duplicates.\n\n- **Upsert data** (`upsert`):\n - Updates existing records if they match a key or identifier.\n - Inserts new records into the database if they don't already exist.\n - Ideal for ensuring the database contains the most up-to-date data, avoiding duplicates.\n\n- **Update changed data based on deltas** (`deltaUpdates`):\n - Performs incremental updates by identifying differences (deltas) between the new dataset and the existing records.\n - Only adds new records and updates those that have changed.\n - Unchanged records are left untouched.\n - Maximizes efficiency by reducing unnecessary updates.\n\nSelect the strategy that best fits your use case.",
|
||||
title='Update strategy (add, upsert, deltaUpdates (default))',
|
||||
)
|
||||
dataUpdatesPrimaryDatasetFields: Optional[List] = Field(
|
||||
['url'],
|
||||
description='This array contains fields that are used to uniquely identify dataset items, which helps to handle content changes across different runs.\n\nFor instance, in a web content crawling scenario, the `url` field could serve as a unique identifier for each item.',
|
||||
title='Dataset fields to uniquely identify dataset items (only relevant when dataUpdatesStrategy is `upsert` or `deltaUpdates`)',
|
||||
)
|
||||
enableDeltaUpdates: Optional[bool] = Field(
|
||||
True,
|
||||
description='When set to true, this setting enables incremental updates for objects in the database by comparing the changes (deltas) between the crawled dataset items and the existing objects, uniquely identified by the `datasetKeysToItemId` field.\n\n The integration will only add new objects and update those that have changed, reducing unnecessary updates. The `datasetFields`, `metadataDatasetFields`, and `metadataObject` fields are used to determine the changes.',
|
||||
title='Enable incremental updates for objects based on deltas (deprecated)',
|
||||
)
|
||||
deltaUpdatesPrimaryDatasetFields: Optional[List] = Field(
|
||||
['url'],
|
||||
description='This array contains fields that are used to uniquely identify dataset items, which helps to handle content changes across different runs.\n\nFor instance, in a web content crawling scenario, the `url` field could serve as a unique identifier for each item.',
|
||||
title='Dataset fields to uniquely identify dataset items (only relevant when `enableDeltaUpdates` is enabled) (deprecated)',
|
||||
)
|
||||
deleteExpiredObjects: Optional[bool] = Field(
|
||||
True,
|
||||
description='When set to true, delete objects from the database that have not been crawled for a specified period.',
|
||||
title='Delete expired objects from the database',
|
||||
)
|
||||
expiredObjectDeletionPeriodDays: Optional[int] = Field(
|
||||
30,
|
||||
description='This setting allows the integration to manage the deletion of objects from the database that have not been crawled for a specified period. It is typically used in subsequent runs after the initial crawl.\n\nWhen the value is greater than 0, the integration checks if objects have been seen within the last X days (determined by the expiration period). If the objects are expired, they are deleted from the database. The specific value for `deletedExpiredObjectsDays` depends on your use case and how frequently you crawl data.\n\nFor example, if you crawl data daily, you can set `deletedExpiredObjectsDays` to 7 days. If you crawl data weekly, you can set `deletedExpiredObjectsDays` to 30 days.',
|
||||
ge=0,
|
||||
title='Delete expired objects from the database after a specified number of days',
|
||||
)
|
||||
performChunking: Optional[bool] = Field(
|
||||
True,
|
||||
description='When set to true, the text will be divided into smaller chunks based on the settings provided below. Proper chunking helps optimize retrieval and ensures accurate and efficient responses.',
|
||||
title='Enable text chunking',
|
||||
)
|
||||
chunkSize: Optional[int] = Field(
|
||||
2000,
|
||||
description='Defines the maximum number of characters in each text chunk. Choosing the right size balances between detailed context and system performance. Optimal sizes ensure high relevancy and minimal response time.',
|
||||
ge=1,
|
||||
title='Maximum chunk size',
|
||||
)
|
||||
chunkOverlap: Optional[int] = Field(
|
||||
0,
|
||||
description='Specifies the number of overlapping characters between consecutive text chunks. Adjusting this helps maintain context across chunks, which is crucial for accuracy in retrieval-augmented generation systems.',
|
||||
ge=0,
|
||||
title='Chunk overlap',
|
||||
)
|
||||
|
|
@ -1,159 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_apify import ApifyDatasetLoader
|
||||
from langchain_core.documents import Document
|
||||
|
||||
EXCLUDE_KEYS_FROM_CHECKSUM = {"metadata": {"chunk_id", "id", "checksum", "last_seen_at", "item_id"}}
|
||||
DAY_IN_SECONDS = 24 * 3600
|
||||
|
||||
logger = logging.getLogger("apify")
|
||||
|
||||
|
||||
def get_nested_value(d: dict, keys: str) -> Any:
|
||||
"""
|
||||
Extract nested value from dict.
|
||||
|
||||
Example:
|
||||
>>> get_nested_value({"a": "v1", "c1": {"c2": "v2"}}, "c1.c2")
|
||||
'v2'
|
||||
"""
|
||||
|
||||
d = copy.deepcopy(d)
|
||||
for key in keys.split("."):
|
||||
if d and isinstance(d, dict) and d.get(key):
|
||||
d = d[key]
|
||||
else:
|
||||
return ""
|
||||
return d
|
||||
|
||||
|
||||
def stringify_dict(d: dict, keys: list[str]) -> str:
|
||||
"""Stringify all values in a dictionary.
|
||||
|
||||
Example:
|
||||
>>> d_ = {"a": {"text": "Apify is cool"}, "description": "Apify platform"}
|
||||
>>> stringify_dict(d_, ["a.text", "description"])
|
||||
'a.text: Apify is cool\\ndescription: Apify platform'
|
||||
"""
|
||||
return "\n".join([f"{key}: {value}" for key in keys if (value := get_nested_value(d, key))])
|
||||
|
||||
|
||||
def get_dataset_loader(dataset_id: str, fields: list[str], meta_object: dict, meta_fields: dict) -> ApifyDatasetLoader:
|
||||
"""Load dataset by dataset_id using ApifyDatasetLoader.
|
||||
|
||||
The dataset_mapping_function is used to map the dataset item to a Document object.
|
||||
Stringify dict using the fields.
|
||||
"""
|
||||
|
||||
return ApifyDatasetLoader(
|
||||
dataset_id,
|
||||
dataset_mapping_function=lambda dataset_item: Document(
|
||||
page_content=stringify_dict(dataset_item, fields) or "",
|
||||
metadata={
|
||||
**meta_object,
|
||||
**{key: get_nested_value(dataset_item, value) for key, value in meta_fields.items()},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def compute_hash(text: str) -> str:
|
||||
"""Compute hash of the text."""
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def get_chunks_to_delete(chunks_prev: list[Document], chunks_current: list[Document], expired_days: float) -> tuple[list[Document], list[Document]]:
|
||||
"""
|
||||
Identifies chunks to be deleted based on their last seen timestamp and presence in the current run.
|
||||
|
||||
Compare the chunks from the previous and current runs and identify chunks that are not present
|
||||
in the current run and have not been updated within the specified 'expired_days'. These chunks are marked for deletion.
|
||||
"""
|
||||
ids_current = {d.metadata["item_id"] for d in chunks_current}
|
||||
|
||||
ts_expired = int(datetime.now(timezone.utc).timestamp() - expired_days * DAY_IN_SECONDS)
|
||||
chunks_expired_delete, chunks_old_keep = [], []
|
||||
|
||||
# chunks that have been crawled in the current run and are older than ts_expired => to delete
|
||||
for d in chunks_prev:
|
||||
if d.metadata["item_id"] not in ids_current:
|
||||
if d.metadata["last_seen_at"] < ts_expired:
|
||||
chunks_expired_delete.append(d)
|
||||
else:
|
||||
chunks_old_keep.append(d)
|
||||
|
||||
return chunks_expired_delete, chunks_old_keep
|
||||
|
||||
|
||||
def get_chunks_to_update(chunks_prev: list[Document], chunks_current: list[Document]) -> tuple[list[Document], list[Document]]:
|
||||
"""
|
||||
Identifies chunks that need to be updated or added based on their unique identifiers and checksums.
|
||||
|
||||
Compare the chunks from the previous and current runs and identify chunks that are new or have
|
||||
undergone content changes by comparing their checksums. These chunks are marked for addition. chunks that are
|
||||
present in both runs but have not undergone content changes are marked for metadata update.
|
||||
"""
|
||||
|
||||
prev_id_checksum = defaultdict(list)
|
||||
for chunk in chunks_prev:
|
||||
prev_id_checksum[chunk.metadata["item_id"]].append(chunk.metadata["checksum"])
|
||||
|
||||
chunks_add = []
|
||||
chunks_update_metadata = []
|
||||
for chunk in chunks_current:
|
||||
if chunk.metadata["item_id"] in prev_id_checksum:
|
||||
if chunk.metadata["checksum"] in prev_id_checksum[chunk.metadata["item_id"]]:
|
||||
chunks_update_metadata.append(chunk)
|
||||
else:
|
||||
chunks_add.append(chunk)
|
||||
else:
|
||||
chunks_add.append(chunk)
|
||||
|
||||
return chunks_add, chunks_update_metadata
|
||||
|
||||
|
||||
def add_item_last_seen_at(items: list[Document]) -> list[Document]:
|
||||
"""Add last_seen_at timestamp to the metadata of each dataset item."""
|
||||
for item in items:
|
||||
item.metadata["last_seen_at"] = int(datetime.now(timezone.utc).timestamp())
|
||||
return items
|
||||
|
||||
|
||||
def add_item_checksum(items: list[Document], dataset_fields_to_item_id: list[str]) -> list[Document]:
|
||||
"""
|
||||
Adds a checksum and unique item_id to the metadata of each dataset item.
|
||||
|
||||
This function computes a checksum for each item based on its content and metadata, excluding certain keys.
|
||||
The checksum is then added to the document's metadata. Additionally, a unique item ID is generated based on
|
||||
specified keys in the document's metadata and added to the metadata as well.
|
||||
"""
|
||||
for item in items:
|
||||
item.metadata["checksum"] = compute_hash(item.json(exclude=EXCLUDE_KEYS_FROM_CHECKSUM))
|
||||
hash_str = "".join([str(item.metadata[key]) for key in dataset_fields_to_item_id])
|
||||
item.metadata["item_id"] = compute_hash(hash_str)
|
||||
if not hash_str:
|
||||
logger.warning(
|
||||
"Item_id %s was generated with an empty hash. This typically means that `dataUpdatesPrimaryDatasetFields` "
|
||||
"are empty or non-existent.",
|
||||
item.metadata["item_id"],
|
||||
)
|
||||
|
||||
return add_item_last_seen_at(items)
|
||||
|
||||
|
||||
def add_chunk_id(chunks: list[Document]) -> list[Document]:
|
||||
"""For every chunk (document stored in vector db) add chunk_id to metadata.
|
||||
|
||||
The chunk_id is a unique identifier for each chunk and is not required, but it is better to keep it in metadata.
|
||||
"""
|
||||
for d in chunks:
|
||||
d.metadata["chunk_id"] = d.metadata.get("chunk_id", str(uuid4()))
|
||||
return chunks
|
||||
|
|
@ -1,177 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from apify import Actor
|
||||
from langchain_core.documents import Document
|
||||
from langchain_core.vectorstores import VectorStore
|
||||
|
||||
from .models import OpengaussIntegration
|
||||
from .utils import get_chunks_to_delete, get_chunks_to_update
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langchain.vectorstores import VectorStore
|
||||
from langchain_core.embeddings import Embeddings
|
||||
|
||||
from ._types import ActorInputsDb, VectorDb
|
||||
|
||||
|
||||
async def get_vector_database(actor_input: ActorInputsDb | None, embeddings: Embeddings) -> VectorDb:
|
||||
"""Get database based on the integration type."""
|
||||
|
||||
if isinstance(actor_input, OpengaussIntegration):
|
||||
from .vector_stores.opengauss import OpenGaussDatabase
|
||||
|
||||
return OpenGaussDatabase(actor_input, embeddings)
|
||||
|
||||
raise ValueError("Unknown integration type")
|
||||
|
||||
|
||||
def update_db_with_crawled_data(vector_store: VectorDb, documents: list[Document]) -> None:
|
||||
"""Update the database with new crawled data."""
|
||||
|
||||
Actor.log.info("Comparing crawled data with the database ...")
|
||||
data_add, ids_update_last_seen, ids_del = compare_crawled_data_with_db(vector_store, documents)
|
||||
Actor.log.info("Objects: to add: %s, to update last_seen_at: %s, to delete: %s", len(data_add), len(ids_update_last_seen), len(ids_del))
|
||||
|
||||
# Delete data that were updated
|
||||
if ids_del:
|
||||
vector_store.delete(ids_del)
|
||||
Actor.log.info("Deleted %s objects from the vector store where the content has changed since the last update", len(ids_del))
|
||||
|
||||
# Add new data
|
||||
if data_add:
|
||||
Actor.log.info("Adding %s new objects to the vector store", len(data_add))
|
||||
vector_store.add_documents(data_add, ids=[d.metadata["chunk_id"] for d in data_add])
|
||||
Actor.log.info("Added %s new objects to the vector store", len(data_add))
|
||||
|
||||
# Update metadata data
|
||||
if ids_update_last_seen:
|
||||
vector_store.update_last_seen_at(ids_update_last_seen)
|
||||
Actor.log.info("Updated last_seen_at metadata for %s objects", len(ids_update_last_seen))
|
||||
|
||||
|
||||
def upsert_db_with_crawled_data(vector_store: VectorDb, documents: list[Document]) -> None:
|
||||
"""Upsert crawled data into the database by first deleting all documents and then adding all the documents."""
|
||||
Actor.log.info("Upsert crawled data into database")
|
||||
Actor.log.info("Delete documents by item_id. This might take a while as documents are deleted one by one.")
|
||||
for d in documents:
|
||||
vector_store.delete_by_item_id(d.metadata["item_id"])
|
||||
Actor.log.info("Delete documents by item_id. Done")
|
||||
|
||||
Actor.log.info("Add documents")
|
||||
vector_store.add_documents(documents, ids=[d.metadata["chunk_id"] for d in documents])
|
||||
Actor.log.info("Added %s new objects to the vector store", len(documents))
|
||||
|
||||
|
||||
def delete_expired_objects(vector_store: VectorDb, timestamp_expired: int) -> None:
|
||||
"""Delete expired objects from the database."""
|
||||
|
||||
if timestamp_expired:
|
||||
dt = datetime.datetime.fromtimestamp(timestamp_expired, tz=datetime.timezone.utc)
|
||||
Actor.log.info("About to delete objects from the database that were not seen since %s (timestamp: %s)", dt, timestamp_expired)
|
||||
vector_store.delete_expired(timestamp_expired)
|
||||
|
||||
|
||||
def get_items_ids_from_db(vector_store: VectorDb, data: list[Document]) -> dict[str, list[Document]]:
|
||||
"""Get documents from the database by item_id."""
|
||||
|
||||
items_ids = {d.metadata["item_id"] for d in data}
|
||||
|
||||
def _get_item_id(item_id: str) -> tuple[str, list[Document]]:
|
||||
return item_id, vector_store.get_by_item_id(item_id)
|
||||
|
||||
crawled_db = defaultdict(list)
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future_to_item_id = {executor.submit(_get_item_id, item_id): item_id for item_id in items_ids}
|
||||
|
||||
for k, future in enumerate(concurrent.futures.as_completed(future_to_item_id)):
|
||||
item_id = future_to_item_id[future]
|
||||
if k % 1000 == 0:
|
||||
Actor.log.info("Processing item_id %s (%d/%d) to compare crawled data with the database", item_id, k, len(items_ids))
|
||||
try:
|
||||
item_id, documents = future.result()
|
||||
crawled_db[item_id].extend(documents)
|
||||
except Exception as e:
|
||||
Actor.log.error("Item_id %s generated an error", item_id, e)
|
||||
|
||||
return dict(crawled_db)
|
||||
|
||||
|
||||
def compare_crawled_data_with_db(vector_store: VectorDb, data: list[Document]) -> tuple[list[Document], list[str], list[str]]:
|
||||
"""Compare current crawled data with the data in the database. Return data to add, delete and update.
|
||||
|
||||
New data is added
|
||||
Data that was not changed -> update metadata last_seen_at
|
||||
Data that was changed -> delete and add new
|
||||
"""
|
||||
data_add = []
|
||||
ids_delete: set[str] = set()
|
||||
ids_update_last_seen: set[str] = set()
|
||||
|
||||
if hasattr(vector_store, "count") and vector_store.count() == 0:
|
||||
return data, [], []
|
||||
|
||||
crawled_db = get_items_ids_from_db(vector_store, data)
|
||||
|
||||
for d in data:
|
||||
if res := crawled_db.get(d.metadata["item_id"]):
|
||||
if d.metadata["checksum"] in {r.metadata["checksum"] for r in res}:
|
||||
# Because of weaviate database, we need to use chunk_id instead of id
|
||||
ids_update_last_seen.update({r.metadata.get("id") or r.metadata.get("chunk_id", ""): r for r in res})
|
||||
else:
|
||||
ids_delete.update({r.metadata.get("id") or r.metadata.get("chunk_id", ""): r for r in res})
|
||||
data_add.append(d)
|
||||
else:
|
||||
data_add.append(d)
|
||||
|
||||
return data_add, list(ids_update_last_seen), list(ids_delete)
|
||||
|
||||
|
||||
async def update_db_with_crawled_data_using_internal_cache(
|
||||
vector_store: VectorStore, documents: list[Document], cache_key_name: str, cache_kv_store_name: str, expired_days: float
|
||||
) -> None:
|
||||
"""
|
||||
Updates the vector store with new documents and removes outdated ones.
|
||||
|
||||
This function uses Apify's key-value store to handle documents. Each document, along with its metadata,
|
||||
is hashed and stored as a key in the key-value store.
|
||||
|
||||
The function performs a comparison between the current set of documents and the set from the previous runs.
|
||||
It identifies new documents and those no longer present. New documents are added to the vector store, while
|
||||
documents older than the specified 'expired_days' are removed.
|
||||
"""
|
||||
|
||||
Actor.log.info("Load previous cache %s from the key-value store: %s", cache_key_name, cache_kv_store_name)
|
||||
|
||||
kv_store = await Actor.open_key_value_store(name=cache_kv_store_name)
|
||||
previous_runs = await kv_store.get_value(cache_key_name) or {}
|
||||
previous_runs = [Document.parse_obj(doc) for doc in previous_runs]
|
||||
Actor.log.info("Previous runs contains: %s records", len(previous_runs))
|
||||
|
||||
chunks_to_add, chunks_to_update = get_chunks_to_update(previous_runs, documents)
|
||||
chunks_to_delete, chunks_old_keep = get_chunks_to_delete(previous_runs, documents, expired_days=expired_days)
|
||||
|
||||
Actor.log.info("Chunks to add: %s, chunks to update last_seen metadata: %s", len(chunks_to_add), len(chunks_to_update))
|
||||
Actor.log.info("Chunks to delete: %s", len(chunks_to_delete))
|
||||
|
||||
if chunks_to_delete:
|
||||
vector_store.delete(ids=[x.metadata["id"] for x in chunks_to_delete if x.metadata["id"]])
|
||||
Actor.log.info("Deleted %s from database", len(chunks_to_delete))
|
||||
|
||||
if chunks_to_add:
|
||||
inserted = vector_store.add_documents(chunks_to_add, ids=[x.metadata["id"] for x in chunks_to_add if x.metadata["id"]])
|
||||
Actor.log.info("Added %s documents to the vector store", len(inserted))
|
||||
else:
|
||||
Actor.log.info("No new documents to add")
|
||||
|
||||
# update cache
|
||||
if current_cache := chunks_to_add + chunks_to_update + chunks_old_keep:
|
||||
await kv_store.set_value(cache_key_name, current_cache)
|
||||
Actor.log.info("Updated cache: %s in the key-value store with %s entries", cache_key_name, len(current_cache))
|
||||
|
||||
Actor.log.info("Push chunked data to the unnamed output dataset")
|
||||
await Actor.push_data([doc.dict() for doc in current_cache])
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .opengauss import OpenGaussDatabase
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue