96 lines
2.5 KiB
SQL
96 lines
2.5 KiB
SQL
|
|
-- list markdown files and vectorize content
|
|
select filename, md_clob from markdown_files;
|
|
|
|
-- add embedding column
|
|
alter table markdown_files add embedding VECTOR(768, FLOAT32);
|
|
|
|
-- generate embeddings
|
|
update markdown_files
|
|
set embedding = VECTOR_EMBEDDING(
|
|
MULTILINGUAL_E5_BASE
|
|
USING md_clob as DATA);
|
|
|
|
commit;
|
|
|
|
select substr(md_clob, 50, 60) as md_preview, embedding from markdown_files;
|
|
|
|
|
|
|
|
-- test query
|
|
select
|
|
id,
|
|
VECTOR_DISTANCE(embedding,
|
|
VECTOR_EMBEDDING(
|
|
MULTILINGUAL_E5_BASE
|
|
USING 'tarjeta de crédito' as DATA)
|
|
) as distance
|
|
from markdown_files;
|
|
|
|
|
|
|
|
-- crear chunks y embeddings con DBMS_VECTOR_CHAIN
|
|
drop table if exists markdown_chunks;
|
|
|
|
CREATE TABLE markdown_chunks AS
|
|
SELECT
|
|
m.id as doc_id,
|
|
m.filename as file_name,
|
|
et.embed_id as chunk_id,
|
|
et.embed_data as chunk_text,
|
|
TO_VECTOR(et.embed_vector) as embedding
|
|
FROM markdown_files m,
|
|
DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS(
|
|
-- primero dividimos el CLOB en partes pequenas (chunks)
|
|
DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS(
|
|
m.md_clob,
|
|
JSON('{
|
|
"by" : "words",
|
|
"max" : "120",
|
|
"overlap" : "20",
|
|
"split" : "recursively",
|
|
"language" : "spanish",
|
|
"normalize" : "all"
|
|
}')
|
|
),
|
|
-- despues generamos el embedding de cada chunk
|
|
JSON('{
|
|
"provider" : "database",
|
|
"model" : "MULTILINGUAL_E5_BASE"
|
|
}')
|
|
) t,
|
|
JSON_TABLE(
|
|
t.column_value,
|
|
'$[*]' COLUMNS (
|
|
embed_id NUMBER PATH '$.embed_id',
|
|
embed_data VARCHAR2(4000) PATH '$.embed_data',
|
|
embed_vector CLOB PATH '$.embed_vector'
|
|
)
|
|
) et;
|
|
|
|
|
|
-- ver los chunks generados
|
|
select doc_id, file_name, chunk_id, chunk_text
|
|
from markdown_chunks
|
|
order by doc_id, chunk_id;
|
|
|
|
|
|
|
|
-- buscar chunks por similaridad semantica
|
|
select
|
|
doc_id,
|
|
chunk_id,
|
|
VECTOR_DISTANCE(embedding,
|
|
VECTOR_EMBEDDING(MULTILINGUAL_E5_BASE
|
|
USING 'tarjeta de crédito' as DATA
|
|
)
|
|
) as distance,
|
|
chunk_text
|
|
from markdown_chunks
|
|
where doc_id = 1
|
|
order by distance
|
|
fetch first 3 rows only;
|
|
|
|
|
|
|