fernando.melo/itti_adb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add labs

Browse Source
This commit is contained in:
Fernando Melo
2026-05-27 19:51:52 -03:00
parent dbef87dd0e
commit 440b1cfadb
18 changed files with 1687 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
lab/vector/01_load_markdown.sql Normal file
View File

@@ -0,0 +1,57 @@
set serveroutput on
-- list markdown files in object storage
SELECT object_name, bytes, checksum, created, last_modified
FROM DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => 'https://objectstorage.us-chicago-1.oraclecloud.com/n/idi1o0a010nx/b/md-processed/o/'
);
-- create table and load markdown content into the database
DROP TABLE IF EXISTS markdown_files;
CREATE TABLE IF NOT EXISTS markdown_files (
id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
source_uri VARCHAR2(1000) NOT NULL,
filename VARCHAR2(500) NOT NULL,
file_hash VARCHAR2(128),
file_size NUMBER,
md_clob CLOB,
status VARCHAR2(30) DEFAULT 'PENDING',
ingested_at TIMESTAMP DEFAULT SYSTIMESTAMP,
error_message VARCHAR2(4000)
);
--
DECLARE
l_clob CLOB;
l_blob BLOB;
l_uri VARCHAR2(1000) := 'https://objectstorage.us-chicago-1.oraclecloud.com/n/idi1o0a010nx/b/md-processed/o/';
l_filename VARCHAR2(500) := 'Banco Atlas - Beneficios Estaciones de Servicio.md';
BEGIN
l_clob := blob_to_clob(DBMS_CLOUD.GET_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => l_uri || l_filename));
INSERT INTO markdown_files (source_uri, filename, file_size, md_clob, file_hash, status)
VALUES (l_uri || l_filename,
l_filename,
DBMS_LOB.GETLENGTH(l_clob),
l_clob,
DBMS_CRYPTO.HASH(l_clob, DBMS_CRYPTO.HASH_SH256),
'INGESTED'
);
COMMIT;
DBMS_OUTPUT.PUT_LINE('Markdown file loaded. Size: ' || DBMS_LOB.GETLENGTH(l_clob) || ' bytes');
END;
/
-- test
SELECT * FROM markdown_files;

42
lab/vector/02_load_embedding.sql Normal file
View File

@@ -0,0 +1,42 @@
exec DBMS_VECTOR.DROP_ONNX_MODEL('MULTILINGUAL_E5_BASE');
-- list onnx models in object storage
SELECT object_name, bytes, checksum, created, last_modified
FROM DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => 'https://idi1o0a010nx.objectstorage.us-chicago-1.oci.customer-oci.com/n/idi1o0a010nx/b/uploads/o/'
);
-- load onnx model into vector database
declare
model_source blob := NULL;
model_uri varchar2(1000) := 'https://idi1o0a010nx.objectstorage.us-chicago-1.oci.customer-oci.com/n/idi1o0a010nx/b/uploads/o/multilingual-e5-base.onnx';
begin
model_source := DBMS_CLOUD.GET_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => model_uri);
DBMS_VECTOR.LOAD_ONNX_MODEL(
'MULTILINGUAL_E5_BASE', -- 768 dimensions
model_source,
metadata => JSON('{"function" : "embedding", "embeddingOutput" : "embedding", "input": {"input": ["DATA"]}}')
);
END;
/
-- test using onnx model
SELECT VECTOR_EMBEDDING(
MULTILINGUAL_E5_BASE
USING 'la veloce volpe marrone saltò' as DATA)
AS embedding;
-- test using external provider
SELECT DBMS_VECTOR.UTL_TO_EMBEDDING(
'la veloce volpe marrone saltò',
JSON('{
"provider" : "ocigenai",
"credential_name" : "OCI$RESOURCE_PRINCIPAL",
"url" : "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText",
"compartmentId" : "ocid1.compartment.oc1..aaaaaaaa33ogmhasyvcqzuvkrfzo5mavh3q2le7mgwmy74yzpyqi7byxgrlq",
"model" : "openai.text-embedding-3-large"
}')
) AS emb;

96
lab/vector/03_vectorizing.sql Normal file
View File

@@ -0,0 +1,96 @@
-- list markdown files and vectorize content
select filename, md_clob from markdown_files;
-- add embedding column
alter table markdown_files add embedding VECTOR(768, FLOAT32);
-- generate embeddings
update markdown_files
set embedding = VECTOR_EMBEDDING(
MULTILINGUAL_E5_BASE
USING md_clob as DATA);
commit;
select md_clob, embedding from markdown_files;
-- test query
select
id,
VECTOR_DISTANCE(embedding,
VECTOR_EMBEDDING(
MULTILINGUAL_E5_BASE
USING 'estaciones de servicio' as DATA)
) as distance
from markdown_files;
-- crear chunks y embeddings con DBMS_VECTOR_CHAIN
drop table if exists markdown_chunks;
CREATE TABLE markdown_chunks AS
SELECT
m.id as doc_id,
m.filename as file_name,
et.embed_id as chunk_id,
et.embed_data as chunk_text,
TO_VECTOR(et.embed_vector) as embedding
FROM markdown_files m,
DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS(
-- primero dividimos el CLOB en partes pequenas (chunks)
DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS(
m.md_clob,
JSON('{
"by" : "words",
"max" : "120",
"overlap" : "20",
"split" : "recursively",
"language" : "spanish",
"normalize" : "all"
}')
),
-- despues generamos el embedding de cada chunk
JSON('{
"provider" : "database",
"model" : "MULTILINGUAL_E5_BASE"
}')
) t,
JSON_TABLE(
t.column_value,
'$[*]' COLUMNS (
embed_id NUMBER PATH '$.embed_id',
embed_data VARCHAR2(4000) PATH '$.embed_data',
embed_vector CLOB PATH '$.embed_vector'
)
) et;
commit;
-- ver los chunks generados
select doc_id, file_name, chunk_id, chunk_text
from markdown_chunks
order by doc_id, chunk_id;
-- buscar chunks por similaridad semantica
select
doc_id,
chunk_id,
VECTOR_DISTANCE(embedding,
VECTOR_EMBEDDING(MULTILINGUAL_E5_BASE
USING 'estaciones de servicio' as DATA
)
) as distance,
chunk_text
from markdown_chunks
where doc_id = 1
order by distance
fetch first 3 rows only;

154
lab/vector/04_indexing.sql Normal file
View File

@@ -0,0 +1,154 @@
set long 999999
set pages 1000
set lines 200
/* --------------------------------
HNSW INDEX
-------------------------------- */
-- crear indice HNSW para busquedas vectoriales aproximadas
DROP INDEX IF EXISTS markdown_chunks_hnsw_idx;
CREATE VECTOR INDEX markdown_chunks_hnsw_idx
ON markdown_chunks (embedding)
ORGANIZATION INMEMORY NEIGHBOR GRAPH
DISTANCE COSINE
WITH TARGET ACCURACY 90
PARAMETERS (
TYPE HNSW,
NEIGHBORS 32,
EFCONSTRUCTION 200
);
/* --------------------------------
IVF INDEX
-------------------------------- */
-- crear indice IVF con DOC_ID incluido para filtros por documento
DROP INDEX IF EXISTS markdown_chunks_ivf_idx;
CREATE VECTOR INDEX markdown_chunks_ivf_idx
ON markdown_chunks (embedding)
INCLUDE (doc_id)
ORGANIZATION NEIGHBOR PARTITIONS
DISTANCE COSINE
WITH TARGET ACCURACY 90
PARAMETERS (
TYPE IVF,
NEIGHBOR PARTITIONS 4,
MIN_VECTORS_PER_PARTITION 1
);
/* --------------------------------
TEXT INDEX + VECTOR SEARCH
-------------------------------- */
-- hybrid indexes / text indexes para busquedas textuales y combinadas
DROP INDEX IF EXISTS markdown_chunks_text_idx;
CREATE SEARCH INDEX markdown_chunks_text_idx
ON markdown_chunks (chunk_text)
FOR TEXT;
select
doc_id,
chunk_id,
SCORE(1) as text_score,
VECTOR_DISTANCE(embedding,
VECTOR_EMBEDDING(MULTILINGUAL_E5_BASE
USING 'estaciones de servicio' as DATA
)
) as vector_distance,
chunk_text
from markdown_chunks
where contains(chunk_text, 'Beneficio
AND FUZZY(Benefcio)
AND ABOUT(estaciones)
AND (Visa accum Mastercard)
AND (Clásica OR Oro)
AND (crédito NOT débito)
AND NEAR((POS, Infonet), 5)
AND NEAR((App, Premmia, Petrobras), 1)
AND (consumo AND personal)', 1) > 0
order by vector_distance
fetch first 3 rows only;
/* --------------------------------
HYBRID VECTOR INDEX
-------------------------------- */
-- crear indice hibrido para combinar busqueda textual y semantica
DROP INDEX IF EXISTS markdown_chunks_hybrid_idx;
CREATE HYBRID VECTOR INDEX markdown_chunks_hybrid_idx
ON markdown_chunks (chunk_text)
PARAMETERS ('MODEL MULTILINGUAL_E5_BASE
VECTOR_IDXTYPE HNSW
MEMORY 128M')
PARALLEL 2;
-- ejemplo de consulta usando el indice hibrido
SELECT JSON_SERIALIZE(
DBMS_HYBRID_VECTOR.SEARCH(
JSON('{
"hybrid_index_name" : "markdown_chunks_hybrid_idx",
"search_fusion" : "UNION",
"vector" : {
"search_text" : "beneficios en estaciones de servicio",
"search_mode" : "CHUNK"
},
"text" : {
"contains" : "estaciones AND Petrobras"
},
"return" : {
"values" : [
"chunk_id",
"chunk_text",
"score",
"vector_score",
"text_score"
],
"topN" : 5
}
}')
) RETURNING CLOB PRETTY
) AS hybrid_results;
SELECT jt.*
FROM
JSON_TABLE(
dbms_hybrid_vector.search(
json_object(
'hybrid_index_name' VALUE 'markdown_chunks_hybrid_idx',
'search_fusion' VALUE 'INTERSECT',
'search_scorer' VALUE 'rsf',
'vector' VALUE json_object('search_text' VALUE 'beneficios en estaciones de servicio'),
'text' VALUE json_object('contains' VALUE 'estaciones AND Petrobras'),
'return' VALUE json_object(
'values' VALUE json_array('rowid', 'score', 'vector_score', 'vector_rank', 'text_score', 'text_rank', 'chunk_text'),
'topN' VALUE 3
)
RETURNING JSON
)
),
'$[*]' COLUMNS idx for ORDINALITY,
score NUMBER PATH '$.score',
vector_score NUMBER PATH '$.vector_score',
vector_rank NUMBER PATH '$.vector_rank',
text_score NUMBER PATH '$.text_score',
text_rank NUMBER PATH '$.text_rank',
chunk_text VARCHAR2(4000) PATH '$.chunk_text'
) jt;