add labs

lab/vector/03_vectorizing.sql

@@ -0,0 +1,96 @@
+
+-- list markdown files and vectorize content
+select filename, md_clob from markdown_files;
+
+-- add embedding column
+alter table markdown_files add embedding VECTOR(768, FLOAT32);
+
+-- generate embeddings
+update markdown_files
+ set embedding = VECTOR_EMBEDDING(
+        MULTILINGUAL_E5_BASE 
+        USING md_clob as DATA);
+
+commit;
+
+select md_clob, embedding from markdown_files;
+
+
+
+-- test query
+select 
+    id,
+    VECTOR_DISTANCE(embedding,  
+                          VECTOR_EMBEDDING(
+                            MULTILINGUAL_E5_BASE 
+                            USING 'estaciones de servicio' as DATA)
+                        ) as distance 
+from markdown_files;
+
+
+
+-- crear chunks y embeddings con DBMS_VECTOR_CHAIN
+drop table if exists markdown_chunks;
+
+CREATE TABLE markdown_chunks AS
+SELECT
+    m.id                       as doc_id,
+    m.filename                 as file_name,
+    et.embed_id                as chunk_id,
+    et.embed_data              as chunk_text,
+    TO_VECTOR(et.embed_vector) as embedding
+FROM markdown_files m,
+    DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS(
+        -- primero dividimos el CLOB en partes pequenas (chunks)
+        DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS(
+            m.md_clob,
+            JSON('{
+                "by"        : "words",
+                "max"       : "120",
+                "overlap"   : "20",
+                "split"     : "recursively",
+                "language"  : "spanish",
+                "normalize" : "all"
+            }')
+        ),
+        -- despues generamos el embedding de cada chunk
+        JSON('{
+            "provider" : "database",
+            "model"    : "MULTILINGUAL_E5_BASE"
+        }')
+    ) t,
+    JSON_TABLE(
+        t.column_value,
+        '$[*]' COLUMNS (
+            embed_id     NUMBER         PATH '$.embed_id',
+            embed_data   VARCHAR2(4000) PATH '$.embed_data',
+            embed_vector CLOB           PATH '$.embed_vector'
+        )
+    ) et;
+
+commit;
+
+-- ver los chunks generados
+select doc_id, file_name, chunk_id, chunk_text
+from markdown_chunks
+order by doc_id, chunk_id;
+
+
+
+-- buscar chunks por similaridad semantica
+select
+    doc_id,
+    chunk_id,
+    VECTOR_DISTANCE(embedding, 
+                    VECTOR_EMBEDDING(MULTILINGUAL_E5_BASE 
+                                    USING 'estaciones de servicio' as DATA
+                                    )
+                    ) as distance,
+    chunk_text    
+from markdown_chunks
+where doc_id = 1 
+order by distance
+fetch first 3 rows only;
+
+
+