-- list markdown files and vectorize content
select filename, md_clob from markdown_files;

-- add embedding column
alter table markdown_files add embedding VECTOR(768, FLOAT32);

-- generate embeddings
update markdown_files
 set embedding = VECTOR_EMBEDDING(
        MULTILINGUAL_E5_BASE 
        USING md_clob as DATA);

commit;

select md_clob, embedding from markdown_files;



-- test query
select 
    id,
    VECTOR_DISTANCE(embedding,  
                          VECTOR_EMBEDDING(
                            MULTILINGUAL_E5_BASE 
                            USING 'estaciones de servicio' as DATA)
                        ) as distance 
from markdown_files;



-- crear chunks y embeddings con DBMS_VECTOR_CHAIN
drop table if exists markdown_chunks;

CREATE TABLE markdown_chunks AS
SELECT
    m.id                       as doc_id,
    m.filename                 as file_name,
    et.embed_id                as chunk_id,
    et.embed_data              as chunk_text,
    TO_VECTOR(et.embed_vector) as embedding
FROM markdown_files m,
    DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS(
        -- primero dividimos el CLOB en partes pequenas (chunks)
        DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS(
            m.md_clob,
            JSON('{
                "by"        : "words",
                "max"       : "120",
                "overlap"   : "20",
                "split"     : "recursively",
                "language"  : "spanish",
                "normalize" : "all"
            }')
        ),
        -- despues generamos el embedding de cada chunk
        JSON('{
            "provider" : "database",
            "model"    : "MULTILINGUAL_E5_BASE"
        }')
    ) t,
    JSON_TABLE(
        t.column_value,
        '$[*]' COLUMNS (
            embed_id     NUMBER         PATH '$.embed_id',
            embed_data   VARCHAR2(4000) PATH '$.embed_data',
            embed_vector CLOB           PATH '$.embed_vector'
        )
    ) et;

commit;

-- ver los chunks generados
select doc_id, file_name, chunk_id, chunk_text
from markdown_chunks
order by doc_id, chunk_id;



-- buscar chunks por similaridad semantica
select
    doc_id,
    chunk_id,
    VECTOR_DISTANCE(embedding, 
                    VECTOR_EMBEDDING(MULTILINGUAL_E5_BASE 
                                    USING 'estaciones de servicio' as DATA
                                    )
                    ) as distance,
    chunk_text    
from markdown_chunks
where doc_id = 1 
order by distance
fetch first 3 rows only;