create or replace package body pkg_26ai_auto_load as /* Criado por: fernando.leal@oracle.com Data: Mar/2026 Objetivo: demonstrar casos de Vetorizacao em massa com ONNX e modelos externos do Oracle AI Database 26ai v1 - Auto Load - leal */ -- -- Rotina que define a lista de dados a serem vetorizados -- Define-se o tipo de embedding para que outros jobs ja existentes possam continuar execucao sem impacto de novas cargas, e assim, testar novos embeddings -- PROCEDURE prc_refresh_files(p_oci_cred IN VARCHAR2 default 'OCI_CRED', p_bucket in varchar2 default null, p_mimetype in varchar2 default 'PDF', p_embedding_name in varchar2 default 'COHERE') as -- ao levar para ambiente de cliente, definir parametro para p_bucket -- default: associado ao bucket de testes - Leal 25-03-26 v_bucket varchar2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/n/idi1o0a010nx/b/bucket-database-26ai/o/'; -- := p_bucket begin if p_mimetype = 'PDF' then -- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load -- tabela de controle dos arquivos que devem ser processados -- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquoivos da fila -- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina PROC_EMBED_FILES -- (3) os arquivos com status "P" foram vetorizados pela rotina PROC_EMBED_FILES insert into tb_26ai_auto_load(dt_ref, object_name, status, worker_id, bytes, created, last_modified, EMBEDDING_NAME, EMBEDDING_MIMETYPE) SELECT sysdate, UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8'), -- cuidado: caracteres especiais tem tratamento de acesso para object storage null, null, BYTES, CREATED, LAST_MODIFIED, p_embedding_name, p_mimetype FROM DBMS_CLOUD.LIST_OBJECTS(p_oci_cred, v_bucket) WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN ( SELECT object_name FROM TB_26ai_AUTO_LOAD where EMBEDDING_MIMETYPE = 'PDF' and EMBEDDING_NAME = p_embedding_name) AND UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%pdf'; -- ajuste leal 250226 end if; -- pdf if p_mimetype = 'CLOB' then -- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load insert into tb_26ai_auto_load(dt_ref, object_name, status, worker_id, bytes, created, last_modified, EMBEDDING_NAME, EMBEDDING_MIMETYPE) SELECT sysdate, src.p_partkey, null, null, -- inicia em null, mas o job faz asociacao ao seu ID em prc_reserve_files dbms_lob.getlength( 'Name: ' || src.p_name || ' Color: ' || src.p_color || ' Size: ' || src.p_size || ' Type: ' || src.p_type || ' Container: ' || src.p_container ), sysdate, null, p_embedding_name, p_mimetype FROM ssb.PART src --WHERE src.dat_ref = p_data_id where rownum<=5000 and (src.p_partkey) NOT IN ( SELECT al.object_name FROM TB_26ai_AUTO_LOAD al where EMBEDDING_MIMETYPE = 'CLOB' and EMBEDDING_NAME = p_embedding_name); end if; if p_mimetype = 'JPG' then -- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load -- tabela de controle dos arquivos que devem ser processados -- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquoivos da fila -- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina PROC_EMBED_FILES -- (3) os arquivos com status "P" foram vetorizados pela rotina PROC_EMBED_FILES insert into tb_26ai_auto_load(dt_ref, object_name, status, worker_id, bytes, created, last_modified, EMBEDDING_NAME, EMBEDDING_MIMETYPE) SELECT sysdate, UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8'), -- cuidado: caracteres especiais tem tratamento de acesso para object storage null, null, BYTES, CREATED, LAST_MODIFIED, p_embedding_name, p_mimetype FROM DBMS_CLOUD.LIST_OBJECTS(p_oci_cred, v_bucket) WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN ( SELECT object_name FROM TB_26ai_AUTO_LOAD where EMBEDDING_MIMETYPE = 'JPG' and EMBEDDING_NAME = p_embedding_name) AND ( UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpg' or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpeg' ) ; end if; -- jpg commit; end; -- -- Isola de modo unico, por worker id (job), o batch de dados que deverao ser processados. Status definido em R (Reservado) -- Nao é chamado diretamente, mas faz parte da rotina de embedding -- PROCEDURE prc_reserve_files ( p_limit IN NUMBER, p_worker_id IN VARCHAR2, -- sera setado via job para definir que este bloco de dados sera usado pelo job N p_oci_cred IN VARCHAR2 default 'OCI_CRED', p_bucket in varchar2 , p_docs OUT SYS.ODCIVARCHAR2LIST, p_mimetype in varchar2 default 'PDF', p_embedding_name in varchar2 default 'COHERE' ) IS BEGIN -- inicialização obrigatória p_docs := SYS.ODCIVARCHAR2LIST(); FOR r1 IN ( SELECT object_name FROM tb_26ai_auto_load WHERE status IS NULL and ROWNUM <= p_limit -- CUIDADO: requer ajustes de acordo com tamanho dos dados e servidor and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype FOR UPDATE SKIP LOCKED ) LOOP UPDATE tb_26ai_auto_load SET status = 'R', -- reserved worker_id = p_worker_id -- sera setado via job para definir que este bloco de dados sera usado pelo job N WHERE object_name = r1.object_name and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype and status IS NULL; p_docs.EXTEND; p_docs(p_docs.COUNT) := r1.object_name; END LOOP; commit; END; -- -- Rotina principal de Embedding -- Status de Reserva (R) torna se Started (S) -- Se concluir com sucesso, Started (S) torna se Processado (P) -- Senao, torna-se Error (E) -- -- Para usar ONNX, importe previamente o ONNX ao banco com comando abaixo. Em seguida, ajuste string de uso, com nome do modelo, nas linhas de codigo do inicio desta rotina. -- /* -- importacao do modelo ao banco BEGIN DBMS_VECTOR.LOAD_ONNX_MODEL(directory=>'DATA_PUMP_DIR', file_name=>'clip-vit-large-patch14_img.onnx', model_name=>'OPENAI_CLIP_MULTI_IMG', metadata=>JSON('{"function" : "embedding", "embeddingOutput":"embedding", "input": {"input": ["DATA"]}}') ); END; */ -- -- Pre requisito: create sequence seq_26ai_auto_load MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ; -- -- PROCEDURE proc_embed_files(p_limit in number default 10, p_worker_id in number, p_stop_process_list in varchar2 default 'N', p_mimetype in varchar2 default 'PDF', p_embedding_name in varchar2 default 'COHERE') AS v_error CLOB; v_session_id NUMBER; v_dt_start TIMESTAMP; v_oci_cred VARCHAR2(20) := 'OCI_CRED'; v_bucket VARCHAR2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/n/idi1o0a010nx/b/bucket-database-26ai/o/'; l_docs SYS.ODCIVARCHAR2LIST; -- CREATE OR REPLACE TYPE t_audio_id_list AS TABLE OF VARCHAR2(600); v_all_ids SYS.ODCIVARCHAR2LIST; v_json_embedding varchar2(2000); BEGIN IF p_stop_process_list != 'N' THEN RETURN; END IF; if p_embedding_name = 'COHERE' then v_json_embedding := '{"provider": "OCIGenAI","credential_name": "OCI_CRED","url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText","batch_size": 50,"model": "cohere.embed-v4.0"}'; elsif p_embedding_name = 'VLLM' then v_json_embedding := '{"provider": "openai","url": "https://hub-gpus.DOMINIO.com.br/embed/v1/embeddings","host":"local","batch_size": 100,"model": "Qwen/Qwen3-Embedding-4B"}'; elsif p_embedding_name = 'OPENAI' then v_json_embedding := '{"provider" : "openai","credential_name" : "CRED_OPENAI", "url":"https://api.openai.com/v1/chat/completions", "model" : "gpt-4.1-mini" }'; elsif p_embedding_name = 'ONNX-E5' then v_json_embedding := '{"provider":"database", "model":"MULTILINGUAL_E5_BASE"}'; elsif p_embedding_name = 'ONNX-VIT' then v_json_embedding := '{"provider":"database", "model":"VIT_BASE_PATCH16_224"}'; end if; -- -- rotina que reserva arquivos de modo exclusivo, permitindo uso de scheduler paralelos no banco -- objetivo: embedding em sessoes paralelas do banco para diminui tempo de carga -- (1) deve ser definido um valor adequado de arquivos por job, definido no limite de linhas (limit) -- (2) cada job tem seu worker id definido pelo proprio scheduler -- prc_reserve_files( p_limit => p_limit, p_worker_id=>p_worker_id, p_oci_cred=> v_oci_cred, p_bucket=> v_bucket, p_docs=>l_docs, p_mimetype => p_mimetype, p_embedding_name => p_embedding_name ) ; v_dt_start := CURRENT_TIMESTAMP; -- create sequence seq_26ai_auto_load MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ; v_session_id := seq_26ai_auto_load.nextval; if p_mimetype = 'CLOB' then -- Marca batch como Iniciado UPDATE tb_26ai_auto_load SET status = 'S', -- started dt_start_process = SYSDATE WHERE worker_id = p_worker_id and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype and status = 'R'; COMMIT; -- embedding BEGIN INSERT INTO tb_26ai_auto_load_vector ( ID, FILE_NAME, CREATED_DATE, CREATED_BY, EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, attr1, attr2, attr3, attr4, attr5, attr6, attr7, attr8, attr9, attr10, attr11, attr12, attr13, attr14, attr15, attr16 ) SELECT v_session_id, src.p_partkey, CURRENT_TIMESTAMP, 'admin3', et.embed_id, et.text_chunk, et.embed_vector, p_embedding_name, NULL, src.p_name, src.p_color, src.p_type, src.p_size, src.p_container, null, null, null, null, null, null, null, null, null, p_mimetype FROM SSB.PART src CROSS JOIN TABLE( DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS( DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS( dbms_vector_chain.utl_to_text( 'Name: ' || src.p_name || ' Color: ' || src.p_color || ' Size: ' || src.p_size || ' Type: ' || src.p_type || ' Container: ' || src.p_container ) , JSON('{"by":"words","max":"300","split":"sentence","normalize":"all","overlap":"30"}') ), JSON(v_json_embedding) ) ) t CROSS JOIN JSON_TABLE( t.column_value, '$[*]' COLUMNS ( embed_id NUMBER PATH '$.embed_id', text_chunk VARCHAR2(4000) PATH '$.embed_data', embed_vector CLOB PATH '$.embed_vector' ) ) AS et WHERE src.p_partkey IN ( SELECT object_name FROM tb_26ai_auto_load src WHERE worker_id = p_worker_id and status = 'S' -- started ); --AND src.dat_ref = p_worker_id; -- Marca batch como processado UPDATE tb_26ai_auto_load SET status = 'P', -- processado dt_end_process = SYSDATE WHERE worker_id = p_worker_id and status = 'S'; COMMIT; EXCEPTION WHEN OTHERS THEN v_error := SQLERRM; INSERT INTO tb_26ai_auto_load_debug VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE); -- Marca batch como ERRO UPDATE tb_26ai_auto_load SET status = 'E', dt_end_process = SYSDATE, error_msg = v_error WHERE worker_id = p_worker_id and status = 'S'; COMMIT; END; end if; -- clob if p_mimetype = 'PDF' then SELECT object_name BULK COLLECT INTO v_all_ids FROM ( SELECT object_name FROM tb_26ai_auto_load src WHERE worker_id = p_worker_id and status = 'R' and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype ); -- Marca batch de PDF como Iniciado UPDATE tb_26ai_auto_load SET status = 'S', -- started dt_start_process = SYSDATE WHERE worker_id = p_worker_id and status = 'R' and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype; COMMIT; -- copia temporaria para banco FOR i IN 1 .. v_all_ids.COUNT LOOP DBMS_CLOUD.GET_OBJECT( credential_name => v_oci_cred, object_uri => v_bucket || UTL_URL.ESCAPE( v_all_ids(i) ,TRUE,'AL32UTF8') , directory_name => 'DATA_PUMP_DIR', file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); END LOOP; -- embedding BEGIN FORALL i IN 1 .. v_all_ids.COUNT INSERT INTO tb_26ai_auto_load_vector ( ID, FILE_NAME, CREATED_DATE, CREATED_BY, EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, attr1, attr2, attr3, attr4, attr5, attr6, attr7, attr8, attr9, attr10, attr11, attr12, attr13, attr14, attr15, attr16 ) SELECT v_session_id, lower(replace( v_all_ids(i) ,' ','_')), CURRENT_TIMESTAMP, 'admin', et.embed_id, et.text_chunk, et.embed_vector, p_embedding_name, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, p_mimetype FROM dual CROSS JOIN TABLE( DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS( DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS( dbms_vector_chain.utl_to_text( to_blob(bfilename('DATA_PUMP_DIR', lower( replace( v_all_ids(i) ,' ','_') ) ) ) ), JSON('{"by":"words","max":"' || 400 || '","split":"sentence","normalize":"all","overlap":' || 40 || '}') ), JSON(v_json_embedding) ) ) t CROSS JOIN JSON_TABLE( t.column_value, '$[*]' COLUMNS ( embed_id NUMBER PATH '$.embed_id', text_chunk VARCHAR2(4000) PATH '$.embed_data', embed_vector CLOB PATH '$.embed_vector' ) ) AS et; -- Marca batch de PDF como processado UPDATE tb_26ai_auto_load SET status = 'P', -- processado dt_end_process = SYSDATE WHERE worker_id = p_worker_id and status = 'S'; COMMIT; EXCEPTION WHEN OTHERS THEN v_error := SQLERRM; INSERT INTO tb_26ai_auto_load_debug VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE); -- Marca batch de PDF como erro UPDATE tb_26ai_auto_load SET status = 'E', -- erro dt_end_process = SYSDATE WHERE worker_id = p_worker_id and status = 'S'; COMMIT; END; -- eliminacao dos arquivos temporarianete gravados paras DATA_PUMP_DIR FOR i IN 1 .. v_all_ids.COUNT LOOP DBMS_CLOUD.DELETE_FILE( directory_name => 'DATA_PUMP_DIR', file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); END LOOP; end if; -- pdf if p_mimetype = 'JPG' then SELECT object_name BULK COLLECT INTO v_all_ids FROM ( SELECT object_name FROM tb_26ai_auto_load src WHERE worker_id = p_worker_id and status = 'R' and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype ); -- Marca batch de PDF como Iniciado UPDATE tb_26ai_auto_load SET status = 'S', -- started dt_start_process = SYSDATE WHERE worker_id = p_worker_id and status = 'R' and EMBEDDING_NAME = p_embedding_name and EMBEDDING_MIMETYPE = p_mimetype; COMMIT; -- copia temporaria para banco FOR i IN 1 .. v_all_ids.COUNT LOOP DBMS_CLOUD.GET_OBJECT( credential_name => v_oci_cred, object_uri => v_bucket || UTL_URL.ESCAPE( v_all_ids(i) ,TRUE,'AL32UTF8') , directory_name => 'DATA_PUMP_DIR', file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); END LOOP; -- embedding BEGIN FORALL i IN 1 .. v_all_ids.COUNT INSERT INTO tb_26ai_auto_load_vector ( ID, FILE_NAME, CREATED_DATE, CREATED_BY, EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, attr1, attr2, attr3, attr4, attr5, attr6, attr7, attr8, attr9, attr10, attr11, attr12, attr13, attr14, attr15, attr16 ) SELECT v_session_id, lower(replace( v_all_ids(i) ,' ','_')), CURRENT_TIMESTAMP, 'admin', rownum embed_id, null text_chunk, t.vec, p_embedding_name, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, p_mimetype FROM ( select DBMS_VECTOR.UTL_TO_EMBEDDING( to_blob(bfilename('DATA_PUMP_DIR', lower( replace( v_all_ids(i) ,' ','_') ) ) ) , 'image', JSON('{ "provider": "OCIGenAI", "credential_name": "OCI_CRED", "url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText", "model": "cohere.embed-v4.0"}') ) vec) t; -- Marca batch de PDF como processado UPDATE tb_26ai_auto_load SET status = 'P', -- processado dt_end_process = SYSDATE WHERE worker_id = p_worker_id and status = 'S'; COMMIT; EXCEPTION WHEN OTHERS THEN v_error := SQLERRM; INSERT INTO tb_26ai_auto_load_debug VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE); -- Marca batch de PDF como erro UPDATE tb_26ai_auto_load SET status = 'E', -- erro dt_end_process = SYSDATE WHERE worker_id = p_worker_id and status = 'S'; COMMIT; END; -- eliminacao dos arquivos temporarianete gravados paras DATA_PUMP_DIR FOR i IN 1 .. v_all_ids.COUNT LOOP DBMS_CLOUD.DELETE_FILE( directory_name => 'DATA_PUMP_DIR', file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); END LOOP; end if; -- jpg COMMIT; END; PROCEDURE proc_remove_jobs AS BEGIN -- nao pode fazer pelo numero de jobs existentes pois senao a eliminacao seria falha: -- cada job tem um padrao de ome associado ao worker id, e nao a contabilizacao que pode ter gaps -- ajustar de acordo com maximo permitido pelo item de definicao de novos jobs FOR i IN 1..300 LOOP begin DBMS_SCHEDULER.STOP_JOB( job_name => 'JOB_WORKER_' || i ); exception when others then null; end; begin DBMS_SCHEDULER.DROP_JOB( job_name => 'JOB_WORKER_' || i ); exception when others then null; end; update tb_26ai_auto_load set status = null, worker_id = null where status = 'R'; -- estava reservado, mas com remocao do job volta pra status null sem worker id definido commit; END LOOP; END; PROCEDURE proc_add_jobs(p_limit in number, p_total_jobs in number, p_mimetype in varchar2, p_embedding_name in varchar2 default 'COHERE') AS v_job_count number; v_has number; BEGIN select count(1) into v_job_count from user_scheduler_jobs where JOB_NAME like 'JOB_WORKER_%'; -- if p_total_jobs <= v_job_count or p_total_jobs is null then --raise_application_error(-20002,'The number of scheduler jobs must be greater than what already exists'); -- else FOR i IN 1..p_total_jobs LOOP select count(1) into v_has from user_scheduler_jobs where JOB_NAME = 'JOB_WORKER_' || i; if v_has = 0 then -- nao existe com worker id "i" DBMS_SCHEDULER.CREATE_JOB( job_name => 'JOB_WORKER_' || i, job_type => 'PLSQL_BLOCK', job_action => 'BEGIN pkg_26ai_auto_load.proc_embed_files(p_limit=>' || p_limit || ',p_worker_id=>' || i || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;', start_date => SYSTIMESTAMP, repeat_interval => 'FREQ=SECONDLY; INTERVAL=10;', enabled => TRUE ); else -- cria-se um novo job alem do ultimo DBMS_SCHEDULER.CREATE_JOB( job_name => 'JOB_WORKER_' || to_char(i+v_job_count), job_type => 'PLSQL_BLOCK', job_action => 'BEGIN pkg_26ai_auto_load.proc_embed_files(p_limit=>' || p_limit || ',p_worker_id=>' || to_char(i+v_job_count) || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;', start_date => SYSTIMESTAMP, repeat_interval => 'FREQ=SECONDLY; INTERVAL=10;', enabled => TRUE ); end if; END LOOP; -- end if; END; end; /