695 lines
23 KiB
Plaintext
695 lines
23 KiB
Plaintext
create or replace package body pkg_26ai_auto_load
|
|
as
|
|
|
|
/*
|
|
|
|
Criado por: fernando.leal@oracle.com
|
|
Data: Mar/2026
|
|
Objetivo: demonstrar casos de Vetorizacao em massa com ONNX e modelos externos do Oracle AI Database 26ai
|
|
|
|
v1 - Auto Load - leal
|
|
|
|
*/
|
|
|
|
--
|
|
-- Rotina que define a lista de dados a serem vetorizados
|
|
-- Define-se o tipo de embedding para que outros jobs ja existentes possam continuar execucao sem impacto de novas cargas, e assim, testar novos embeddings
|
|
--
|
|
PROCEDURE prc_refresh_files(p_oci_cred IN VARCHAR2 default 'OCI_CRED',
|
|
p_bucket in varchar2 default null,
|
|
p_mimetype in varchar2 default 'PDF',
|
|
p_embedding_name in varchar2 default 'COHERE')
|
|
as
|
|
-- ao levar para ambiente de cliente, definir parametro para p_bucket
|
|
-- default: associado ao bucket de testes - Leal 25-03-26
|
|
v_bucket varchar2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/n/idi1o0a010nx/b/bucket-database-26ai/o/'; -- := p_bucket
|
|
begin
|
|
|
|
if p_mimetype = 'PDF' then
|
|
|
|
-- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load
|
|
-- tabela de controle dos arquivos que devem ser processados
|
|
-- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquoivos da fila
|
|
-- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina PROC_EMBED_FILES
|
|
-- (3) os arquivos com status "P" foram vetorizados pela rotina PROC_EMBED_FILES
|
|
insert into tb_26ai_auto_load(dt_ref,
|
|
object_name,
|
|
status,
|
|
worker_id,
|
|
bytes,
|
|
created,
|
|
last_modified,
|
|
EMBEDDING_NAME,
|
|
EMBEDDING_MIMETYPE)
|
|
SELECT sysdate,
|
|
UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8'), -- cuidado: caracteres especiais tem tratamento de acesso para object storage
|
|
null,
|
|
null,
|
|
BYTES,
|
|
CREATED,
|
|
LAST_MODIFIED,
|
|
p_embedding_name,
|
|
p_mimetype
|
|
FROM DBMS_CLOUD.LIST_OBJECTS(p_oci_cred, v_bucket)
|
|
WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN (
|
|
SELECT object_name
|
|
FROM TB_26ai_AUTO_LOAD
|
|
where EMBEDDING_MIMETYPE = 'PDF'
|
|
and EMBEDDING_NAME = p_embedding_name)
|
|
AND UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%pdf'; -- ajuste leal 250226
|
|
end if; -- pdf
|
|
|
|
|
|
|
|
if p_mimetype = 'CLOB' then
|
|
|
|
|
|
-- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load
|
|
insert into tb_26ai_auto_load(dt_ref,
|
|
object_name,
|
|
status,
|
|
worker_id,
|
|
bytes,
|
|
created,
|
|
last_modified,
|
|
EMBEDDING_NAME,
|
|
EMBEDDING_MIMETYPE)
|
|
SELECT sysdate,
|
|
src.p_partkey,
|
|
null,
|
|
null, -- inicia em null, mas o job faz asociacao ao seu ID em prc_reserve_files
|
|
dbms_lob.getlength( 'Name: ' || src.p_name || ' Color: ' || src.p_color || ' Size: ' || src.p_size || ' Type: ' || src.p_type || ' Container: ' || src.p_container ),
|
|
sysdate,
|
|
null,
|
|
p_embedding_name,
|
|
p_mimetype
|
|
FROM ssb.PART src
|
|
--WHERE src.dat_ref = p_data_id
|
|
where rownum<=5000
|
|
and (src.p_partkey) NOT IN (
|
|
SELECT al.object_name
|
|
FROM TB_26ai_AUTO_LOAD al
|
|
where EMBEDDING_MIMETYPE = 'CLOB'
|
|
and EMBEDDING_NAME = p_embedding_name);
|
|
end if;
|
|
|
|
if p_mimetype = 'JPG' then
|
|
|
|
-- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load
|
|
-- tabela de controle dos arquivos que devem ser processados
|
|
-- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquoivos da fila
|
|
-- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina PROC_EMBED_FILES
|
|
-- (3) os arquivos com status "P" foram vetorizados pela rotina PROC_EMBED_FILES
|
|
insert into tb_26ai_auto_load(dt_ref,
|
|
object_name,
|
|
status,
|
|
worker_id,
|
|
bytes,
|
|
created,
|
|
last_modified,
|
|
EMBEDDING_NAME,
|
|
EMBEDDING_MIMETYPE)
|
|
SELECT sysdate,
|
|
UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8'), -- cuidado: caracteres especiais tem tratamento de acesso para object storage
|
|
null,
|
|
null,
|
|
BYTES,
|
|
CREATED,
|
|
LAST_MODIFIED,
|
|
p_embedding_name,
|
|
p_mimetype
|
|
FROM DBMS_CLOUD.LIST_OBJECTS(p_oci_cred, v_bucket)
|
|
WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN (
|
|
SELECT object_name
|
|
FROM TB_26ai_AUTO_LOAD
|
|
where EMBEDDING_MIMETYPE = 'JPG'
|
|
and EMBEDDING_NAME = p_embedding_name)
|
|
AND ( UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpg'
|
|
or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpeg' ) ;
|
|
end if; -- jpg
|
|
|
|
|
|
commit;
|
|
|
|
end;
|
|
|
|
|
|
--
|
|
-- Isola de modo unico, por worker id (job), o batch de dados que deverao ser processados. Status definido em R (Reservado)
|
|
-- Nao é chamado diretamente, mas faz parte da rotina de embedding
|
|
--
|
|
PROCEDURE prc_reserve_files (
|
|
p_limit IN NUMBER,
|
|
p_worker_id IN VARCHAR2, -- sera setado via job para definir que este bloco de dados sera usado pelo job N
|
|
p_oci_cred IN VARCHAR2 default 'OCI_CRED',
|
|
p_bucket in varchar2 ,
|
|
p_docs OUT SYS.ODCIVARCHAR2LIST,
|
|
p_mimetype in varchar2 default 'PDF',
|
|
p_embedding_name in varchar2 default 'COHERE'
|
|
)
|
|
IS
|
|
BEGIN
|
|
|
|
-- inicialização obrigatória
|
|
p_docs := SYS.ODCIVARCHAR2LIST();
|
|
|
|
FOR r1 IN (
|
|
SELECT object_name
|
|
FROM tb_26ai_auto_load
|
|
WHERE status IS NULL
|
|
and ROWNUM <= p_limit -- CUIDADO: requer ajustes de acordo com tamanho dos dados e servidor
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype
|
|
FOR UPDATE SKIP LOCKED
|
|
) LOOP
|
|
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'R', -- reserved
|
|
worker_id = p_worker_id -- sera setado via job para definir que este bloco de dados sera usado pelo job N
|
|
WHERE object_name = r1.object_name
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype
|
|
and status IS NULL;
|
|
|
|
p_docs.EXTEND;
|
|
p_docs(p_docs.COUNT) := r1.object_name;
|
|
|
|
END LOOP;
|
|
|
|
commit;
|
|
|
|
END;
|
|
|
|
|
|
--
|
|
-- Rotina principal de Embedding
|
|
-- Status de Reserva (R) torna se Started (S)
|
|
-- Se concluir com sucesso, Started (S) torna se Processado (P)
|
|
-- Senao, torna-se Error (E)
|
|
--
|
|
-- Para usar ONNX, importe previamente o ONNX ao banco com comando abaixo. Em seguida, ajuste string de uso, com nome do modelo, nas linhas de codigo do inicio desta rotina.
|
|
--
|
|
/* -- importacao do modelo ao banco
|
|
BEGIN
|
|
DBMS_VECTOR.LOAD_ONNX_MODEL(directory=>'DATA_PUMP_DIR',
|
|
file_name=>'clip-vit-large-patch14_img.onnx',
|
|
model_name=>'OPENAI_CLIP_MULTI_IMG',
|
|
metadata=>JSON('{"function" : "embedding", "embeddingOutput":"embedding", "input": {"input": ["DATA"]}}') );
|
|
|
|
END;
|
|
*/
|
|
--
|
|
-- Pre requisito: create sequence seq_26ai_auto_load MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ;
|
|
--
|
|
--
|
|
PROCEDURE proc_embed_files(p_limit in number default 10,
|
|
p_worker_id in number,
|
|
p_stop_process_list in varchar2 default 'N',
|
|
p_mimetype in varchar2 default 'PDF',
|
|
p_embedding_name in varchar2 default 'COHERE')
|
|
AS
|
|
v_error CLOB;
|
|
v_session_id NUMBER;
|
|
v_dt_start TIMESTAMP;
|
|
v_oci_cred VARCHAR2(20) := 'OCI_CRED';
|
|
v_bucket VARCHAR2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/n/idi1o0a010nx/b/bucket-database-26ai/o/';
|
|
l_docs SYS.ODCIVARCHAR2LIST;
|
|
|
|
-- CREATE OR REPLACE TYPE t_audio_id_list AS TABLE OF VARCHAR2(600);
|
|
v_all_ids SYS.ODCIVARCHAR2LIST;
|
|
v_json_embedding varchar2(2000);
|
|
BEGIN
|
|
|
|
IF p_stop_process_list != 'N' THEN
|
|
RETURN;
|
|
END IF;
|
|
|
|
if p_embedding_name = 'COHERE' then
|
|
v_json_embedding := '{"provider": "OCIGenAI","credential_name": "OCI_CRED","url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText","batch_size": 50,"model": "cohere.embed-v4.0"}';
|
|
|
|
elsif p_embedding_name = 'VLLM' then
|
|
v_json_embedding := '{"provider": "openai","url": "https://hub-gpus.DOMINIO.com.br/embed/v1/embeddings","host":"local","batch_size": 100,"model": "Qwen/Qwen3-Embedding-4B"}';
|
|
|
|
elsif p_embedding_name = 'OPENAI' then
|
|
v_json_embedding := '{"provider" : "openai","credential_name" : "CRED_OPENAI", "url":"https://api.openai.com/v1/chat/completions", "model" : "gpt-4.1-mini" }';
|
|
|
|
elsif p_embedding_name = 'ONNX-E5' then
|
|
v_json_embedding := '{"provider":"database", "model":"MULTILINGUAL_E5_BASE"}';
|
|
|
|
elsif p_embedding_name = 'ONNX-VIT' then
|
|
v_json_embedding := '{"provider":"database", "model":"VIT_BASE_PATCH16_224"}';
|
|
|
|
end if;
|
|
|
|
--
|
|
-- rotina que reserva arquivos de modo exclusivo, permitindo uso de scheduler paralelos no banco
|
|
-- objetivo: embedding em sessoes paralelas do banco para diminui tempo de carga
|
|
-- (1) deve ser definido um valor adequado de arquivos por job, definido no limite de linhas (limit)
|
|
-- (2) cada job tem seu worker id definido pelo proprio scheduler
|
|
--
|
|
prc_reserve_files(
|
|
p_limit => p_limit,
|
|
p_worker_id=>p_worker_id,
|
|
p_oci_cred=> v_oci_cred,
|
|
p_bucket=> v_bucket,
|
|
p_docs=>l_docs,
|
|
p_mimetype => p_mimetype,
|
|
p_embedding_name => p_embedding_name ) ;
|
|
|
|
v_dt_start := CURRENT_TIMESTAMP;
|
|
-- create sequence seq_26ai_auto_load MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ;
|
|
v_session_id := seq_26ai_auto_load.nextval;
|
|
|
|
|
|
if p_mimetype = 'CLOB' then
|
|
|
|
-- Marca batch como Iniciado
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'S', -- started
|
|
dt_start_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype
|
|
and status = 'R';
|
|
COMMIT;
|
|
|
|
-- embedding
|
|
BEGIN
|
|
|
|
INSERT INTO tb_26ai_auto_load_vector (
|
|
ID, FILE_NAME, CREATED_DATE, CREATED_BY,
|
|
EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE,
|
|
attr1, attr2, attr3, attr4, attr5, attr6, attr7,
|
|
attr8, attr9, attr10, attr11, attr12, attr13,
|
|
attr14, attr15, attr16
|
|
)
|
|
SELECT
|
|
v_session_id,
|
|
src.p_partkey,
|
|
CURRENT_TIMESTAMP,
|
|
'admin3',
|
|
et.embed_id,
|
|
et.text_chunk,
|
|
et.embed_vector,
|
|
p_embedding_name,
|
|
NULL,
|
|
src.p_name,
|
|
src.p_color,
|
|
src.p_type,
|
|
src.p_size,
|
|
src.p_container,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
p_mimetype
|
|
FROM SSB.PART src
|
|
CROSS JOIN TABLE(
|
|
DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS(
|
|
DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS(
|
|
dbms_vector_chain.utl_to_text( 'Name: ' || src.p_name || ' Color: ' || src.p_color || ' Size: ' || src.p_size || ' Type: ' || src.p_type || ' Container: ' || src.p_container ) ,
|
|
JSON('{"by":"words","max":"300","split":"sentence","normalize":"all","overlap":"30"}')
|
|
),
|
|
JSON(v_json_embedding)
|
|
)
|
|
) t
|
|
CROSS JOIN JSON_TABLE(
|
|
t.column_value, '$[*]'
|
|
COLUMNS (
|
|
embed_id NUMBER PATH '$.embed_id',
|
|
text_chunk VARCHAR2(4000) PATH '$.embed_data',
|
|
embed_vector CLOB PATH '$.embed_vector'
|
|
)
|
|
) AS et
|
|
WHERE src.p_partkey IN ( SELECT object_name
|
|
FROM tb_26ai_auto_load src
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S' -- started
|
|
);
|
|
--AND src.dat_ref = p_worker_id;
|
|
|
|
-- Marca batch como processado
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'P', -- processado
|
|
dt_end_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S';
|
|
COMMIT;
|
|
|
|
|
|
EXCEPTION
|
|
WHEN OTHERS THEN
|
|
|
|
v_error := SQLERRM;
|
|
|
|
INSERT INTO tb_26ai_auto_load_debug
|
|
VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE);
|
|
|
|
-- Marca batch como ERRO
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'E',
|
|
dt_end_process = SYSDATE,
|
|
error_msg = v_error
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S';
|
|
|
|
COMMIT;
|
|
END;
|
|
|
|
end if; -- clob
|
|
|
|
if p_mimetype = 'PDF' then
|
|
|
|
SELECT object_name
|
|
BULK COLLECT INTO v_all_ids
|
|
FROM (
|
|
SELECT object_name
|
|
FROM tb_26ai_auto_load src
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'R'
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype
|
|
);
|
|
|
|
-- Marca batch de PDF como Iniciado
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'S', -- started
|
|
dt_start_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'R'
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype;
|
|
|
|
COMMIT;
|
|
|
|
-- copia temporaria para banco
|
|
FOR i IN 1 .. v_all_ids.COUNT LOOP
|
|
DBMS_CLOUD.GET_OBJECT(
|
|
credential_name => v_oci_cred,
|
|
object_uri => v_bucket || UTL_URL.ESCAPE( v_all_ids(i) ,TRUE,'AL32UTF8') ,
|
|
directory_name => 'DATA_PUMP_DIR',
|
|
file_name => lower( replace( v_all_ids(i) ,' ','_') ) );
|
|
END LOOP;
|
|
|
|
-- embedding
|
|
BEGIN
|
|
FORALL i IN 1 .. v_all_ids.COUNT
|
|
INSERT INTO tb_26ai_auto_load_vector (
|
|
ID, FILE_NAME, CREATED_DATE, CREATED_BY,
|
|
EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE,
|
|
attr1, attr2, attr3, attr4, attr5, attr6, attr7,
|
|
attr8, attr9, attr10, attr11, attr12, attr13,
|
|
attr14, attr15, attr16
|
|
)
|
|
SELECT
|
|
v_session_id,
|
|
lower(replace( v_all_ids(i) ,' ','_')),
|
|
CURRENT_TIMESTAMP,
|
|
'admin',
|
|
et.embed_id,
|
|
et.text_chunk,
|
|
et.embed_vector,
|
|
p_embedding_name,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
p_mimetype
|
|
FROM dual
|
|
CROSS JOIN TABLE(
|
|
DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS(
|
|
DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS(
|
|
dbms_vector_chain.utl_to_text( to_blob(bfilename('DATA_PUMP_DIR', lower( replace( v_all_ids(i) ,' ','_') ) ) ) ),
|
|
JSON('{"by":"words","max":"' || 400 || '","split":"sentence","normalize":"all","overlap":' || 40 || '}')
|
|
),
|
|
JSON(v_json_embedding)
|
|
)
|
|
) t
|
|
CROSS JOIN JSON_TABLE(
|
|
t.column_value, '$[*]'
|
|
COLUMNS (
|
|
embed_id NUMBER PATH '$.embed_id',
|
|
text_chunk VARCHAR2(4000) PATH '$.embed_data',
|
|
embed_vector CLOB PATH '$.embed_vector'
|
|
)
|
|
) AS et;
|
|
|
|
-- Marca batch de PDF como processado
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'P', -- processado
|
|
dt_end_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S';
|
|
|
|
COMMIT;
|
|
EXCEPTION
|
|
WHEN OTHERS THEN
|
|
|
|
v_error := SQLERRM;
|
|
|
|
INSERT INTO tb_26ai_auto_load_debug
|
|
VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE);
|
|
|
|
-- Marca batch de PDF como erro
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'E', -- erro
|
|
dt_end_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S';
|
|
COMMIT;
|
|
END;
|
|
|
|
-- eliminacao dos arquivos temporarianete gravados paras DATA_PUMP_DIR
|
|
FOR i IN 1 .. v_all_ids.COUNT LOOP
|
|
DBMS_CLOUD.DELETE_FILE( directory_name => 'DATA_PUMP_DIR',
|
|
file_name => lower( replace( v_all_ids(i) ,' ','_') ) );
|
|
END LOOP;
|
|
|
|
|
|
end if; -- pdf
|
|
|
|
|
|
if p_mimetype = 'JPG' then
|
|
|
|
SELECT object_name
|
|
BULK COLLECT INTO v_all_ids
|
|
FROM (
|
|
SELECT object_name
|
|
FROM tb_26ai_auto_load src
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'R'
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype
|
|
);
|
|
|
|
-- Marca batch de PDF como Iniciado
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'S', -- started
|
|
dt_start_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'R'
|
|
and EMBEDDING_NAME = p_embedding_name
|
|
and EMBEDDING_MIMETYPE = p_mimetype;
|
|
|
|
COMMIT;
|
|
|
|
-- copia temporaria para banco
|
|
FOR i IN 1 .. v_all_ids.COUNT LOOP
|
|
DBMS_CLOUD.GET_OBJECT(
|
|
credential_name => v_oci_cred,
|
|
object_uri => v_bucket || UTL_URL.ESCAPE( v_all_ids(i) ,TRUE,'AL32UTF8') ,
|
|
directory_name => 'DATA_PUMP_DIR',
|
|
file_name => lower( replace( v_all_ids(i) ,' ','_') ) );
|
|
END LOOP;
|
|
|
|
-- embedding
|
|
BEGIN
|
|
FORALL i IN 1 .. v_all_ids.COUNT
|
|
INSERT INTO tb_26ai_auto_load_vector (
|
|
ID, FILE_NAME, CREATED_DATE, CREATED_BY,
|
|
EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE,
|
|
attr1, attr2, attr3, attr4, attr5, attr6, attr7,
|
|
attr8, attr9, attr10, attr11, attr12, attr13,
|
|
attr14, attr15, attr16
|
|
)
|
|
SELECT
|
|
v_session_id,
|
|
lower(replace( v_all_ids(i) ,' ','_')),
|
|
CURRENT_TIMESTAMP,
|
|
'admin',
|
|
rownum embed_id,
|
|
null text_chunk,
|
|
t.vec,
|
|
p_embedding_name,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
p_mimetype
|
|
FROM (
|
|
select DBMS_VECTOR.UTL_TO_EMBEDDING(
|
|
to_blob(bfilename('DATA_PUMP_DIR', lower( replace( v_all_ids(i) ,' ','_') ) ) ) ,
|
|
'image',
|
|
JSON('{
|
|
"provider": "OCIGenAI",
|
|
"credential_name": "OCI_CRED",
|
|
"url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText",
|
|
"model": "cohere.embed-v4.0"}')
|
|
) vec) t;
|
|
|
|
-- Marca batch de PDF como processado
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'P', -- processado
|
|
dt_end_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S';
|
|
|
|
COMMIT;
|
|
EXCEPTION
|
|
WHEN OTHERS THEN
|
|
|
|
v_error := SQLERRM;
|
|
|
|
INSERT INTO tb_26ai_auto_load_debug
|
|
VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE);
|
|
|
|
-- Marca batch de PDF como erro
|
|
UPDATE tb_26ai_auto_load
|
|
SET status = 'E', -- erro
|
|
dt_end_process = SYSDATE
|
|
WHERE worker_id = p_worker_id
|
|
and status = 'S';
|
|
COMMIT;
|
|
END;
|
|
|
|
-- eliminacao dos arquivos temporarianete gravados paras DATA_PUMP_DIR
|
|
FOR i IN 1 .. v_all_ids.COUNT LOOP
|
|
DBMS_CLOUD.DELETE_FILE( directory_name => 'DATA_PUMP_DIR',
|
|
file_name => lower( replace( v_all_ids(i) ,' ','_') ) );
|
|
END LOOP;
|
|
|
|
|
|
end if; -- jpg
|
|
|
|
COMMIT;
|
|
|
|
END;
|
|
|
|
|
|
PROCEDURE proc_remove_jobs
|
|
AS
|
|
BEGIN
|
|
-- nao pode fazer pelo numero de jobs existentes pois senao a eliminacao seria falha:
|
|
-- cada job tem um padrao de ome associado ao worker id, e nao a contabilizacao que pode ter gaps
|
|
-- ajustar de acordo com maximo permitido pelo item de definicao de novos jobs
|
|
FOR i IN 1..300 LOOP
|
|
begin
|
|
DBMS_SCHEDULER.STOP_JOB( job_name => 'JOB_WORKER_' || i );
|
|
exception
|
|
when others then null;
|
|
end;
|
|
|
|
begin
|
|
DBMS_SCHEDULER.DROP_JOB( job_name => 'JOB_WORKER_' || i );
|
|
exception
|
|
when others then null;
|
|
end;
|
|
|
|
|
|
update tb_26ai_auto_load
|
|
set status = null, worker_id = null
|
|
where status = 'R'; -- estava reservado, mas com remocao do job volta pra status null sem worker id definido
|
|
|
|
commit;
|
|
|
|
END LOOP;
|
|
END;
|
|
|
|
|
|
PROCEDURE proc_add_jobs(p_limit in number,
|
|
p_total_jobs in number,
|
|
p_mimetype in varchar2,
|
|
p_embedding_name in varchar2 default 'COHERE')
|
|
AS
|
|
v_job_count number;
|
|
v_has number;
|
|
BEGIN
|
|
select count(1)
|
|
into v_job_count
|
|
from user_scheduler_jobs
|
|
where JOB_NAME like 'JOB_WORKER_%';
|
|
|
|
-- if p_total_jobs <= v_job_count or p_total_jobs is null then
|
|
|
|
--raise_application_error(-20002,'The number of scheduler jobs must be greater than what already exists');
|
|
|
|
-- else
|
|
|
|
|
|
FOR i IN 1..p_total_jobs LOOP
|
|
|
|
select count(1)
|
|
into v_has
|
|
from user_scheduler_jobs
|
|
where JOB_NAME = 'JOB_WORKER_' || i;
|
|
|
|
if v_has = 0 then -- nao existe com worker id "i"
|
|
|
|
DBMS_SCHEDULER.CREATE_JOB(
|
|
job_name => 'JOB_WORKER_' || i,
|
|
job_type => 'PLSQL_BLOCK',
|
|
job_action => 'BEGIN pkg_26ai_auto_load.proc_embed_files(p_limit=>' || p_limit || ',p_worker_id=>' || i || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;',
|
|
start_date => SYSTIMESTAMP,
|
|
repeat_interval => 'FREQ=SECONDLY; INTERVAL=10;',
|
|
enabled => TRUE
|
|
);
|
|
|
|
else -- cria-se um novo job alem do ultimo
|
|
|
|
DBMS_SCHEDULER.CREATE_JOB(
|
|
job_name => 'JOB_WORKER_' || to_char(i+v_job_count),
|
|
job_type => 'PLSQL_BLOCK',
|
|
job_action => 'BEGIN pkg_26ai_auto_load.proc_embed_files(p_limit=>' || p_limit || ',p_worker_id=>' || to_char(i+v_job_count) || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;',
|
|
start_date => SYSTIMESTAMP,
|
|
repeat_interval => 'FREQ=SECONDLY; INTERVAL=10;',
|
|
enabled => TRUE
|
|
);
|
|
|
|
end if;
|
|
|
|
END LOOP;
|
|
|
|
|
|
-- end if;
|
|
|
|
END;
|
|
|
|
end;
|
|
/ |