Files
26ai_mvp_poc/pkg_26ai_traffic_load.plb
2026-05-08 13:08:08 +00:00

534 lines
20 KiB
Plaintext

create or replace package body pkg_26ai_TRAFFIC_LOAD
as
/*
Criado por: fernando.leal@oracle.com
Data: Abril/2026
Objetivo: Identificar informacoes de imagens como placas, modelo do veiculo e infracoes (uso de celular)
v1 - Tarffic Load - leal
*/
--
-- chamada a API do Vision. Definir modo de extracao com p_feature_type
--
function fnc_26ai_traffic_vision( p_base64_image in clob,
p_feature_type in varchar2)
return clob
--
-- Visio: https://docs.oracle.com/pt-br/solutions/ai-vision-extract-data/index.html#GUID-A4FD65D0-BF62-472B-B4C7-0ADF5425A566
--
as
/*
p_feature_type: https://docs.oracle.com/en-us/iaas/api/#/en/vision/20220125/datatypes/ImageFeature
IMAGE_CLASSIFICATION: Label the image.
OBJECT_DETECTION: Identify objects in the image with bounding boxes.
TEXT_DETECTION: Recognize text at the word and line level.
FACE_DETECTION: Identify faces in the image with bounding boxes and face landmarks.
*/
v_endpoint varchar2(500) := 'https://vision.aiservice.us-chicago-1.oci.oraclecloud.com/20220125/actions/analyzeImage';
request_json CLOB;
l_response_body clob;
begin
request_json := to_clob('{
"compartmentId": "' || g_comp_id || '",
"image": {
"source":"INLINE",
"data":"' || p_base64_image || '"
},
"features":[
{
"featureType":"' || p_feature_type || '",
"maxResults": 1
}
]
}' );
-- Definir os cabeçalhos da requisição
APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).NAME := 'Content-Type';
APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).VALUE := 'application/json';
APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).NAME := 'Accept';
APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).VALUE := 'application/json';
-- Faça a chamada POST usando APEX_WEB_SERVICE e a credencial OCI
l_response_body := APEX_WEB_SERVICE.make_rest_request(
p_url => v_endpoint,
p_http_method => 'POST',
p_body => request_json,
p_credential_static_id => 'apex_cred'
);
return l_response_body;
end;
--
-- funcao de rag para extracao de informacoes da imagem
--
function fnc_26ai_traffic_rag (p_base64_image IN clob,
p_oci_cred IN VARCHAR2 )
return clob
as
-- modelos: https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm
-- cuidado com pre requisito (1)
gen_ai_endpoint varchar2(500) := 'https://inference.generativeai.us-chicago-1.oci.oraclecloud.com';
gen_ai_model varchar2(500) := 'meta.llama-4-maverick-17b-128e-instruct-fp8';
chat_resp dbms_cloud_types.RESP;
request_json_part1 CLOB;
request_json_part2 CLOB;
request_body BLOB;
BEGIN
-- create temp blobs
dbms_lob.createtemporary(request_body, FALSE);
request_json_part1 := to_clob(
'{
"compartmentId": "' || g_comp_id || '",
"servingMode":
{
"modelId": "' || gen_ai_model || '",
"servingType": "ON_DEMAND"
}
,
"chatRequest": {
"apiFormat": "GENERIC",
"messages": [
{
"role": "USER",
"content": [
{
"type": "TEXT",
"text": "' || 'Gere um JSON com a placa do veiculo, modelo do veiculo e se motorista estiver visivel com uso celular na direção, aponte a infração. Exemplo de saida: {placa:XXXX,modelo:XXXXXXXXXXXXXXXXX,infracao:XXXXXXXXXXXXXX}. Retorne apenas o JSON, sem nenhuma mensagem de introducao nem de explicacao.' || '"
},
{
"type": "IMAGE",
"imageUrl": {
"url": "data:image/' || 'png' || ';base64,');
request_json_part2 := to_clob('"
}
}
]
}
],
"temperature": 0.4,
"numGenerations": 5,
"topK": 1
}
}');
-- append part1 json to request blob
dbms_lob.append(request_body, apex_util.clob_to_blob(p_clob => request_json_part1,p_charset => 'AL32UTF8'));
-- append base64 image to request blob
dbms_lob.append(request_body, apex_util.clob_to_blob(p_clob => p_base64_image,p_charset => 'AL32UTF8'));
-- append part2 json to request blob
dbms_lob.append(request_body, apex_util.clob_to_blob(p_clob => request_json_part2,p_charset => 'AL32UTF8'));
-- call Gen AI Chat
chat_resp := dbms_cloud.send_request(
credential_name => p_oci_cred,
uri => gen_ai_endpoint || '/20231130/actions/chat',
method => dbms_cloud.METHOD_POST,
body => request_body
);
-- clear temp blobs
dbms_lob.freetemporary(request_body);
RETURN json_value( dbms_cloud.get_response_text(chat_resp),'$.chatResponse.choices[0].message.content[0].text') ;
END;
--
-- Rotina que define a lista de dados a serem vetorizados
-- Define-se o tipo de embedding para que outros jobs ja existentes possam continuar execucao sem impacto de novas cargas, e assim, testar novos embeddings
--
PROCEDURE prc_refresh_files(p_oci_cred IN VARCHAR2 default 'OCI_CRED',
p_bucket in varchar2 default null,
p_mimetype in varchar2 default 'PNG',
p_embedding_name in varchar2 default 'COHERE')
as
-- ao levar para ambiente de cliente, definir parametro para p_bucket
-- default: associado ao bucket de testes - Leal 17-04-26
v_bucket varchar2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/p/cCeVS9davcdjSieWS1H4JOkJs51Ae_-roo4Cr9DGMCE0A7tmx3cHs60ex75D-BX7/n/idi1o0a010nx/b/bucket-public-sector/o/'; -- := p_bucket
begin
-- apenas processar arquivos nao existentes no log de controle tb_26ai_TRAFFIC
-- tabela de controle dos arquivos que devem ser processados
-- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquivos da fila
-- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina proc_process_files
-- (3) os arquivos com status "P" foram vetorizados pela rotina proc_process_files
insert into tb_26ai_traffic(dt_ref,
object_name,
status,
worker_id,
bytes,
created,
last_modified,
EMBEDDING_NAME,
EMBEDDING_MIMETYPE,
IMAGE_BASE64)
SELECT sysdate,
object_name , -- cuidado: caracteres especiais tem tratamento de acesso para object storage
null,
null,
BYTES,
CREATED,
LAST_MODIFIED,
p_embedding_name,
p_mimetype,
APEX_WEB_SERVICE.BLOB2CLOBBASE64(
DBMS_CLOUD.GET_OBJECT(
credential_name => p_oci_cred,
object_uri => v_bucket || object_name
),'N','N' )
FROM table(dbms_cloud.list_objects(credential_name => p_oci_cred,
location_uri => v_bucket)) mod
WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN (
SELECT UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8')
FROM tb_26ai_TRAFFIC)
AND ( UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpg'
or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpeg'
or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%png')
order by CREATED;
commit;
end;
--
-- Isola de modo unico, por worker id (job), o batch de dados que deverao ser processados. Status definido em R (Reservado)
-- Nao é chamado diretamente, mas faz parte da rotina de embedding
--
PROCEDURE prc_reserve_files (
p_limit IN NUMBER,
p_worker_id IN VARCHAR2, -- sera setado via job para definir que este bloco de dados sera usado pelo job N
p_oci_cred IN VARCHAR2 default 'OCI_CRED',
p_bucket in varchar2 ,
p_docs OUT SYS.ODCIVARCHAR2LIST,
p_mimetype in varchar2 default 'PNG',
p_embedding_name in varchar2 default 'COHERE'
)
IS
BEGIN
-- inicialização obrigatória
p_docs := SYS.ODCIVARCHAR2LIST();
FOR r1 IN (
SELECT object_name
FROM tb_26ai_TRAFFIC
WHERE status IS NULL
and ROWNUM <= p_limit -- CUIDADO: requer ajustes de acordo com tamanho dos dados e servidor
and EMBEDDING_NAME = p_embedding_name
and EMBEDDING_MIMETYPE = p_mimetype
FOR UPDATE SKIP LOCKED
) LOOP
UPDATE tb_26ai_TRAFFIC
SET status = 'R', -- reserved
worker_id = p_worker_id -- sera setado via job para definir que este bloco de dados sera usado pelo job N
WHERE object_name = r1.object_name
and EMBEDDING_NAME = p_embedding_name
and EMBEDDING_MIMETYPE = p_mimetype
and status IS NULL;
p_docs.EXTEND;
p_docs(p_docs.COUNT) := r1.object_name;
END LOOP;
commit;
END;
--
-- Rotina principal de Embedding
-- Status de Reserva (R) torna se Started (S)
-- Se concluir com sucesso, Started (S) torna se Processado (P)
-- Senao, torna-se Error (E)
--
-- Para usar ONNX, importe previamente o ONNX ao banco com comando abaixo. Em seguida, ajuste string de uso, com nome do modelo, nas linhas de codigo do inicio desta rotina.
--
/* -- importacao do modelo ao banco
BEGIN
DBMS_VECTOR.LOAD_ONNX_MODEL(directory=>'DATA_PUMP_DIR',
file_name=>'clip-vit-large-patch14_img.onnx',
model_name=>'OPENAI_CLIP_MULTI_IMG',
metadata=>JSON('{"function" : "embedding", "embeddingOutput":"embedding", "input": {"input": ["DATA"]}}') );
END;
*/
--
-- Pre requisito: create sequence seq_26ai_TRAFFIC MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ;
--
--
PROCEDURE proc_process_files(p_limit in number default 10,
p_worker_id in number,
p_stop_process_list in varchar2 default 'N',
p_mimetype in varchar2 default 'PNG',
p_embedding_name in varchar2 default 'COHERE')
AS
v_error CLOB;
v_session_id NUMBER;
v_dt_start TIMESTAMP;
v_oci_cred VARCHAR2(20) := 'OCI_CRED';
v_bucket VARCHAR2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/p/cCeVS9davcdjSieWS1H4JOkJs51Ae_-roo4Cr9DGMCE0A7tmx3cHs60ex75D-BX7/n/idi1o0a010nx/b/bucket-public-sector/o/';
l_docs SYS.ODCIVARCHAR2LIST;
-- CREATE OR REPLACE TYPE t_audio_id_list AS TABLE OF VARCHAR2(600);
v_all_ids SYS.ODCIVARCHAR2LIST;
v_json_embedding varchar2(2000);
--
-- CUIDADI: geracao de vetores ira onerar tempo de processamento das imagens
--
v_call_vector_embedding varchar2(1) := 'N';
v_call_rag varchar2(1) := 'Y';
BEGIN
IF p_stop_process_list != 'N' THEN
RETURN;
END IF;
if p_embedding_name = 'COHERE' then
v_json_embedding := '{"provider": "OCIGenAI","credential_name": "OCI_CRED","url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText","batch_size": 50,"model": "cohere.embed-v4.0"}';
elsif p_embedding_name = 'VLLM' then
v_json_embedding := '{"provider": "openai","url": "https://hub-gpus.DOMINIO.com.br/embed/v1/embeddings","host":"local","batch_size": 100,"model": "Qwen/Qwen3-Embedding-4B"}';
elsif p_embedding_name = 'OPENAI' then
v_json_embedding := '{"provider" : "openai","credential_name" : "CRED_OPENAI", "url":"https://api.openai.com/v1/chat/completions", "model" : "gpt-4.1-mini" }';
elsif p_embedding_name = 'ONNX-E5' then
v_json_embedding := '{"provider":"database", "model":"MULTILINGUAL_E5_BASE"}';
elsif p_embedding_name = 'ONNX-VIT' then
v_json_embedding := '{"provider":"database", "model":"VIT_BASE_PATCH16_224"}';
end if;
--
-- rotina que reserva arquivos de modo exclusivo, permitindo uso de scheduler paralelos no banco
-- objetivo: embedding em sessoes paralelas do banco para diminui tempo de carga
-- (1) deve ser definido um valor adequado de arquivos por job, definido no limite de linhas (limit)
-- (2) cada job tem seu worker id definido pelo proprio scheduler
--
prc_reserve_files(
p_limit => p_limit,
p_worker_id=>p_worker_id,
p_oci_cred=> v_oci_cred,
p_bucket=> v_bucket,
p_docs=>l_docs,
p_mimetype => p_mimetype,
p_embedding_name => p_embedding_name ) ;
v_dt_start := CURRENT_TIMESTAMP;
-- create sequence seq_26ai_TRAFFIC MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ;
v_session_id := seq_26ai_TRAFFIC.nextval;
SELECT object_name
BULK COLLECT INTO v_all_ids
FROM (
SELECT UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') object_name
FROM tb_26ai_TRAFFIC src
WHERE worker_id = p_worker_id
and status = 'R'
and EMBEDDING_NAME = p_embedding_name
and EMBEDDING_MIMETYPE = p_mimetype
);
-- Marca batch de processamento como Iniciado
UPDATE tb_26ai_TRAFFIC
SET status = 'S', -- started
dt_start_process = SYSDATE
WHERE worker_id = p_worker_id
and status = 'R'
and EMBEDDING_NAME = p_embedding_name
and EMBEDDING_MIMETYPE = p_mimetype;
COMMIT;
-- embedding
BEGIN
if v_call_vector_embedding = 'Y' then
FORALL i IN 1 .. v_all_ids.COUNT
INSERT INTO tb_26ai_TRAFFIC_vector (
ID, FILE_NAME, CREATED_DATE, CREATED_BY,
EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, MIMETYPE
)
SELECT
v_session_id,
lower(replace( v_all_ids(i) ,' ','_')),
CURRENT_TIMESTAMP,
'admin',
rownum embed_id,
null text_chunk,
t.vec,
p_embedding_name,
p_mimetype
FROM (
select DBMS_VECTOR.UTL_TO_EMBEDDING(
DBMS_CLOUD.GET_OBJECT(
credential_name => v_oci_cred,
object_uri => v_bucket || v_all_ids(i) ),
'image',
JSON('{
"provider": "OCIGenAI",
"credential_name": "OCI_CRED",
"url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText",
"model": "cohere.embed-v4.0"}')
) vec) t;
end if; -- v_call_vector_embedding
-- Marca batch omo processado e gera analise da imagem com RAG
if v_call_rag = 'Y' then
UPDATE tb_26ai_TRAFFIC
SET status = 'P', -- processado
dt_end_process = SYSDATE,
JSON_DATA = fnc_26ai_traffic_rag (p_base64_image => IMAGE_BASE64, p_oci_cred => v_oci_cred )
WHERE worker_id = p_worker_id
and status = 'S';
else -- Marca batch omo processado
UPDATE tb_26ai_TRAFFIC
SET status = 'P', -- processado
dt_end_process = SYSDATE
WHERE worker_id = p_worker_id
and status = 'S';
end if;
COMMIT;
EXCEPTION
WHEN OTHERS THEN
v_error := SQLERRM;
INSERT INTO tb_26ai_TRAFFIC_debug
VALUES ('JOB_TRAFFIC_WORKER_' || p_worker_id , v_error, null, SYSDATE);
-- Marca batch de PDF como erro
UPDATE tb_26ai_TRAFFIC
SET status = 'E', -- erro
dt_end_process = SYSDATE
WHERE worker_id = p_worker_id
and status = 'S';
COMMIT;
END;
COMMIT;
END;
PROCEDURE proc_remove_jobs
AS
BEGIN
-- nao pode fazer pelo numero de jobs existentes pois senao a eliminacao seria falha:
-- cada job tem um padrao de ome associado ao worker id, e nao a contabilizacao que pode ter gaps
-- ajustar de acordo com maximo permitido pelo item de definicao de novos jobs
FOR i IN 1..300 LOOP
begin
DBMS_SCHEDULER.STOP_JOB( job_name => 'JOB_TRAFFIC_WORKER_' || i );
exception
when others then null;
end;
begin
DBMS_SCHEDULER.DROP_JOB( job_name => 'JOB_TRAFFIC_WORKER_' || i );
exception
when others then null;
end;
update tb_26ai_TRAFFIC
set status = null, worker_id = null
where status = 'R'; -- estava reservado, mas com remocao do job volta pra status null sem worker id definido
commit;
END LOOP;
END;
PROCEDURE proc_add_jobs(p_limit in number,
p_total_jobs in number,
p_mimetype in varchar2,
p_embedding_name in varchar2 default 'COHERE')
AS
v_job_count number;
v_has number;
BEGIN
select count(1)
into v_job_count
from user_scheduler_jobs
where JOB_NAME like 'JOB_TRAFFIC_WORKER_%';
-- if p_total_jobs <= v_job_count or p_total_jobs is null then
--raise_application_error(-20002,'The number of scheduler jobs must be greater than what already exists');
-- else
FOR i IN 1..p_total_jobs LOOP
select count(1)
into v_has
from user_scheduler_jobs
where JOB_NAME = 'JOB_TRAFFIC_WORKER_' || i;
if v_has = 0 then -- nao existe com worker id "i"
DBMS_SCHEDULER.CREATE_JOB(
job_name => 'JOB_TRAFFIC_WORKER_' || i,
job_type => 'PLSQL_BLOCK',
job_action => 'BEGIN pkg_26ai_TRAFFIC_LOAD.proc_process_files(p_limit=>' || p_limit || ',p_worker_id=>' || i || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;',
start_date => SYSTIMESTAMP,
repeat_interval => 'FREQ=SECONDLY; INTERVAL=2;',
enabled => TRUE
);
else -- cria-se um novo job alem do ultimo
DBMS_SCHEDULER.CREATE_JOB(
job_name => 'JOB_TRAFFIC_WORKER_' || to_char(i+v_job_count),
job_type => 'PLSQL_BLOCK',
job_action => 'BEGIN pkg_26ai_TRAFFIC_LOAD.proc_process_files(p_limit=>' || p_limit || ',p_worker_id=>' || to_char(i+v_job_count) || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;',
start_date => SYSTIMESTAMP,
repeat_interval => 'FREQ=SECONDLY; INTERVAL=2;',
enabled => TRUE
);
end if;
END LOOP;
-- end if;
END;
end;
/