commit dc6c1662edd87228b19b853e04db1fc60127e2e3 Author: flealdba Date: Fri May 8 13:08:08 2026 +0000 Upload files to "/" diff --git a/biu_tb_26ai_tech.sql b/biu_tb_26ai_tech.sql new file mode 100644 index 0000000..93a2c9c --- /dev/null +++ b/biu_tb_26ai_tech.sql @@ -0,0 +1,8 @@ +create or replace TRIGGER "BIU_TB_26AI_TECH" +before insert or update +on TB_26AI_TECH +for each row +begin + :new.ID_INSIGHT := SEQ_26AI_TECH.NEXTVAL; +end; +/ \ No newline at end of file diff --git a/biu_tb_26ai_tech_v2.sql b/biu_tb_26ai_tech_v2.sql new file mode 100644 index 0000000..5b96cfa --- /dev/null +++ b/biu_tb_26ai_tech_v2.sql @@ -0,0 +1,8 @@ +create or replace TRIGGER "BIU_TB_26AI_TECH_V2" +before insert or update +on TB_26AI_TECH_V2 +for each row +begin + :new.ID_INSIGHT := SEQ_26AI_TECH.NEXTVAL; +end; +/ \ No newline at end of file diff --git a/pck_26ai_apis.plb b/pck_26ai_apis.plb new file mode 100644 index 0000000..6b98667 --- /dev/null +++ b/pck_26ai_apis.plb @@ -0,0 +1,275 @@ +create or replace package body pck_26ai_apis +as + + procedure api_text2speech (p_text in varchar2, + p_filename in varchar2, + p_bucket in varchar2 , + p_comp_id in varchar2, + p_credential in varchar2 default 'OCI_CRED', + p_speaker in varchar2 default 'Felix', + p_language_code in varchar2 default 'pt-BR') + as + -- + -- Gerar audio a partir de TXT + -- + -- API para gerar speech do resultado: https://docs.oracle.com/en-us/iaas/api/#/en/speech/20220101/SynthesizeSpeech/SynthesizeSpeech + -- https://apexapps.oracle.com/pls/apex/r/dbpm/livelabs/run-workshop?p210_wid=3135&p210_wec=&session=107708964662539 + -- + -- Text to Speech is only available in the US West (Phoenix) commercial region. + -- https://docs.oracle.com/en-us/iaas/Content/speech/using/speech.htm + -- Text to speech supports maximum 10000 characters per request. + -- + -- Pre requisito: https://docs.oracle.com/en-us/iaas/Content/speech/using/policies.htm + -- Policy: ai-service-speech-family + -- + -- ALLOW GROUP TO USE ai-services IN COMPARTMENT autonomous-db-compartment + -- + -- + -- URL da API OCI Synthesize Speech + l_url VARCHAR2(4000) := 'https://speech.aiservice.us-phoenix-1.oci.oraclecloud.com/20220101/actions/synthesizeSpeech'; + + -- definicao do object storage para armazenar o MP3 + -- especifique com / no final + v_object_storage varchar2(500) := p_bucket; + + -- Variáveis para a requisição e resposta + l_request_body CLOB; + l_request_blob BLOB; + l_response_body BLOB; + + begin + -- Criar o JSON do corpo da requisição + l_request_body := '{ + "audioConfig": { + "configType": "BASE_AUDIO_CONFIG" + }, + "compartmentId": "' || p_comp_id || '", + "configuration": { + "modelDetails": { + "modelName": "TTS_2_NATURAL", + "languageCode":"' || p_language_code || '", + "voiceId": "' || p_speaker || '" + }, + "modelFamily": "ORACLE", + "speechSettings": { + "outputFormat": "MP3", + "sampleRateInHz": 23600, + "textType": "TEXT" + } + }, + "isStreamEnabled": false, + "text": "' || replace(replace(replace(replace(p_text, + chr(13),''), + chr(10),''), + '"',''), + '\n','') || '" }'; + + l_request_blob := apex_util.clob_to_blob(p_clob => l_request_body,p_charset => 'AL32UTF8'); + + -- Limpar os cabeçalhos antes de definir novos + APEX_WEB_SERVICE.CLEAR_REQUEST_HEADERS; + + -- Definir os cabeçalhos da requisição + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).NAME := 'Content-Type'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).VALUE := 'application/json'; + + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).NAME := 'Accept'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).VALUE := 'audio/mpeg'; + + -- Chamar a API REST da OCI + l_response_body := APEX_WEB_SERVICE.MAKE_REST_REQUEST_B( + p_url => l_url, + p_http_method => 'POST', + p_body_blob => l_request_blob, + p_credential_static_id => 'apex_cred' + ); + + -- Salvar o áudio no Object Storage + DBMS_CLOUD.PUT_OBJECT( + credential_name => p_credential, + object_uri => v_object_storage || p_filename, + contents => l_response_body + ); + + end; + + -- + -- Visio: https://docs.oracle.com/pt-br/solutions/ai-vision-extract-data/index.html#GUID-A4FD65D0-BF62-472B-B4C7-0ADF5425A566 + -- + function api_visio( p_id in number, p_feature_type in varchar2 , p_comp_id in varchar2) + return clob + as + /* + p_feature_type: https://docs.oracle.com/en-us/iaas/api/#/en/vision/20220125/datatypes/ImageFeature + + IMAGE_CLASSIFICATION: Label the image. + OBJECT_DETECTION: Identify objects in the image with bounding boxes. + TEXT_DETECTION: Recognize text at the word and line level. + FACE_DETECTION: Identify faces in the image with bounding boxes and face landmarks. + + */ + base64_image CLOB; + v_endpoint varchar2(500) := 'https://vision.aiservice.us-chicago-1.oci.oraclecloud.com/20220125/actions/analyzeImage'; + request_json CLOB; + l_response_body clob; + + begin + + select APEX_WEB_SERVICE.BLOB2CLOBBASE64( FILE_BLOB ,'N','N' ) + into base64_image + from TB_26AI_MANUFATURA --TB_26AI_FINANCE + where id = p_id; + + request_json := to_clob('{ + "compartmentId": "' || p_comp_id || '", + "image": { + "source":"INLINE", + "data":"' || base64_image || '" + }, + "features":[ + { + "featureType":"' || p_feature_type || '", + "maxResults": 5 + } + ] + }' ); + + -- Definir os cabeçalhos da requisição + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).NAME := 'Content-Type'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).VALUE := 'application/json'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).NAME := 'Accept'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).VALUE := 'application/json'; + + -- Faça a chamada POST usando APEX_WEB_SERVICE e a credencial OCI + l_response_body := APEX_WEB_SERVICE.make_rest_request( + p_url => v_endpoint, + p_http_method => 'POST', + p_body => request_json, + p_credential_static_id => 'apex_cred' + ); + + return l_response_body; + + end ; + + -- + -- Document Understanding: https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ + -- + function api_doc_understanding(p_id in number, p_feature_type in varchar2 , p_comp_id in varchar2) + return clob + as + /* + p_feature_type: https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/datatypes/DocumentClassificationFeature + + DOCUMENT_CLASSIFICATION + TABLE_EXTRACTION + TEXT_EXTRACTION + LANGUAGE_CLASSIFICATION + KEY_VALUE_EXTRACTION + + */ + v_endpoint varchar2(500) := 'https://document.aiservice.sa-saopaulo-1.oci.oraclecloud.com/20221109/actions/analyzeDocument'; + request_json CLOB; + v_base64 CLOB; + l_response_body clob; + + begin + + select APEX_WEB_SERVICE.BLOB2CLOBBASE64( FILE_BLOB ,'N','N' ) + into v_base64 + from TB_26AI_MANUFATURA_CATALOGO + where id = p_id; + + + if upper(p_feature_type) = 'KEY_VALUE_EXTRACTION' then + request_json := to_clob('{ + "compartmentId": "' || p_comp_id || '", + "features": [ + { + "featureType": "' || upper(p_feature_type) || '", + "selectionMarkDetection": true + } + ], + "documentType": "INVOICE", + "document": { + "source": "INLINE", + "data": "' || v_base64 || '" + } + }'); + else + + request_json := to_clob('{ + "compartmentId": "' || p_comp_id || '", + "features": [ + { + "featureType": "' || upper(p_feature_type) || '", + "selectionMarkDetection": true + } + ], + "document": { + "source": "INLINE", + "data": "' || v_base64 || '" + } + }'); + end if; + + -- Definir os cabeçalhos da requisição + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).NAME := 'Content-Type'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).VALUE := 'application/json'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).NAME := 'Accept'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).VALUE := 'application/json'; + + -- Faça a chamada POST usando APEX_WEB_SERVICE e a credencial OCI + l_response_body := APEX_WEB_SERVICE.make_rest_request( + p_url => v_endpoint, + p_http_method => 'POST', + p_body => request_json, + p_credential_static_id => 'apex_cred' + ); + + return l_response_body; + + end; + + -- + -- Language: https://docs.oracle.com/en-us/iaas/api/#/en/language/20221001/ + -- + function api_translate(p_data_text in clob,p_source_code in varchar2 default 'en', p_target_code in varchar2 default 'pt-BR' , p_comp_id in varchar2 ) + return clob + as + /* + https://docs.oracle.com/en-us/iaas/api/#/en/language/20221001/BatchLanguageTranslation/BatchLanguageTranslation + */ + v_endpoint varchar2(500) := 'https://language.aiservice.sa-saopaulo-1.oci.oraclecloud.com/20221001/actions/batchLanguageTranslation'; + request_json CLOB; + l_response_body clob; + + begin + request_json := to_clob('{ + "compartmentId": "' || p_comp_id || '", + "documents":[{ + "key":"1", + "text":"' || p_data_text || '", + "languageCode":"' || p_source_code || '" }], + "targetLanguageCode":"' || p_target_code || '" + } + '); + + -- Definir os cabeçalhos da requisição + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).NAME := 'Content-Type'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).VALUE := 'application/json'; + + -- Faça a chamada POST usando APEX_WEB_SERVICE e a credencial OCI + l_response_body := APEX_WEB_SERVICE.make_rest_request( + p_url => v_endpoint, + p_http_method => 'POST', + p_body => request_json, + p_credential_static_id => 'apex_cred' + ); + + return json_value( l_response_body, '$.documents[0].translatedText'); + + end; + +end; -- package; +/ \ No newline at end of file diff --git a/pkg_26ai_auto_load.plb b/pkg_26ai_auto_load.plb new file mode 100644 index 0000000..3dff79a --- /dev/null +++ b/pkg_26ai_auto_load.plb @@ -0,0 +1,695 @@ +create or replace package body pkg_26ai_auto_load +as + +/* + + Criado por: fernando.leal@oracle.com + Data: Mar/2026 + Objetivo: demonstrar casos de Vetorizacao em massa com ONNX e modelos externos do Oracle AI Database 26ai + + v1 - Auto Load - leal + +*/ + + -- + -- Rotina que define a lista de dados a serem vetorizados + -- Define-se o tipo de embedding para que outros jobs ja existentes possam continuar execucao sem impacto de novas cargas, e assim, testar novos embeddings + -- + PROCEDURE prc_refresh_files(p_oci_cred IN VARCHAR2 default 'OCI_CRED', + p_bucket in varchar2 default null, + p_mimetype in varchar2 default 'PDF', + p_embedding_name in varchar2 default 'COHERE') + as + -- ao levar para ambiente de cliente, definir parametro para p_bucket + -- default: associado ao bucket de testes - Leal 25-03-26 + v_bucket varchar2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/n/idi1o0a010nx/b/bucket-database-26ai/o/'; -- := p_bucket + begin + + if p_mimetype = 'PDF' then + + -- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load + -- tabela de controle dos arquivos que devem ser processados + -- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquoivos da fila + -- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina PROC_EMBED_FILES + -- (3) os arquivos com status "P" foram vetorizados pela rotina PROC_EMBED_FILES + insert into tb_26ai_auto_load(dt_ref, + object_name, + status, + worker_id, + bytes, + created, + last_modified, + EMBEDDING_NAME, + EMBEDDING_MIMETYPE) + SELECT sysdate, + UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8'), -- cuidado: caracteres especiais tem tratamento de acesso para object storage + null, + null, + BYTES, + CREATED, + LAST_MODIFIED, + p_embedding_name, + p_mimetype + FROM DBMS_CLOUD.LIST_OBJECTS(p_oci_cred, v_bucket) + WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN ( + SELECT object_name + FROM TB_26ai_AUTO_LOAD + where EMBEDDING_MIMETYPE = 'PDF' + and EMBEDDING_NAME = p_embedding_name) + AND UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%pdf'; -- ajuste leal 250226 + end if; -- pdf + + + + if p_mimetype = 'CLOB' then + + + -- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load + insert into tb_26ai_auto_load(dt_ref, + object_name, + status, + worker_id, + bytes, + created, + last_modified, + EMBEDDING_NAME, + EMBEDDING_MIMETYPE) + SELECT sysdate, + src.p_partkey, + null, + null, -- inicia em null, mas o job faz asociacao ao seu ID em prc_reserve_files + dbms_lob.getlength( 'Name: ' || src.p_name || ' Color: ' || src.p_color || ' Size: ' || src.p_size || ' Type: ' || src.p_type || ' Container: ' || src.p_container ), + sysdate, + null, + p_embedding_name, + p_mimetype + FROM ssb.PART src + --WHERE src.dat_ref = p_data_id + where rownum<=5000 + and (src.p_partkey) NOT IN ( + SELECT al.object_name + FROM TB_26ai_AUTO_LOAD al + where EMBEDDING_MIMETYPE = 'CLOB' + and EMBEDDING_NAME = p_embedding_name); + end if; + + if p_mimetype = 'JPG' then + + -- apenas processar arquivos nao existentes no log de controle tb_26ai_auto_load + -- tabela de controle dos arquivos que devem ser processados + -- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquoivos da fila + -- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina PROC_EMBED_FILES + -- (3) os arquivos com status "P" foram vetorizados pela rotina PROC_EMBED_FILES + insert into tb_26ai_auto_load(dt_ref, + object_name, + status, + worker_id, + bytes, + created, + last_modified, + EMBEDDING_NAME, + EMBEDDING_MIMETYPE) + SELECT sysdate, + UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8'), -- cuidado: caracteres especiais tem tratamento de acesso para object storage + null, + null, + BYTES, + CREATED, + LAST_MODIFIED, + p_embedding_name, + p_mimetype + FROM DBMS_CLOUD.LIST_OBJECTS(p_oci_cred, v_bucket) + WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN ( + SELECT object_name + FROM TB_26ai_AUTO_LOAD + where EMBEDDING_MIMETYPE = 'JPG' + and EMBEDDING_NAME = p_embedding_name) + AND ( UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpg' + or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpeg' ) ; + end if; -- jpg + + + commit; + + end; + + + -- + -- Isola de modo unico, por worker id (job), o batch de dados que deverao ser processados. Status definido em R (Reservado) + -- Nao é chamado diretamente, mas faz parte da rotina de embedding + -- + PROCEDURE prc_reserve_files ( + p_limit IN NUMBER, + p_worker_id IN VARCHAR2, -- sera setado via job para definir que este bloco de dados sera usado pelo job N + p_oci_cred IN VARCHAR2 default 'OCI_CRED', + p_bucket in varchar2 , + p_docs OUT SYS.ODCIVARCHAR2LIST, + p_mimetype in varchar2 default 'PDF', + p_embedding_name in varchar2 default 'COHERE' + ) + IS + BEGIN + + -- inicialização obrigatória + p_docs := SYS.ODCIVARCHAR2LIST(); + + FOR r1 IN ( + SELECT object_name + FROM tb_26ai_auto_load + WHERE status IS NULL + and ROWNUM <= p_limit -- CUIDADO: requer ajustes de acordo com tamanho dos dados e servidor + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + FOR UPDATE SKIP LOCKED + ) LOOP + + UPDATE tb_26ai_auto_load + SET status = 'R', -- reserved + worker_id = p_worker_id -- sera setado via job para definir que este bloco de dados sera usado pelo job N + WHERE object_name = r1.object_name + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + and status IS NULL; + + p_docs.EXTEND; + p_docs(p_docs.COUNT) := r1.object_name; + + END LOOP; + + commit; + + END; + + + -- + -- Rotina principal de Embedding + -- Status de Reserva (R) torna se Started (S) + -- Se concluir com sucesso, Started (S) torna se Processado (P) + -- Senao, torna-se Error (E) + -- + -- Para usar ONNX, importe previamente o ONNX ao banco com comando abaixo. Em seguida, ajuste string de uso, com nome do modelo, nas linhas de codigo do inicio desta rotina. + -- + /* -- importacao do modelo ao banco + BEGIN + DBMS_VECTOR.LOAD_ONNX_MODEL(directory=>'DATA_PUMP_DIR', + file_name=>'clip-vit-large-patch14_img.onnx', + model_name=>'OPENAI_CLIP_MULTI_IMG', + metadata=>JSON('{"function" : "embedding", "embeddingOutput":"embedding", "input": {"input": ["DATA"]}}') ); + + END; + */ + -- + -- Pre requisito: create sequence seq_26ai_auto_load MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ; + -- + -- + PROCEDURE proc_embed_files(p_limit in number default 10, + p_worker_id in number, + p_stop_process_list in varchar2 default 'N', + p_mimetype in varchar2 default 'PDF', + p_embedding_name in varchar2 default 'COHERE') + AS + v_error CLOB; + v_session_id NUMBER; + v_dt_start TIMESTAMP; + v_oci_cred VARCHAR2(20) := 'OCI_CRED'; + v_bucket VARCHAR2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/n/idi1o0a010nx/b/bucket-database-26ai/o/'; + l_docs SYS.ODCIVARCHAR2LIST; + + -- CREATE OR REPLACE TYPE t_audio_id_list AS TABLE OF VARCHAR2(600); + v_all_ids SYS.ODCIVARCHAR2LIST; + v_json_embedding varchar2(2000); + BEGIN + + IF p_stop_process_list != 'N' THEN + RETURN; + END IF; + + if p_embedding_name = 'COHERE' then + v_json_embedding := '{"provider": "OCIGenAI","credential_name": "OCI_CRED","url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText","batch_size": 50,"model": "cohere.embed-v4.0"}'; + + elsif p_embedding_name = 'VLLM' then + v_json_embedding := '{"provider": "openai","url": "https://hub-gpus.DOMINIO.com.br/embed/v1/embeddings","host":"local","batch_size": 100,"model": "Qwen/Qwen3-Embedding-4B"}'; + + elsif p_embedding_name = 'OPENAI' then + v_json_embedding := '{"provider" : "openai","credential_name" : "CRED_OPENAI", "url":"https://api.openai.com/v1/chat/completions", "model" : "gpt-4.1-mini" }'; + + elsif p_embedding_name = 'ONNX-E5' then + v_json_embedding := '{"provider":"database", "model":"MULTILINGUAL_E5_BASE"}'; + + elsif p_embedding_name = 'ONNX-VIT' then + v_json_embedding := '{"provider":"database", "model":"VIT_BASE_PATCH16_224"}'; + + end if; + + -- + -- rotina que reserva arquivos de modo exclusivo, permitindo uso de scheduler paralelos no banco + -- objetivo: embedding em sessoes paralelas do banco para diminui tempo de carga + -- (1) deve ser definido um valor adequado de arquivos por job, definido no limite de linhas (limit) + -- (2) cada job tem seu worker id definido pelo proprio scheduler + -- + prc_reserve_files( + p_limit => p_limit, + p_worker_id=>p_worker_id, + p_oci_cred=> v_oci_cred, + p_bucket=> v_bucket, + p_docs=>l_docs, + p_mimetype => p_mimetype, + p_embedding_name => p_embedding_name ) ; + + v_dt_start := CURRENT_TIMESTAMP; + -- create sequence seq_26ai_auto_load MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ; + v_session_id := seq_26ai_auto_load.nextval; + + + if p_mimetype = 'CLOB' then + + -- Marca batch como Iniciado + UPDATE tb_26ai_auto_load + SET status = 'S', -- started + dt_start_process = SYSDATE + WHERE worker_id = p_worker_id + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + and status = 'R'; + COMMIT; + + -- embedding + BEGIN + + INSERT INTO tb_26ai_auto_load_vector ( + ID, FILE_NAME, CREATED_DATE, CREATED_BY, + EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, + attr1, attr2, attr3, attr4, attr5, attr6, attr7, + attr8, attr9, attr10, attr11, attr12, attr13, + attr14, attr15, attr16 + ) + SELECT + v_session_id, + src.p_partkey, + CURRENT_TIMESTAMP, + 'admin3', + et.embed_id, + et.text_chunk, + et.embed_vector, + p_embedding_name, + NULL, + src.p_name, + src.p_color, + src.p_type, + src.p_size, + src.p_container, + null, + null, + null, + null, + null, + null, + null, + null, + null, + p_mimetype + FROM SSB.PART src + CROSS JOIN TABLE( + DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS( + DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS( + dbms_vector_chain.utl_to_text( 'Name: ' || src.p_name || ' Color: ' || src.p_color || ' Size: ' || src.p_size || ' Type: ' || src.p_type || ' Container: ' || src.p_container ) , + JSON('{"by":"words","max":"300","split":"sentence","normalize":"all","overlap":"30"}') + ), + JSON(v_json_embedding) + ) + ) t + CROSS JOIN JSON_TABLE( + t.column_value, '$[*]' + COLUMNS ( + embed_id NUMBER PATH '$.embed_id', + text_chunk VARCHAR2(4000) PATH '$.embed_data', + embed_vector CLOB PATH '$.embed_vector' + ) + ) AS et + WHERE src.p_partkey IN ( SELECT object_name + FROM tb_26ai_auto_load src + WHERE worker_id = p_worker_id + and status = 'S' -- started + ); + --AND src.dat_ref = p_worker_id; + + -- Marca batch como processado + UPDATE tb_26ai_auto_load + SET status = 'P', -- processado + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + COMMIT; + + + EXCEPTION + WHEN OTHERS THEN + + v_error := SQLERRM; + + INSERT INTO tb_26ai_auto_load_debug + VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE); + + -- Marca batch como ERRO + UPDATE tb_26ai_auto_load + SET status = 'E', + dt_end_process = SYSDATE, + error_msg = v_error + WHERE worker_id = p_worker_id + and status = 'S'; + + COMMIT; + END; + + end if; -- clob + + if p_mimetype = 'PDF' then + + SELECT object_name + BULK COLLECT INTO v_all_ids + FROM ( + SELECT object_name + FROM tb_26ai_auto_load src + WHERE worker_id = p_worker_id + and status = 'R' + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + ); + + -- Marca batch de PDF como Iniciado + UPDATE tb_26ai_auto_load + SET status = 'S', -- started + dt_start_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'R' + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype; + + COMMIT; + + -- copia temporaria para banco + FOR i IN 1 .. v_all_ids.COUNT LOOP + DBMS_CLOUD.GET_OBJECT( + credential_name => v_oci_cred, + object_uri => v_bucket || UTL_URL.ESCAPE( v_all_ids(i) ,TRUE,'AL32UTF8') , + directory_name => 'DATA_PUMP_DIR', + file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); + END LOOP; + + -- embedding + BEGIN + FORALL i IN 1 .. v_all_ids.COUNT + INSERT INTO tb_26ai_auto_load_vector ( + ID, FILE_NAME, CREATED_DATE, CREATED_BY, + EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, + attr1, attr2, attr3, attr4, attr5, attr6, attr7, + attr8, attr9, attr10, attr11, attr12, attr13, + attr14, attr15, attr16 + ) + SELECT + v_session_id, + lower(replace( v_all_ids(i) ,' ','_')), + CURRENT_TIMESTAMP, + 'admin', + et.embed_id, + et.text_chunk, + et.embed_vector, + p_embedding_name, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + p_mimetype + FROM dual + CROSS JOIN TABLE( + DBMS_VECTOR_CHAIN.UTL_TO_EMBEDDINGS( + DBMS_VECTOR_CHAIN.UTL_TO_CHUNKS( + dbms_vector_chain.utl_to_text( to_blob(bfilename('DATA_PUMP_DIR', lower( replace( v_all_ids(i) ,' ','_') ) ) ) ), + JSON('{"by":"words","max":"' || 400 || '","split":"sentence","normalize":"all","overlap":' || 40 || '}') + ), + JSON(v_json_embedding) + ) + ) t + CROSS JOIN JSON_TABLE( + t.column_value, '$[*]' + COLUMNS ( + embed_id NUMBER PATH '$.embed_id', + text_chunk VARCHAR2(4000) PATH '$.embed_data', + embed_vector CLOB PATH '$.embed_vector' + ) + ) AS et; + + -- Marca batch de PDF como processado + UPDATE tb_26ai_auto_load + SET status = 'P', -- processado + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + + COMMIT; + EXCEPTION + WHEN OTHERS THEN + + v_error := SQLERRM; + + INSERT INTO tb_26ai_auto_load_debug + VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE); + + -- Marca batch de PDF como erro + UPDATE tb_26ai_auto_load + SET status = 'E', -- erro + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + COMMIT; + END; + + -- eliminacao dos arquivos temporarianete gravados paras DATA_PUMP_DIR + FOR i IN 1 .. v_all_ids.COUNT LOOP + DBMS_CLOUD.DELETE_FILE( directory_name => 'DATA_PUMP_DIR', + file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); + END LOOP; + + + end if; -- pdf + + + if p_mimetype = 'JPG' then + + SELECT object_name + BULK COLLECT INTO v_all_ids + FROM ( + SELECT object_name + FROM tb_26ai_auto_load src + WHERE worker_id = p_worker_id + and status = 'R' + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + ); + + -- Marca batch de PDF como Iniciado + UPDATE tb_26ai_auto_load + SET status = 'S', -- started + dt_start_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'R' + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype; + + COMMIT; + + -- copia temporaria para banco + FOR i IN 1 .. v_all_ids.COUNT LOOP + DBMS_CLOUD.GET_OBJECT( + credential_name => v_oci_cred, + object_uri => v_bucket || UTL_URL.ESCAPE( v_all_ids(i) ,TRUE,'AL32UTF8') , + directory_name => 'DATA_PUMP_DIR', + file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); + END LOOP; + + -- embedding + BEGIN + FORALL i IN 1 .. v_all_ids.COUNT + INSERT INTO tb_26ai_auto_load_vector ( + ID, FILE_NAME, CREATED_DATE, CREATED_BY, + EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, + attr1, attr2, attr3, attr4, attr5, attr6, attr7, + attr8, attr9, attr10, attr11, attr12, attr13, + attr14, attr15, attr16 + ) + SELECT + v_session_id, + lower(replace( v_all_ids(i) ,' ','_')), + CURRENT_TIMESTAMP, + 'admin', + rownum embed_id, + null text_chunk, + t.vec, + p_embedding_name, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + p_mimetype + FROM ( + select DBMS_VECTOR.UTL_TO_EMBEDDING( + to_blob(bfilename('DATA_PUMP_DIR', lower( replace( v_all_ids(i) ,' ','_') ) ) ) , + 'image', + JSON('{ + "provider": "OCIGenAI", + "credential_name": "OCI_CRED", + "url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText", + "model": "cohere.embed-v4.0"}') + ) vec) t; + + -- Marca batch de PDF como processado + UPDATE tb_26ai_auto_load + SET status = 'P', -- processado + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + + COMMIT; + EXCEPTION + WHEN OTHERS THEN + + v_error := SQLERRM; + + INSERT INTO tb_26ai_auto_load_debug + VALUES ('JOB_WORKER_' || p_worker_id , v_error, null, SYSDATE); + + -- Marca batch de PDF como erro + UPDATE tb_26ai_auto_load + SET status = 'E', -- erro + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + COMMIT; + END; + + -- eliminacao dos arquivos temporarianete gravados paras DATA_PUMP_DIR + FOR i IN 1 .. v_all_ids.COUNT LOOP + DBMS_CLOUD.DELETE_FILE( directory_name => 'DATA_PUMP_DIR', + file_name => lower( replace( v_all_ids(i) ,' ','_') ) ); + END LOOP; + + + end if; -- jpg + + COMMIT; + + END; + + + PROCEDURE proc_remove_jobs + AS + BEGIN + -- nao pode fazer pelo numero de jobs existentes pois senao a eliminacao seria falha: + -- cada job tem um padrao de ome associado ao worker id, e nao a contabilizacao que pode ter gaps + -- ajustar de acordo com maximo permitido pelo item de definicao de novos jobs + FOR i IN 1..300 LOOP + begin + DBMS_SCHEDULER.STOP_JOB( job_name => 'JOB_WORKER_' || i ); + exception + when others then null; + end; + + begin + DBMS_SCHEDULER.DROP_JOB( job_name => 'JOB_WORKER_' || i ); + exception + when others then null; + end; + + + update tb_26ai_auto_load + set status = null, worker_id = null + where status = 'R'; -- estava reservado, mas com remocao do job volta pra status null sem worker id definido + + commit; + + END LOOP; + END; + + + PROCEDURE proc_add_jobs(p_limit in number, + p_total_jobs in number, + p_mimetype in varchar2, + p_embedding_name in varchar2 default 'COHERE') + AS + v_job_count number; + v_has number; + BEGIN + select count(1) + into v_job_count + from user_scheduler_jobs + where JOB_NAME like 'JOB_WORKER_%'; + + -- if p_total_jobs <= v_job_count or p_total_jobs is null then + + --raise_application_error(-20002,'The number of scheduler jobs must be greater than what already exists'); + + -- else + + + FOR i IN 1..p_total_jobs LOOP + + select count(1) + into v_has + from user_scheduler_jobs + where JOB_NAME = 'JOB_WORKER_' || i; + + if v_has = 0 then -- nao existe com worker id "i" + + DBMS_SCHEDULER.CREATE_JOB( + job_name => 'JOB_WORKER_' || i, + job_type => 'PLSQL_BLOCK', + job_action => 'BEGIN pkg_26ai_auto_load.proc_embed_files(p_limit=>' || p_limit || ',p_worker_id=>' || i || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;', + start_date => SYSTIMESTAMP, + repeat_interval => 'FREQ=SECONDLY; INTERVAL=10;', + enabled => TRUE + ); + + else -- cria-se um novo job alem do ultimo + + DBMS_SCHEDULER.CREATE_JOB( + job_name => 'JOB_WORKER_' || to_char(i+v_job_count), + job_type => 'PLSQL_BLOCK', + job_action => 'BEGIN pkg_26ai_auto_load.proc_embed_files(p_limit=>' || p_limit || ',p_worker_id=>' || to_char(i+v_job_count) || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;', + start_date => SYSTIMESTAMP, + repeat_interval => 'FREQ=SECONDLY; INTERVAL=10;', + enabled => TRUE + ); + + end if; + + END LOOP; + + + -- end if; + + END; + +end; +/ \ No newline at end of file diff --git a/pkg_26ai_traffic_load.plb b/pkg_26ai_traffic_load.plb new file mode 100644 index 0000000..80ccd6a --- /dev/null +++ b/pkg_26ai_traffic_load.plb @@ -0,0 +1,534 @@ +create or replace package body pkg_26ai_TRAFFIC_LOAD +as + +/* + + + Criado por: fernando.leal@oracle.com + Data: Abril/2026 + Objetivo: Identificar informacoes de imagens como placas, modelo do veiculo e infracoes (uso de celular) + + v1 - Tarffic Load - leal + +*/ + + -- + -- chamada a API do Vision. Definir modo de extracao com p_feature_type + -- + function fnc_26ai_traffic_vision( p_base64_image in clob, + p_feature_type in varchar2) + return clob + -- + -- Visio: https://docs.oracle.com/pt-br/solutions/ai-vision-extract-data/index.html#GUID-A4FD65D0-BF62-472B-B4C7-0ADF5425A566 + -- + as + /* + p_feature_type: https://docs.oracle.com/en-us/iaas/api/#/en/vision/20220125/datatypes/ImageFeature + IMAGE_CLASSIFICATION: Label the image. + OBJECT_DETECTION: Identify objects in the image with bounding boxes. + TEXT_DETECTION: Recognize text at the word and line level. + FACE_DETECTION: Identify faces in the image with bounding boxes and face landmarks. + */ + v_endpoint varchar2(500) := 'https://vision.aiservice.us-chicago-1.oci.oraclecloud.com/20220125/actions/analyzeImage'; + request_json CLOB; + l_response_body clob; + begin + + request_json := to_clob('{ + "compartmentId": "' || g_comp_id || '", + "image": { + "source":"INLINE", + "data":"' || p_base64_image || '" + }, + "features":[ + { + "featureType":"' || p_feature_type || '", + "maxResults": 1 + } + ] + }' ); + + -- Definir os cabeçalhos da requisição + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).NAME := 'Content-Type'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(1).VALUE := 'application/json'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).NAME := 'Accept'; + APEX_WEB_SERVICE.G_REQUEST_HEADERS(2).VALUE := 'application/json'; + + -- Faça a chamada POST usando APEX_WEB_SERVICE e a credencial OCI + l_response_body := APEX_WEB_SERVICE.make_rest_request( + p_url => v_endpoint, + p_http_method => 'POST', + p_body => request_json, + p_credential_static_id => 'apex_cred' + ); + + return l_response_body; + + end; + + -- + -- funcao de rag para extracao de informacoes da imagem + -- + function fnc_26ai_traffic_rag (p_base64_image IN clob, + p_oci_cred IN VARCHAR2 ) + return clob + as + + -- modelos: https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm + -- cuidado com pre requisito (1) + gen_ai_endpoint varchar2(500) := 'https://inference.generativeai.us-chicago-1.oci.oraclecloud.com'; + gen_ai_model varchar2(500) := 'meta.llama-4-maverick-17b-128e-instruct-fp8'; + + chat_resp dbms_cloud_types.RESP; + + request_json_part1 CLOB; + request_json_part2 CLOB; + request_body BLOB; + BEGIN + -- create temp blobs + dbms_lob.createtemporary(request_body, FALSE); + + request_json_part1 := to_clob( + '{ + "compartmentId": "' || g_comp_id || '", + "servingMode": + { + "modelId": "' || gen_ai_model || '", + "servingType": "ON_DEMAND" + } + , + "chatRequest": { + "apiFormat": "GENERIC", + "messages": [ + { + "role": "USER", + "content": [ + { + "type": "TEXT", + "text": "' || 'Gere um JSON com a placa do veiculo, modelo do veiculo e se motorista estiver visivel com uso celular na direção, aponte a infração. Exemplo de saida: {placa:XXXX,modelo:XXXXXXXXXXXXXXXXX,infracao:XXXXXXXXXXXXXX}. Retorne apenas o JSON, sem nenhuma mensagem de introducao nem de explicacao.' || '" + }, + { + "type": "IMAGE", + "imageUrl": { + "url": "data:image/' || 'png' || ';base64,'); + + request_json_part2 := to_clob('" + } + } + ] + } + ], + "temperature": 0.4, + "numGenerations": 5, + "topK": 1 + } + }'); + + -- append part1 json to request blob + dbms_lob.append(request_body, apex_util.clob_to_blob(p_clob => request_json_part1,p_charset => 'AL32UTF8')); + + -- append base64 image to request blob + dbms_lob.append(request_body, apex_util.clob_to_blob(p_clob => p_base64_image,p_charset => 'AL32UTF8')); + + -- append part2 json to request blob + dbms_lob.append(request_body, apex_util.clob_to_blob(p_clob => request_json_part2,p_charset => 'AL32UTF8')); + + -- call Gen AI Chat + chat_resp := dbms_cloud.send_request( + credential_name => p_oci_cred, + uri => gen_ai_endpoint || '/20231130/actions/chat', + method => dbms_cloud.METHOD_POST, + body => request_body + ); + + -- clear temp blobs + dbms_lob.freetemporary(request_body); + + RETURN json_value( dbms_cloud.get_response_text(chat_resp),'$.chatResponse.choices[0].message.content[0].text') ; + END; + + + -- + -- Rotina que define a lista de dados a serem vetorizados + -- Define-se o tipo de embedding para que outros jobs ja existentes possam continuar execucao sem impacto de novas cargas, e assim, testar novos embeddings + -- + PROCEDURE prc_refresh_files(p_oci_cred IN VARCHAR2 default 'OCI_CRED', + p_bucket in varchar2 default null, + p_mimetype in varchar2 default 'PNG', + p_embedding_name in varchar2 default 'COHERE') + as + -- ao levar para ambiente de cliente, definir parametro para p_bucket + -- default: associado ao bucket de testes - Leal 17-04-26 + v_bucket varchar2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/p/cCeVS9davcdjSieWS1H4JOkJs51Ae_-roo4Cr9DGMCE0A7tmx3cHs60ex75D-BX7/n/idi1o0a010nx/b/bucket-public-sector/o/'; -- := p_bucket + begin + + -- apenas processar arquivos nao existentes no log de controle tb_26ai_TRAFFIC + -- tabela de controle dos arquivos que devem ser processados + -- (1) os nomes de arquivos sao unicos para processamento, por isso ha uma clausula not in que nao insere nomes de arquivos da fila + -- (2) os arquivos com status "NULL" ainda nao foram vetorizados pela rotina proc_process_files + -- (3) os arquivos com status "P" foram vetorizados pela rotina proc_process_files + insert into tb_26ai_traffic(dt_ref, + object_name, + status, + worker_id, + bytes, + created, + last_modified, + EMBEDDING_NAME, + EMBEDDING_MIMETYPE, + IMAGE_BASE64) + SELECT sysdate, + object_name , -- cuidado: caracteres especiais tem tratamento de acesso para object storage + null, + null, + BYTES, + CREATED, + LAST_MODIFIED, + p_embedding_name, + p_mimetype, + APEX_WEB_SERVICE.BLOB2CLOBBASE64( + DBMS_CLOUD.GET_OBJECT( + credential_name => p_oci_cred, + object_uri => v_bucket || object_name + ),'N','N' ) + FROM table(dbms_cloud.list_objects(credential_name => p_oci_cred, + location_uri => v_bucket)) mod + WHERE UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') NOT IN ( + SELECT UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') + FROM tb_26ai_TRAFFIC) + AND ( UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpg' + or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%jpeg' + or UTL_URL.ESCAPE( lower(object_name) ,TRUE,'AL32UTF8') like '%png') + order by CREATED; + + commit; + + end; + + + -- + -- Isola de modo unico, por worker id (job), o batch de dados que deverao ser processados. Status definido em R (Reservado) + -- Nao é chamado diretamente, mas faz parte da rotina de embedding + -- + PROCEDURE prc_reserve_files ( + p_limit IN NUMBER, + p_worker_id IN VARCHAR2, -- sera setado via job para definir que este bloco de dados sera usado pelo job N + p_oci_cred IN VARCHAR2 default 'OCI_CRED', + p_bucket in varchar2 , + p_docs OUT SYS.ODCIVARCHAR2LIST, + p_mimetype in varchar2 default 'PNG', + p_embedding_name in varchar2 default 'COHERE' + ) + IS + BEGIN + + -- inicialização obrigatória + p_docs := SYS.ODCIVARCHAR2LIST(); + + FOR r1 IN ( + SELECT object_name + FROM tb_26ai_TRAFFIC + WHERE status IS NULL + and ROWNUM <= p_limit -- CUIDADO: requer ajustes de acordo com tamanho dos dados e servidor + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + FOR UPDATE SKIP LOCKED + ) LOOP + + UPDATE tb_26ai_TRAFFIC + SET status = 'R', -- reserved + worker_id = p_worker_id -- sera setado via job para definir que este bloco de dados sera usado pelo job N + WHERE object_name = r1.object_name + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + and status IS NULL; + + p_docs.EXTEND; + p_docs(p_docs.COUNT) := r1.object_name; + + END LOOP; + + commit; + + END; + + + -- + -- Rotina principal de Embedding + -- Status de Reserva (R) torna se Started (S) + -- Se concluir com sucesso, Started (S) torna se Processado (P) + -- Senao, torna-se Error (E) + -- + -- Para usar ONNX, importe previamente o ONNX ao banco com comando abaixo. Em seguida, ajuste string de uso, com nome do modelo, nas linhas de codigo do inicio desta rotina. + -- + /* -- importacao do modelo ao banco + BEGIN + DBMS_VECTOR.LOAD_ONNX_MODEL(directory=>'DATA_PUMP_DIR', + file_name=>'clip-vit-large-patch14_img.onnx', + model_name=>'OPENAI_CLIP_MULTI_IMG', + metadata=>JSON('{"function" : "embedding", "embeddingOutput":"embedding", "input": {"input": ["DATA"]}}') ); + + END; + */ + -- + -- Pre requisito: create sequence seq_26ai_TRAFFIC MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ; + -- + -- + PROCEDURE proc_process_files(p_limit in number default 10, + p_worker_id in number, + p_stop_process_list in varchar2 default 'N', + p_mimetype in varchar2 default 'PNG', + p_embedding_name in varchar2 default 'COHERE') + AS + v_error CLOB; + v_session_id NUMBER; + v_dt_start TIMESTAMP; + v_oci_cred VARCHAR2(20) := 'OCI_CRED'; + v_bucket VARCHAR2(600) := 'https://objectstorage.sa-saopaulo-1.oraclecloud.com/p/cCeVS9davcdjSieWS1H4JOkJs51Ae_-roo4Cr9DGMCE0A7tmx3cHs60ex75D-BX7/n/idi1o0a010nx/b/bucket-public-sector/o/'; + l_docs SYS.ODCIVARCHAR2LIST; + + -- CREATE OR REPLACE TYPE t_audio_id_list AS TABLE OF VARCHAR2(600); + v_all_ids SYS.ODCIVARCHAR2LIST; + v_json_embedding varchar2(2000); + + -- + -- CUIDADI: geracao de vetores ira onerar tempo de processamento das imagens + -- + v_call_vector_embedding varchar2(1) := 'N'; + v_call_rag varchar2(1) := 'Y'; + + BEGIN + + IF p_stop_process_list != 'N' THEN + RETURN; + END IF; + + if p_embedding_name = 'COHERE' then + v_json_embedding := '{"provider": "OCIGenAI","credential_name": "OCI_CRED","url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText","batch_size": 50,"model": "cohere.embed-v4.0"}'; + + elsif p_embedding_name = 'VLLM' then + v_json_embedding := '{"provider": "openai","url": "https://hub-gpus.DOMINIO.com.br/embed/v1/embeddings","host":"local","batch_size": 100,"model": "Qwen/Qwen3-Embedding-4B"}'; + + elsif p_embedding_name = 'OPENAI' then + v_json_embedding := '{"provider" : "openai","credential_name" : "CRED_OPENAI", "url":"https://api.openai.com/v1/chat/completions", "model" : "gpt-4.1-mini" }'; + + elsif p_embedding_name = 'ONNX-E5' then + v_json_embedding := '{"provider":"database", "model":"MULTILINGUAL_E5_BASE"}'; + + elsif p_embedding_name = 'ONNX-VIT' then + v_json_embedding := '{"provider":"database", "model":"VIT_BASE_PATCH16_224"}'; + + end if; + + -- + -- rotina que reserva arquivos de modo exclusivo, permitindo uso de scheduler paralelos no banco + -- objetivo: embedding em sessoes paralelas do banco para diminui tempo de carga + -- (1) deve ser definido um valor adequado de arquivos por job, definido no limite de linhas (limit) + -- (2) cada job tem seu worker id definido pelo proprio scheduler + -- + prc_reserve_files( + p_limit => p_limit, + p_worker_id=>p_worker_id, + p_oci_cred=> v_oci_cred, + p_bucket=> v_bucket, + p_docs=>l_docs, + p_mimetype => p_mimetype, + p_embedding_name => p_embedding_name ) ; + + v_dt_start := CURRENT_TIMESTAMP; + -- create sequence seq_26ai_TRAFFIC MINVALUE 1 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL ; + v_session_id := seq_26ai_TRAFFIC.nextval; + + SELECT object_name + BULK COLLECT INTO v_all_ids + FROM ( + SELECT UTL_URL.ESCAPE( object_name ,TRUE,'AL32UTF8') object_name + FROM tb_26ai_TRAFFIC src + WHERE worker_id = p_worker_id + and status = 'R' + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype + ); + + -- Marca batch de processamento como Iniciado + UPDATE tb_26ai_TRAFFIC + SET status = 'S', -- started + dt_start_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'R' + and EMBEDDING_NAME = p_embedding_name + and EMBEDDING_MIMETYPE = p_mimetype; + + COMMIT; + + -- embedding + BEGIN + + if v_call_vector_embedding = 'Y' then + + FORALL i IN 1 .. v_all_ids.COUNT + INSERT INTO tb_26ai_TRAFFIC_vector ( + ID, FILE_NAME, CREATED_DATE, CREATED_BY, + EMBED_ID, EMBED_DATA, EMBED_VECTOR, EMBED_MODE, MIMETYPE + ) + SELECT + v_session_id, + lower(replace( v_all_ids(i) ,' ','_')), + CURRENT_TIMESTAMP, + 'admin', + rownum embed_id, + null text_chunk, + t.vec, + p_embedding_name, + p_mimetype + FROM ( + select DBMS_VECTOR.UTL_TO_EMBEDDING( + DBMS_CLOUD.GET_OBJECT( + credential_name => v_oci_cred, + object_uri => v_bucket || v_all_ids(i) ), + 'image', + JSON('{ + "provider": "OCIGenAI", + "credential_name": "OCI_CRED", + "url": "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText", + "model": "cohere.embed-v4.0"}') + ) vec) t; + + end if; -- v_call_vector_embedding + + -- Marca batch omo processado e gera analise da imagem com RAG + if v_call_rag = 'Y' then + + UPDATE tb_26ai_TRAFFIC + SET status = 'P', -- processado + dt_end_process = SYSDATE, + JSON_DATA = fnc_26ai_traffic_rag (p_base64_image => IMAGE_BASE64, p_oci_cred => v_oci_cred ) + WHERE worker_id = p_worker_id + and status = 'S'; + + else -- Marca batch omo processado + + UPDATE tb_26ai_TRAFFIC + SET status = 'P', -- processado + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + + end if; + + + COMMIT; + EXCEPTION + WHEN OTHERS THEN + + v_error := SQLERRM; + + INSERT INTO tb_26ai_TRAFFIC_debug + VALUES ('JOB_TRAFFIC_WORKER_' || p_worker_id , v_error, null, SYSDATE); + + -- Marca batch de PDF como erro + UPDATE tb_26ai_TRAFFIC + SET status = 'E', -- erro + dt_end_process = SYSDATE + WHERE worker_id = p_worker_id + and status = 'S'; + COMMIT; + END; + + + + COMMIT; + + END; + + + PROCEDURE proc_remove_jobs + AS + BEGIN + -- nao pode fazer pelo numero de jobs existentes pois senao a eliminacao seria falha: + -- cada job tem um padrao de ome associado ao worker id, e nao a contabilizacao que pode ter gaps + -- ajustar de acordo com maximo permitido pelo item de definicao de novos jobs + FOR i IN 1..300 LOOP + begin + DBMS_SCHEDULER.STOP_JOB( job_name => 'JOB_TRAFFIC_WORKER_' || i ); + exception + when others then null; + end; + + begin + DBMS_SCHEDULER.DROP_JOB( job_name => 'JOB_TRAFFIC_WORKER_' || i ); + exception + when others then null; + end; + + + update tb_26ai_TRAFFIC + set status = null, worker_id = null + where status = 'R'; -- estava reservado, mas com remocao do job volta pra status null sem worker id definido + + commit; + + END LOOP; + END; + + + PROCEDURE proc_add_jobs(p_limit in number, + p_total_jobs in number, + p_mimetype in varchar2, + p_embedding_name in varchar2 default 'COHERE') + AS + v_job_count number; + v_has number; + BEGIN + select count(1) + into v_job_count + from user_scheduler_jobs + where JOB_NAME like 'JOB_TRAFFIC_WORKER_%'; + + -- if p_total_jobs <= v_job_count or p_total_jobs is null then + + --raise_application_error(-20002,'The number of scheduler jobs must be greater than what already exists'); + + -- else + + + FOR i IN 1..p_total_jobs LOOP + + select count(1) + into v_has + from user_scheduler_jobs + where JOB_NAME = 'JOB_TRAFFIC_WORKER_' || i; + + if v_has = 0 then -- nao existe com worker id "i" + + DBMS_SCHEDULER.CREATE_JOB( + job_name => 'JOB_TRAFFIC_WORKER_' || i, + job_type => 'PLSQL_BLOCK', + job_action => 'BEGIN pkg_26ai_TRAFFIC_LOAD.proc_process_files(p_limit=>' || p_limit || ',p_worker_id=>' || i || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;', + start_date => SYSTIMESTAMP, + repeat_interval => 'FREQ=SECONDLY; INTERVAL=2;', + enabled => TRUE + ); + + else -- cria-se um novo job alem do ultimo + + DBMS_SCHEDULER.CREATE_JOB( + job_name => 'JOB_TRAFFIC_WORKER_' || to_char(i+v_job_count), + job_type => 'PLSQL_BLOCK', + job_action => 'BEGIN pkg_26ai_TRAFFIC_LOAD.proc_process_files(p_limit=>' || p_limit || ',p_worker_id=>' || to_char(i+v_job_count) || ',p_mimetype=>''' || p_mimetype || ''',p_embedding_name=>''' || p_embedding_name || '''); END;', + start_date => SYSTIMESTAMP, + repeat_interval => 'FREQ=SECONDLY; INTERVAL=2;', + enabled => TRUE + ); + + end if; + + END LOOP; + + + -- end if; + + END; + +end; +/ \ No newline at end of file