# enhanced_search_v2.py (Versão Final com Decomposição de Palavras)
###################################################################################################
#
# RESUMO DAS CORREÇÕES E MELHORIAS:
#
# 1. ADIÇÃO DA CAMADA DE DECOMPOSIÇÃO DE PALAVRAS (MUDANÇA PRINCIPAL):
#    - Uma nova camada de pré-processamento foi adicionada para identificar palavras longas
#      que não existem no dicionário (ex: "CirurgicoHisterectomia") e tentar dividi-las
#      em duas palavras válidas (ex: "Cirurgico Histerectomia").
#    - Esta lógica NÃO afeta palavras longas que SÃO válidas e existem no dicionário
#      (ex: "Antipneumococo"), garantindo a precisão da busca.
#
# 2. TODAS AS OUTRAS FUNCIONALIDADES MANTIDAS:
#    - Mapeamento, Blocos, RRF, Boosts, Otimização Fuzzy e todas as correções de bugs
#      anteriores foram 100% preservadas.
#
###################################################################################################


import pandas as pd
import re
from unidecode import unidecode
import time
from sentence_transformers import util
import torch
import math
from collections import defaultdict
from rank_bm25 import BM25Okapi

# --- Tenta usar a biblioteca RapidFuzz que é muito mais rápida ---
try:
    # Usa a versão C++ otimizada se estiver instalada
    from rapidfuzz import process, fuzz
    print("--- [SUCESSO] Biblioteca 'rapidfuzz' encontrada. Usando motor otimizado. ---")
except ImportError:
    # Usa a versão Python pura como fallback
    from thefuzz import process, fuzz
    print("--- [AVISO] 'rapidfuzz' não encontrada. Usando 'thefuzz' como fallback (mais lento). ---")

# --- Dicionário de Mapeamento de Variações ---
COMMON_QUERY_MAPPINGS = {
    "ultra sonografia": "ultrassonografia", "ultra som": "ultrassom", "eco cardiograma": "ecocardiograma",
    "eletro cardiograma": "eletrocardiograma", "eletro encefalograma": "eletroencefalograma",
    "video endoscopia": "videoendoscopia", "video laparoscopia": "videolaparoscopia",
    "eco doppler": "ecodopplercardiograma", "rm": "ressonancia", "rnm": "ressonancia", "tc": "tomografia",
    "usg": "ultrassonografia", "us": "ultrassonografia", "ecg": "eletrocardiograma", "eeg": "eletroencefalograma",
    "raio x": "radiografia", "raiox": "radiografia", "chapa": "radiografia", "ultrasson": "ultrassom",
    "tomografia computadorizada": "tomografia", "ressonancia magnetica": "ressonancia",
    "endoscopia digestiva alta": "endoscopia", "densitometria ossea": "densitometria"
}

# --- Bloco 1: Funções Auxiliares de Normalização e Limpeza --- #

def literal_normalize_text(text):
    if pd.isna(text): return ""
    normalized = unidecode(str(text).lower())
    return re.sub(r'^\W+|\W+$', '', normalized).strip()

def clean_symbols_from_query(text):
    if pd.isna(text): return ""
    return re.sub(r"[´`'\"/*]", "", str(text)).strip()

def normalize_text(text):
    if pd.isna(text): return ""
    normalized = unidecode(str(text).lower())
    normalized = re.sub(r'[^\w\s]', ' ', normalized)
    return re.sub(r'\s+', ' ', normalized).strip()

def get_longest_word(query_text):
    words = re.findall(r'\b\w{4,}\b', query_text)
    if not words: return ""
    return max(words, key=len)


# --- Bloco 2: Funções de Formatação e Destaque de Resultados --- #

def format_result(row_data, row_index, match_type="", score=0, **kwargs):
    data = row_data.copy()
    is_rol = data.get('Correlacao_Rol', '').strip().lower() == 'sim'
    if not is_rol:
        data['Grupo'], data['Subgrupo'], data['Vigencia'], data['Resolucao_Normativa'] = '', '', '', ''
        data['PAC'], data['DUT'] = '---', '---'
    else:
        data['PAC'] = 'Sim' if data.get('PAC', '').strip().lower() == 'pac' else 'Não'
        original_dut_value = data.get('DUT', '').strip()
        if original_dut_value and original_dut_value.replace('.', '', 1).isdigit():
            data['DUT'] = f'Sim, DUT nº {original_dut_value}'
        else: data['DUT'] = 'Não'
    standard_columns = [ 'Codigo_TUSS', 'Descricao_TUSS', 'Correlacao_Rol', 'Procedimento_Rol', 'Resolucao_Normativa', 'Vigencia', 'OD', 'AMB', 'HCO', 'HSO', 'PAC', 'DUT', 'SUBGRUPO', 'GRUPO', 'CAPITULO', 'Sinonimo_1', 'Sinonimo_2', 'Sinonimo_3', 'Sinonimo_4', 'Semantico' ]
    formatted_data = {col: data.get(col, '') for col in standard_columns}
    result = { "row_index": row_index, "score": round(score), "text_score": round(score), "semantic_score": 0, "match_type": match_type, "is_rol_procedure": is_rol }
    result.update(formatted_data)
    result.update(kwargs)
    return result

def _highlight_matches(results, query):
    if not query or not results: return results
    stopwords = {'de', 'do', 'da', 'dos', 'das', 'a', 'o', 'e', 'em', 'um', 'uma', 'para', 'com', 'por', 'para'}
    query_words = {word for word in normalize_text(query).lower().split() if len(word) > 2 and word not in stopwords}
    cols_to_highlight = ['Descricao_TUSS', 'Procedimento_Rol', 'Sinonimo_1', 'Sinonimo_2', 'Sinonimo_3', 'Sinonimo_4', 'Semantico']
    for result in results:
        for col in cols_to_highlight:
            original_text = result.get(col, '')
            if original_text and query_words:
                highlighted_text = original_text
                for word in sorted(list(query_words), key=len, reverse=True):
                    pattern = r'\b(' + re.escape(word) + r')\b'
                    highlighted_text = re.sub(pattern, r'<b>\1</b>', highlighted_text, flags=re.IGNORECASE)
                result[f"{col}_highlighted"] = highlighted_text
            else:
                result[f"{col}_highlighted"] = original_text
    return results


# --- Bloco 3: Funções de Carregamento de Dados e Modelos --- #

def load_and_prepare_database(db_path):
    try:
        print(f"Carregando e preparando a base de dados de: {db_path}...")
        df_original = pd.read_csv(db_path, dtype=str).fillna('')
        search_cols = ['Descricao_TUSS', 'Procedimento_Rol', 'Sinonimo_1', 'Sinonimo_2', 'Sinonimo_3', 'Sinonimo_4', 'Semantico', 'SUBGRUPO', 'GRUPO', 'CAPITULO']
        df_normalized = df_original.copy()
        df_normalized['Codigo_TUSS_literal'] = df_normalized['Codigo_TUSS'].apply(literal_normalize_text)
        df_normalized['Procedimento_Rol_literal'] = df_normalized['Procedimento_Rol'].apply(literal_normalize_text)
        df_normalized['Descricao_TUSS_literal'] = df_normalized['Descricao_TUSS'].apply(literal_normalize_text)
        df_normalized['full_text_norm'] = ""
        for col in search_cols:
            if col in df_normalized.columns:
                df_normalized[f'{col}_norm'] = df_normalized[col].apply(normalize_text)
                df_normalized['full_text_norm'] += ' ' + df_normalized[f'{col}_norm']
        print("Criando modelo BM25 e calculando frequência de palavras...")
        tokenized_corpus = [doc.split() for doc in df_normalized['full_text_norm']]
        bm25_model = BM25Okapi(tokenized_corpus, k1=1.2)
        print(f"Modelo BM25 otimizado com k1={bm25_model.k1}.")
        doc_freq = defaultdict(int)
        for doc_words in tokenized_corpus:
            for word in set(doc_words): doc_freq[word] += 1
        print("Combinando frequências de palavras (masculino/feminino)...")
        combined_doc_freq = {}
        processed_words = set()
        for word, freq in doc_freq.items():
            if word in processed_words: continue
            pair_word = None
            if word.endswith('o'): pair_word = word[:-1] + 'a'
            elif word.endswith('a'): pair_word = word[:-1] + 'o'
            if pair_word and pair_word in doc_freq:
                combined_freq = freq + doc_freq[pair_word]
                combined_doc_freq[word] = combined_freq
                combined_doc_freq[pair_word] = combined_freq
                processed_words.add(word); processed_words.add(pair_word)
            else:
                combined_doc_freq[word] = freq
        print("Criando corpus para busca fuzzy...")
        fuzzy_search_corpus = []
        for index, row in df_normalized.iterrows():
            for col in search_cols:
                if f'{col}_norm' in row and pd.notna(row[f'{col}_norm']):
                    if val := row[f'{col}_norm']: fuzzy_search_corpus.append((val, index))
        print(f"Base de dados pronta com {len(df_original)} procedimentos.")
        return df_original, df_normalized, fuzzy_search_corpus, bm25_model, combined_doc_freq
    except Exception as e:
        print(f"Erro crítico ao carregar/preparar a base de dados: {e}"); raise

def load_correction_corpus(dict_path, column_name='Termo_Correto'):
    try:
        df_dict = pd.read_csv(dict_path, dtype=str).fillna('')
        if column_name not in df_dict.columns: return [], [], set()
        original_corpus = df_dict[column_name].dropna().astype(str).tolist()
        normalized_corpus = [normalize_text(term) for term in original_corpus]
        db_word_set = {word for term in normalized_corpus for word in term.split()}
        return original_corpus, normalized_corpus, db_word_set
    except (FileNotFoundError, Exception): return [], [], set()

def load_general_dictionary(path):
    try:
        with open(path, 'r', encoding='utf-8') as f: words = {normalize_text(line.strip()) for line in f if line.strip()}
        print(f"Dicionário geral carregado com {len(words)} palavras.")
        return words
    except (FileNotFoundError, Exception): return set()

# --- Bloco 4: Funções de Reclassificação, Boosts e IA --- #
def create_unified_document_text(result_dict):
    text_parts = { result_dict.get(key, '') for key in ['Descricao_TUSS', 'Procedimento_Rol', 'Semantico', 'SUBGRUPO', 'GRUPO', 'CAPITULO', 'Sinonimo_1', 'Sinonimo_2', 'Sinonimo_3', 'Sinonimo_4']}
    return ". ".join(sorted([part for part in text_parts if part and str(part).strip()]))

def rerank_with_cross_encoder(query, results_list, model):
    if not model or not results_list or not query:
        return results_list, "Cross-Encoder não fornecido ou lista de candidatos vazia."
    sentence_pairs = [[query, create_unified_document_text(result)] for result in results_list]
    if not sentence_pairs: return results_list, "Não foram encontrados pares para reordenar."
    try:
        raw_scores = model.predict(sentence_pairs, show_progress_bar=False)
        semantic_scores = torch.sigmoid(torch.tensor(raw_scores)).numpy() * 100
        for i, result in enumerate(results_list): result['semantic_score'] = round(semantic_scores[i])
        log_message = f"Scores semânticos calculados para {len(results_list)} candidatos."
        return results_list, log_message
    except Exception as e:
        log_message = f"Erro no Cross-Encoder: {e}"; print(log_message)
        return results_list, log_message

def apply_boosts(results, query_words, doc_freq):
    if not results or not query_words: return results, []
    log_messages = []
    jargon_terms = {w for w in query_words if doc_freq.get(w, 0) <= 15 and len(w) > 4}
    if jargon_terms:
        boosted_count = 0
        for r in results:
            text = normalize_text(create_unified_document_text(r))
            if any(re.search(r'\b' + re.escape(term) + r'\b', text) for term in jargon_terms):
                r['text_score'] = min(r['text_score'] * 1.2, 97)
                boosted_count += 1
        if boosted_count > 0: log_messages.append(f"Boost de Jargão para '{list(jargon_terms)}' ({boosted_count} afetados).")
    rare_terms = {w for w in query_words if doc_freq.get(w, 0) <= 5 and len(w) > 4}
    if rare_terms:
        boosted_count = 0
        for r in results:
            text = normalize_text(create_unified_document_text(r))
            if any(re.search(r'\b' + re.escape(term) + r'\b', text) for term in rare_terms):
                r['text_score'] = min(r['text_score'] * 1.3, 97)
                boosted_count += 1
        if boosted_count > 0: log_messages.append(f"Boost Ultra-Raro para '{list(rare_terms)}' ({boosted_count} afetados).")
    for r in results: r['score'] = r['text_score']
    return results, log_messages

# --- Bloco 5: Função Principal de Orquestração da Busca --- #

def search_procedure_with_log(query, df_original, df_normalized, fuzzy_search_corpus, correction_corpus,
                              valid_words_set, bm25_model, doc_freq,
                              cross_encoder_model=None, user_best_matches_counts=None, user_feedback_threshold=10):
    start_time = time.time(); original_query = str(query).strip()
    response = { "search_log": [], "results_by_layer": {}, "final_semantic_results": [], "original_query": original_query, "corrected_query": "", "was_corrected": False }
    if not original_query: response["search_log"].append("Query vazia."); return response
    response["search_log"].append(f"Buscando por: '{original_query}'")

    # --- Etapa 1: Pré-processamento inicial ---
    query_para_literal = literal_normalize_text(original_query)
    response["search_log"].append(f"\n--- Etapa 1: Pré-processamento da Query ---")
    response["search_log"].append(f"Query para busca literal (após limpeza de bordas): '{query_para_literal}'")

    # --- Etapa 2: Camada de Early Exit ---
    response["search_log"].append("\n--- Etapa 2: Camada de Early Exit (Literal) ---")
    literal_results = []
    seen_indices = set()
    literal_fields = ['Codigo_TUSS_literal', 'Procedimento_Rol_literal', 'Descricao_TUSS_literal']
    for field in literal_fields:
        mask = df_normalized[field] == query_para_literal
        for index in df_normalized.index[mask & ~df_normalized.index.isin(seen_indices)]:
            literal_results.append(format_result(df_original.loc[index], index, "Literal Exata", 100))
            seen_indices.add(index)
    if literal_results:
        response["search_log"].append(f"Encontrado(s) {len(literal_results)} resultado(s) literal(is). ATIVANDO EARLY EXIT.")
        results_with_semantic, log_msg = rerank_with_cross_encoder(normalize_text(original_query), literal_results, cross_encoder_model)
        response["search_log"].append(f"Cálculo de Score Semântico: {log_msg}")
        final_list = sorted(results_with_semantic, key=lambda x: x['text_score'], reverse=True)
        response["final_semantic_results"] = _highlight_matches(final_list, original_query)
        end_time = time.time(); response["search_duration_seconds"] = round(end_time - start_time, 4)
        response["search_log"].append(f"\nBusca concluída em {response['search_duration_seconds']} segundos.")
        print("\n".join(response["search_log"]))
        return response
    else:
        response["search_log"].append("Nenhum resultado literal encontrado. Prosseguindo para a busca completa.")

    # --- Camada de Mapeamento de Variações ---
    temp_query = original_query.lower()
    query_foi_mapeada = False
    for wrong, right in sorted(COMMON_QUERY_MAPPINGS.items(), key=lambda item: len(item[0]), reverse=True):
        if wrong in temp_query:
            temp_query = temp_query.replace(wrong, right)
            query_foi_mapeada = True
    if query_foi_mapeada:
        response["search_log"].append(f"Variação de termo comum mapeada: '{original_query}' -> '{temp_query}'")
        query_para_processamento = temp_query
    else:
        query_para_processamento = original_query

    # --- Etapa 3: Busca Completa (se não houve Early Exit) ---
    query_para_geral = clean_symbols_from_query(query_para_processamento)
    
    # --- INÍCIO DA NOVA FUNCIONALIDADE: Decomposição de Palavras ---
    decomposed_words = []
    words_decomposed = False
    # Itera sobre as palavras da query após a limpeza de símbolos e mapeamento
    for word in query_para_geral.split():
        # 1. IDENTIFICAÇÃO: A palavra é longa E não está no nosso dicionário de palavras válidas?
        if len(word) > 13 and normalize_text(word) not in valid_words_set:
            was_split = False
            # 2. DECOMPOSIÇÃO: Tenta encontrar um ponto de quebra válido
            # Começamos com prefixos de 4 letras e deixamos no mínimo 4 para o sufixo
            for i in range(4, len(word) - 3):
                prefix = normalize_text(word[:i])
                suffix = normalize_text(word[i:])
                # 3. VALIDAÇÃO: Ambas as partes quebradas são palavras válidas?
                if prefix in valid_words_set and suffix in valid_words_set:
                    # 4. SUBSTITUIÇÃO: Se sim, usamos a versão quebrada e paramos de procurar
                    decomposed_words.extend([word[:i], word[i:]])
                    response["search_log"].append(f"Palavra composta '{word}' decomposta em: '{word[:i]} {word[i:]}'")
                    was_split = True
                    words_decomposed = True
                    break
            # Se, após todos os testes, não encontramos uma quebra válida, mantemos a palavra original
            if not was_split:
                decomposed_words.append(word)
        else:
            # Se a palavra não é suspeita, apenas a mantemos
            decomposed_words.append(word)

    if words_decomposed:
        query_para_geral = " ".join(decomposed_words)
    # --- FIM DA NOVA FUNCIONALIDADE ---
    
    response["search_log"].append(f"Query para busca geral (após limpeza e decomposição): '{query_para_geral}'")
    
    original_correction_corpus, normalized_correction_corpus = correction_corpus
    stopwords = {'de', 'do', 'da', 'dos', 'das', 'a', 'o', 'e', 'em', 'um', 'uma', 'para', 'com'}
    final_query_words_list = []
    words_were_corrected = False
    
    for word in query_para_geral.split():
        norm_word = normalize_text(word)
        is_candidate_for_correction = ( len(norm_word) > 2 and norm_word not in stopwords and not norm_word.isdigit() and norm_word not in valid_words_set )
        if is_candidate_for_correction:
            best_match, score, _ = process.extractOne(norm_word, normalized_correction_corpus, scorer=fuzz.ratio)
            if score > 85:
                original_term_index = normalized_correction_corpus.index(best_match)
                corrected_word_original = original_correction_corpus[original_term_index]
                corrected_word = corrected_word_original
                if word.istitle():
                    corrected_word = corrected_word.title()
                elif word.isupper():
                    corrected_word = corrected_word.upper()
                final_query_words_list.append(corrected_word)
                words_were_corrected = True
            else:
                final_query_words_list.append(word)
        else:
            final_query_words_list.append(word)

    query_corrigida = " ".join(final_query_words_list)
    if words_were_corrected and query_corrigida.strip().lower() != query_para_geral.strip().lower():
        response["was_corrected"] = True
        response["corrected_query"] = query_corrigida
        response["search_log"].append(f"Correção ortográfica aplicada: '{query_para_geral}' -> '{query_corrigida}'")

    cleaned_query = " ".join([word for word in query_corrigida.split() if normalize_text(word) not in stopwords])
    normalized_cleaned_query = normalize_text(cleaned_query)
    query_words = [word for word in normalized_cleaned_query.split() if len(word) > 1]
    
    response["search_log"].append(f"Query para busca (após limpeza de stopwords): '{cleaned_query}'")

    # --- Camadas do Bloco Protegido ---
    protected_results = []
    # Camada 1: Busca Exata
    exact_fields = [f"{col}_norm" for col in ['Codigo_TUSS', 'Descricao_TUSS', 'Procedimento_Rol', 'Sinonimo_1', 'Sinonimo_2', 'Sinonimo_3', 'Sinonimo_4', 'Semantico']]
    for field in exact_fields:
        if field in df_normalized.columns:
            for index in df_normalized.index[(df_normalized[field] == normalized_cleaned_query) & (~df_normalized.index.isin(seen_indices))]:
                protected_results.append(format_result(df_original.loc[index], index, "Exato", 100))
                seen_indices.add(index)
    # Camada 2: Frase Exata
    if len(query_words) > 1:
        phrase_pattern = r'\b' + re.escape(normalized_cleaned_query) + r'\b'
        for index in df_normalized.index[df_normalized['full_text_norm'].str.contains(phrase_pattern, na=False, regex=True) & ~df_normalized.index.isin(seen_indices)]:
            protected_results.append(format_result(df_original.loc[index], index, "Frase Exata", 99))
            seen_indices.add(index)
    # Camada 3: Lógica (E)
    search_fields_norm = [f"{col}_norm" for col in ['Descricao_TUSS', 'Procedimento_Rol', 'Sinonimo_1', 'Sinonimo_2', 'Sinonimo_3', 'Sinonimo_4', 'Semantico']]
    for field in search_fields_norm:
        if field in df_normalized.columns and query_words:
            mask = pd.Series(True, index=df_normalized.index)
            for word in query_words: mask &= df_normalized[field].str.contains(r'\b' + re.escape(word) + r'\b', na=False)
            for index in df_normalized.index[mask & ~df_normalized.index.isin(seen_indices)]:
                protected_results.append(format_result(df_original.loc[index], index, "Lógica (E)", 98))
                seen_indices.add(index)
    response["search_log"].append(f"\n--- Detalhamento por Camada ---\nBloco Protegido (Exato, Frase, Lógica E): {len(protected_results)} candidatos.")

    # --- LÓGICA DE RANKING LEXICAL HÍBRIDO OTIMIZADA ---
    broad_candidates_results = []
    if query_words:
        # Passo 1: Geração de Candidatos (União de Fuzzy e BM25)
        broad_candidate_indices = set()
        
        temp_fuzzy_corpus = [
            item for item in fuzzy_search_corpus 
            if any(word in item[0] for word in query_words)
        ]
        response["search_log"].append(f"Fuzzy: Otimização reduziu o corpus de busca para {len(temp_fuzzy_corpus)} candidatos.")
        
        fuzzy_matches = process.extract(
            normalized_cleaned_query,
            [item[0] for item in temp_fuzzy_corpus],
            scorer=fuzz.token_set_ratio,
            limit=40,
            score_cutoff=90
        )
        for match_text, score, _ in fuzzy_matches:
            for _, index in [item for item in temp_fuzzy_corpus if item[0] == match_text]:
                if index not in seen_indices:
                    broad_candidate_indices.add(index)
        
        # Candidatos via BM25
        bm25_scores_all = bm25_model.get_scores(query_words)
        top_n_indices_bm25 = sorted(range(len(bm25_scores_all)), key=lambda i: bm25_scores_all[i], reverse=True)[:25]
        for i in top_n_indices_bm25:
            if bm25_scores_all[i] > 0:
                index = df_normalized.index[i]
                if index not in seen_indices:
                    broad_candidate_indices.add(index)
        
        response["search_log"].append(f"Bloco Amplo (Fuzzy Otimizado + BM25): {len(broad_candidate_indices)} candidatos únicos encontrados.")

        # Passo 2: Reclassificação Unificada com BM25 como fonte da verdade
        if broad_candidate_indices:
            candidate_bm25_scores = {df_normalized.index[i]: bm25_scores_all[i] for i in range(len(bm25_scores_all)) if df_normalized.index[i] in broad_candidate_indices}
            max_score = max(candidate_bm25_scores.values()) if candidate_bm25_scores else 1.0

            for index in broad_candidate_indices:
                score = (candidate_bm25_scores.get(index, 0) / max_score) * 95
                if score > 1:
                    broad_candidates_results.append(format_result(df_original.loc[index], index, "Relevância Híbrida", score))
    
    response["search_log"].append(f"Bloco Amplo: {len(broad_candidates_results)} candidatos com score de relevância > 1.")
    
    unique_candidates = list({r['row_index']: r for r in protected_results + broad_candidates_results}.values())

    query_norm_fb = normalize_text(response.get("corrected_query") or original_query)
    for r in unique_candidates:
        votes = user_best_matches_counts.get(query_norm_fb, {}).get(r['Codigo_TUSS'], 0)
        if votes >= user_feedback_threshold:
            r.update({'is_user_best_match': True, 'feedback_votes': votes})
            
    feedback_priority_list = sorted([r for r in unique_candidates if r.get('is_user_best_match')], key=lambda x: x['feedback_votes'], reverse=True)
    other_candidates = [r for r in unique_candidates if not r.get('is_user_best_match')]
    
    final_list = feedback_priority_list
    response["search_log"].append(f"\n--- Etapa 3.1: Verificação de Feedback de Usuário ---\n{len(final_list)} resultado(s) priorizado(s) por feedback de usuário.")
    
    protected_candidates_final = [r for r in protected_results if r in other_candidates]
    broad_candidates_final = [r for r in broad_candidates_results if r in other_candidates]
    
    response["search_log"].append("\n--- Etapa 4: Processamento dos Blocos e Re-ranking ---")
    
    if protected_candidates_final:
        unique_protected = list({r['row_index']: r for r in protected_candidates_final}.values())
        protected_with_scores, log_msg = rerank_with_cross_encoder(normalized_cleaned_query, unique_protected, cross_encoder_model)
        response["search_log"].append(f"Bloco Protegido: {log_msg}")
        sorted_protected = sorted(protected_with_scores, key=lambda x: (x['text_score'], x['semantic_score']), reverse=True)
        final_list.extend(sorted_protected)

    if broad_candidates_final:
        unique_broad_sorted = sorted(broad_candidates_final, key=lambda x: x['text_score'], reverse=True)[:30]
        response["search_log"].append(f"Bloco Amplo: {len(unique_broad_sorted)} melhores candidatos selecionados para otimização.")
        
        boosted_broad, boost_logs = apply_boosts(unique_broad_sorted, query_words, doc_freq)
        for log in boost_logs: response["search_log"].append(log)
        
        reranked_broad, log_msg = rerank_with_cross_encoder(normalized_cleaned_query, boosted_broad, cross_encoder_model)
        response["search_log"].append(f"Bloco Amplo: {log_msg}")

        def apply_rrf_and_sort(results, k=60):
            if not results: return []
            sorted_by_text = sorted(results, key=lambda x: x.get('text_score', 0), reverse=True)
            sorted_by_semantic = sorted(results, key=lambda x: x.get('semantic_score', 0), reverse=True)
            text_ranks = {res['row_index']: i + 1 for i, res in enumerate(sorted_by_text)}
            semantic_ranks = {res['row_index']: i + 1 for i, res in enumerate(sorted_by_semantic)}
            for res in results:
                text_rank = text_ranks.get(res['row_index'], len(results) + 1)
                semantic_rank = semantic_ranks.get(res['row_index'], len(results) + 1)
                rrf_score = (1 / (k + text_rank)) + (1 / (k + semantic_rank))
                res['rrf_score'] = rrf_score
            return sorted(results, key=lambda x: x['rrf_score'], reverse=True)
        
        final_reranked_broad = apply_rrf_and_sort(reranked_broad)
        final_list.extend(final_reranked_broad)

    response["final_semantic_results"] = _highlight_matches(final_list[:20], query_corrigida)
    end_time = time.time(); response["search_duration_seconds"] = round(end_time - start_time, 4)
    response["search_log"].append(f"\nBusca completa em {response['search_duration_seconds']} segundos.")
    
    print("\n\n" + "="*20 + " LOG DE BUSCA FINAL " + "="*20)
    print("\n".join(response["search_log"]))
    
    return response