leadcast/backend/examples/diag_ddg.rs

use backend::browser::{BrowserFetchOptions, BrowserModule, BrowserModuleConfig};
use backend::config::AppConfig;
use backend::http_client::{HttpClientConfig, HttpClientFactory};
use backend::proxy::ProxyConfig;
use backend::search::{SearchConfig, SearchService};

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let _ = dotenvy::dotenv();
    backend::logs::init();
    let request_id = "diag-ddg";

    let config = AppConfig::from_env()?;
    let proxy = ProxyConfig::from_env(request_id)?;
    let browser_config = BrowserModuleConfig {
        no_sandbox: config.browser_no_sandbox,
        navigation_timeout_secs: config.browser_timeout.as_secs(),
        max_concurrency: config.browser_concurrency,
        ..Default::default()
    };
    let browser = BrowserModule::new(proxy.clone(), browser_config)?;

    let http = HttpClientFactory::new(
        proxy,
        HttpClientConfig {
            request_timeout_secs: config.request_timeout.as_secs(),
            max_concurrency: 8,
            ..HttpClientConfig::default()
        },
    )?;

    // Teste A: modo simulate_interaction (homepage + digitar).
    println!("=== Teste A: SearchService.simulate_interaction (homepage) ===");
    let svc = SearchService::new(browser.clone(), http.clone(), SearchConfig::default())?;
    match svc
        .search(request_id, "willian caprino cibersegurança blaze", 10)
        .await
    {
        Ok(rows) => {
            println!("A: {} resultados", rows.len());
            for r in rows.iter().take(5) {
                println!("  [{}] {} -> {}", r.rank, r.title, r.url);
            }
        }
        Err(e) => println!("A ERRO: {e}"),
    }

    // Teste B: fetch direto da homepage + digitar e salvar HTML.
    println!("\n=== Teste B: fetch direto + salvar HTML ===");
    let options = BrowserFetchOptions {
        warmup_url: Some("https://duckduckgo.com/".into()),
        wait_after_load_ms: 3_500,
        block_heavy_resources: true,
        max_html_bytes: 8 * 1024 * 1024,
        scroll_rounds: 2,
        scroll_delay_ms: 800,
        interaction_query: Some("willian caprino cibersegurança blaze".into()),
        interaction_selector: Some("input[name='q']".into()),
        skip_challenge_check: true,
    };
    let page = browser
        .fetch_html_with_options(request_id, "https://duckduckgo.com/", options)
        .await?;
    println!("B final_url: {}", page.final_url);
    println!("B html_len: {}", page.html.len());
    std::fs::write("/tmp/ddg_home.html", &page.html)?;
    println!("B HTML salvo em /tmp/ddg_home.html");

    // Teste C: fetch direto de /html/?q=...
    println!("\n=== Teste C: fetch direto /html/?q= ===");
    let options2 = BrowserFetchOptions {
        warmup_url: Some("https://duckduckgo.com/".into()),
        wait_after_load_ms: 2_000,
        block_heavy_resources: true,
        max_html_bytes: 8 * 1024 * 1024,
        scroll_rounds: 2,
        scroll_delay_ms: 600,
        interaction_query: None,
        interaction_selector: None,
        skip_challenge_check: true,
    };
    let url = "https://duckduckgo.com/html/?q=willian+caprino+ciberseguran%C3%A7a+blaze&kl=br&kp=1";
    let page2 = browser
        .fetch_html_with_options(request_id, url, options2)
        .await?;
    println!("C final_url: {}", page2.final_url);
    println!("C html_len: {}", page2.html.len());
    std::fs::write("/tmp/ddg_html.html", &page2.html)?;
    println!("C HTML salvo em /tmp/ddg_html.html");

    // Teste D: backend HTTP (reqwest + proxy) direto em /html/?q=.
    println!("\n=== Teste D: backend HTTP /html/?q= ===");
    let session = http.fresh(request_id)?;
    let http_url =
        "https://duckduckgo.com/html/?q=willian+caprino+ciberseguran%C3%A7a+blaze&kl=br&kp=1";
    match session
        .warm_then_get_text(request_id, "https://duckduckgo.com/")
        .await
    {
        Ok(_) => println!("D warm ok"),
        Err(e) => println!("D warm err: {e}"),
    }
    match session.warm_then_get_text(request_id, http_url).await {
        Ok(resp) => {
            println!("D OK len={} status", resp.text.len());
            std::fs::write("/tmp/ddg_http.html", &resp.text).ok();
            println!("D salvo /tmp/ddg_http.html");
            let lowered = resp.text.to_ascii_lowercase();
            for n in [
                "anomaly-modal",
                "bots use duckduckgo",
                "result__a",
                "/l/?uddg=",
            ] {
                let c = lowered.matches(n).count();
                println!("  {n}: {c}");
            }
        }
        Err(e) => println!("D ERRO: {e}"),
    }

    println!("\n=== trace de seletores no /html/ ===");
    let doc_html = std::fs::read_to_string("/tmp/ddg_html.html")?;
    for needle in [
        "result__a",
        "result__snippet",
        "data-testid=\"result",
        "class=\"result",
        "/l/?uddg=",
        "<h2",
    ] {
        let n = doc_html
            .to_ascii_lowercase()
            .matches(needle.to_ascii_lowercase().as_str())
            .count();
        println!("  {needle}: {n}");
    }

    Ok(())
}