first commit

backend/examples/diag_youtube.rs

@@ -0,0 +1,97 @@
+use backend::browser::{BrowserFetchOptions, BrowserModule, BrowserModuleConfig};
+use backend::persona;
+use backend::proxy::ProxyConfig;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let _ = dotenvy::dotenv();
+    backend::logs::init();
+    let request_id = "diag-yt";
+
+    let proxy = ProxyConfig::from_env(request_id)?;
+    let browser_config = BrowserModuleConfig {
+        no_sandbox: true,
+        navigation_timeout_secs: 75,
+        ..Default::default()
+    };
+    let browser = BrowserModule::new(proxy, browser_config)?;
+
+    let persona = persona::generate(request_id, Some("br"));
+    println!(
+        "persona: chrome_major={} platform={:?} locale={}",
+        persona.chrome_major, persona.platform, persona.locale
+    );
+
+    let options = BrowserFetchOptions {
+        warmup_url: Some("https://www.youtube.com/".into()),
+        wait_after_load_ms: 6_000,
+        block_heavy_resources: false,
+        max_html_bytes: 32 * 1024 * 1024,
+        scroll_rounds: 0,
+        scroll_delay_ms: 800,
+        interaction_query: None,
+        interaction_selector: None,
+        skip_challenge_check: false,
+    };
+
+    let url = "https://www.youtube.com/results?search_query=podcast+tecnologia&sp=EgIQAg%3D%3D&hl=pt-BR&gl=BR";
+    let page = browser
+        .fetch_html_with_options_and_persona(request_id, url, options, persona)
+        .await?;
+
+    println!("final_url: {}", page.final_url);
+    println!("html_len: {}", page.html.len());
+    std::fs::write("/tmp/yt_diag.html", &page.html)?;
+    println!("html salvo em /tmp/yt_diag.html");
+
+    // Procurar sinais de challenge/cookiewall
+    let lower = page.html.to_ascii_lowercase();
+    let needles = [
+        "unusual traffic",
+        "detected unusual traffic",
+        "não sou um robô",
+        "recaptcha",
+        "hcaptcha",
+        "cf-chl-challenge",
+        "consent.google.com",
+        "consent.youtube",
+        "g-recaptcha",
+        "sorry/index",
+        "ogat",
+        "servicey",
+        "our systems have detected",
+        "consent.youtube.com",
+        "agree",
+        "aceitar",
+        "sign in",
+        "faça login",
+        "before you continue",
+        "antes de continuar",
+    ];
+    for n in needles {
+        if let Some(idx) = lower.find(n) {
+            let start = idx.saturating_sub(80);
+            let ctx: String = page.html.chars().skip(start).take(200).collect();
+            println!("FOUND needle {n:?}: ...{ctx}...");
+        }
+    }
+
+    if page.html.contains("ytInitialData") {
+        println!("ytInitialData encontrado");
+    } else {
+        println!("ytInitialData AUSENTE");
+    }
+    if page.html.contains("ytInitialPlayerResponse") {
+        println!("ytInitialPlayerResponse encontrado (raro em /results)");
+    }
+    println!("\ntitle (primeiros 500 chars):");
+    if let Some(start) = page.html.find("<title") {
+        let end = page.html[start..]
+            .find("</title>")
+            .map(|e| start + e + 8)
+            .unwrap_or(start + 500);
+        println!("{}", &page.html[start..end.min(page.html.len())]);
+    }
+
+    Ok(())
+}