first commit
This commit is contained in:
gustavooth
@@ -0,0 +1,140 @@
|
||||
use backend::browser::{BrowserFetchOptions, BrowserModule, BrowserModuleConfig};
|
||||
use backend::config::AppConfig;
|
||||
use backend::http_client::{HttpClientConfig, HttpClientFactory};
|
||||
use backend::proxy::ProxyConfig;
|
||||
use backend::search::{SearchConfig, SearchService};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let _ = dotenvy::dotenv();
|
||||
backend::logs::init();
|
||||
let request_id = "diag-ddg";
|
||||
|
||||
let config = AppConfig::from_env()?;
|
||||
let proxy = ProxyConfig::from_env(request_id)?;
|
||||
let browser_config = BrowserModuleConfig {
|
||||
no_sandbox: config.browser_no_sandbox,
|
||||
navigation_timeout_secs: config.browser_timeout.as_secs(),
|
||||
max_concurrency: config.browser_concurrency,
|
||||
..Default::default()
|
||||
};
|
||||
let browser = BrowserModule::new(proxy.clone(), browser_config)?;
|
||||
|
||||
let http = HttpClientFactory::new(
|
||||
proxy,
|
||||
HttpClientConfig {
|
||||
request_timeout_secs: config.request_timeout.as_secs(),
|
||||
max_concurrency: 8,
|
||||
..HttpClientConfig::default()
|
||||
},
|
||||
)?;
|
||||
|
||||
// Teste A: modo simulate_interaction (homepage + digitar).
|
||||
println!("=== Teste A: SearchService.simulate_interaction (homepage) ===");
|
||||
let svc = SearchService::new(browser.clone(), http.clone(), SearchConfig::default())?;
|
||||
match svc
|
||||
.search(request_id, "willian caprino cibersegurança blaze", 10)
|
||||
.await
|
||||
{
|
||||
Ok(rows) => {
|
||||
println!("A: {} resultados", rows.len());
|
||||
for r in rows.iter().take(5) {
|
||||
println!(" [{}] {} -> {}", r.rank, r.title, r.url);
|
||||
}
|
||||
}
|
||||
Err(e) => println!("A ERRO: {e}"),
|
||||
}
|
||||
|
||||
// Teste B: fetch direto da homepage + digitar e salvar HTML.
|
||||
println!("\n=== Teste B: fetch direto + salvar HTML ===");
|
||||
let options = BrowserFetchOptions {
|
||||
warmup_url: Some("https://duckduckgo.com/".into()),
|
||||
wait_after_load_ms: 3_500,
|
||||
block_heavy_resources: true,
|
||||
max_html_bytes: 8 * 1024 * 1024,
|
||||
scroll_rounds: 2,
|
||||
scroll_delay_ms: 800,
|
||||
interaction_query: Some("willian caprino cibersegurança blaze".into()),
|
||||
interaction_selector: Some("input[name='q']".into()),
|
||||
skip_challenge_check: true,
|
||||
};
|
||||
let page = browser
|
||||
.fetch_html_with_options(request_id, "https://duckduckgo.com/", options)
|
||||
.await?;
|
||||
println!("B final_url: {}", page.final_url);
|
||||
println!("B html_len: {}", page.html.len());
|
||||
std::fs::write("/tmp/ddg_home.html", &page.html)?;
|
||||
println!("B HTML salvo em /tmp/ddg_home.html");
|
||||
|
||||
// Teste C: fetch direto de /html/?q=...
|
||||
println!("\n=== Teste C: fetch direto /html/?q= ===");
|
||||
let options2 = BrowserFetchOptions {
|
||||
warmup_url: Some("https://duckduckgo.com/".into()),
|
||||
wait_after_load_ms: 2_000,
|
||||
block_heavy_resources: true,
|
||||
max_html_bytes: 8 * 1024 * 1024,
|
||||
scroll_rounds: 2,
|
||||
scroll_delay_ms: 600,
|
||||
interaction_query: None,
|
||||
interaction_selector: None,
|
||||
skip_challenge_check: true,
|
||||
};
|
||||
let url = "https://duckduckgo.com/html/?q=willian+caprino+ciberseguran%C3%A7a+blaze&kl=br&kp=1";
|
||||
let page2 = browser
|
||||
.fetch_html_with_options(request_id, url, options2)
|
||||
.await?;
|
||||
println!("C final_url: {}", page2.final_url);
|
||||
println!("C html_len: {}", page2.html.len());
|
||||
std::fs::write("/tmp/ddg_html.html", &page2.html)?;
|
||||
println!("C HTML salvo em /tmp/ddg_html.html");
|
||||
|
||||
// Teste D: backend HTTP (reqwest + proxy) direto em /html/?q=.
|
||||
println!("\n=== Teste D: backend HTTP /html/?q= ===");
|
||||
let session = http.fresh(request_id)?;
|
||||
let http_url =
|
||||
"https://duckduckgo.com/html/?q=willian+caprino+ciberseguran%C3%A7a+blaze&kl=br&kp=1";
|
||||
match session
|
||||
.warm_then_get_text(request_id, "https://duckduckgo.com/")
|
||||
.await
|
||||
{
|
||||
Ok(_) => println!("D warm ok"),
|
||||
Err(e) => println!("D warm err: {e}"),
|
||||
}
|
||||
match session.warm_then_get_text(request_id, http_url).await {
|
||||
Ok(resp) => {
|
||||
println!("D OK len={} status", resp.text.len());
|
||||
std::fs::write("/tmp/ddg_http.html", &resp.text).ok();
|
||||
println!("D salvo /tmp/ddg_http.html");
|
||||
let lowered = resp.text.to_ascii_lowercase();
|
||||
for n in [
|
||||
"anomaly-modal",
|
||||
"bots use duckduckgo",
|
||||
"result__a",
|
||||
"/l/?uddg=",
|
||||
] {
|
||||
let c = lowered.matches(n).count();
|
||||
println!(" {n}: {c}");
|
||||
}
|
||||
}
|
||||
Err(e) => println!("D ERRO: {e}"),
|
||||
}
|
||||
|
||||
println!("\n=== trace de seletores no /html/ ===");
|
||||
let doc_html = std::fs::read_to_string("/tmp/ddg_html.html")?;
|
||||
for needle in [
|
||||
"result__a",
|
||||
"result__snippet",
|
||||
"data-testid=\"result",
|
||||
"class=\"result",
|
||||
"/l/?uddg=",
|
||||
"<h2",
|
||||
] {
|
||||
let n = doc_html
|
||||
.to_ascii_lowercase()
|
||||
.matches(needle.to_ascii_lowercase().as_str())
|
||||
.count();
|
||||
println!(" {needle}: {n}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
use backend::browser::{BrowserFetchOptions, BrowserModule, BrowserModuleConfig};
|
||||
use backend::persona;
|
||||
use backend::proxy::ProxyConfig;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let _ = dotenvy::dotenv();
|
||||
backend::logs::init();
|
||||
let request_id = "diag-yt";
|
||||
|
||||
let proxy = ProxyConfig::from_env(request_id)?;
|
||||
let browser_config = BrowserModuleConfig {
|
||||
no_sandbox: true,
|
||||
navigation_timeout_secs: 75,
|
||||
..Default::default()
|
||||
};
|
||||
let browser = BrowserModule::new(proxy, browser_config)?;
|
||||
|
||||
let persona = persona::generate(request_id, Some("br"));
|
||||
println!(
|
||||
"persona: chrome_major={} platform={:?} locale={}",
|
||||
persona.chrome_major, persona.platform, persona.locale
|
||||
);
|
||||
|
||||
let options = BrowserFetchOptions {
|
||||
warmup_url: Some("https://www.youtube.com/".into()),
|
||||
wait_after_load_ms: 6_000,
|
||||
block_heavy_resources: false,
|
||||
max_html_bytes: 32 * 1024 * 1024,
|
||||
scroll_rounds: 0,
|
||||
scroll_delay_ms: 800,
|
||||
interaction_query: None,
|
||||
interaction_selector: None,
|
||||
skip_challenge_check: false,
|
||||
};
|
||||
|
||||
let url = "https://www.youtube.com/results?search_query=podcast+tecnologia&sp=EgIQAg%3D%3D&hl=pt-BR&gl=BR";
|
||||
let page = browser
|
||||
.fetch_html_with_options_and_persona(request_id, url, options, persona)
|
||||
.await?;
|
||||
|
||||
println!("final_url: {}", page.final_url);
|
||||
println!("html_len: {}", page.html.len());
|
||||
std::fs::write("/tmp/yt_diag.html", &page.html)?;
|
||||
println!("html salvo em /tmp/yt_diag.html");
|
||||
|
||||
// Procurar sinais de challenge/cookiewall
|
||||
let lower = page.html.to_ascii_lowercase();
|
||||
let needles = [
|
||||
"unusual traffic",
|
||||
"detected unusual traffic",
|
||||
"não sou um robô",
|
||||
"recaptcha",
|
||||
"hcaptcha",
|
||||
"cf-chl-challenge",
|
||||
"consent.google.com",
|
||||
"consent.youtube",
|
||||
"g-recaptcha",
|
||||
"sorry/index",
|
||||
"ogat",
|
||||
"servicey",
|
||||
"our systems have detected",
|
||||
"consent.youtube.com",
|
||||
"agree",
|
||||
"aceitar",
|
||||
"sign in",
|
||||
"faça login",
|
||||
"before you continue",
|
||||
"antes de continuar",
|
||||
];
|
||||
for n in needles {
|
||||
if let Some(idx) = lower.find(n) {
|
||||
let start = idx.saturating_sub(80);
|
||||
let ctx: String = page.html.chars().skip(start).take(200).collect();
|
||||
println!("FOUND needle {n:?}: ...{ctx}...");
|
||||
}
|
||||
}
|
||||
|
||||
if page.html.contains("ytInitialData") {
|
||||
println!("ytInitialData encontrado");
|
||||
} else {
|
||||
println!("ytInitialData AUSENTE");
|
||||
}
|
||||
if page.html.contains("ytInitialPlayerResponse") {
|
||||
println!("ytInitialPlayerResponse encontrado (raro em /results)");
|
||||
}
|
||||
println!("\ntitle (primeiros 500 chars):");
|
||||
if let Some(start) = page.html.find("<title") {
|
||||
let end = page.html[start..]
|
||||
.find("</title>")
|
||||
.map(|e| start + e + 8)
|
||||
.unwrap_or(start + 500);
|
||||
println!("{}", &page.html[start..end.min(page.html.len())]);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user