290 lines
9.7 KiB
Rust
290 lines
9.7 KiB
Rust
//! Infrastructure — Search utilities.
|
|
//!
|
|
//! Ported from Shirokami-API `scraper/search/*.js`:
|
|
//! bmkg (BMKG earthquake), jadwal-sholat (myquran.com), weather (OpenWeather),
|
|
//! google (HTML scrape), yt (via yt-dlp flat-playlist).
|
|
|
|
use crate::infrastructure::utils::http_client::http_client;
|
|
use reqwest::header::USER_AGENT;
|
|
use scraper::{Html, Selector};
|
|
use serde_json::{json, Value};
|
|
use std::process::Stdio;
|
|
|
|
/// BMKG Indonesia earthquake info (auto-gempa, terkini, dirasakan).
|
|
pub async fn fetch_bmkg() -> Result<Value, String> {
|
|
let urls = [
|
|
"https://data.bmkg.go.id/DataMKG/TEWS/autogempa.json",
|
|
"https://data.bmkg.go.id/DataMKG/TEWS/gempaterkini.json",
|
|
"https://data.bmkg.go.id/DataMKG/TEWS/gempadirasakan.json",
|
|
];
|
|
let mut auto = Value::Null;
|
|
let mut terkini: Vec<Value> = Vec::new();
|
|
let mut dirasakan: Vec<Value> = Vec::new();
|
|
for url in urls {
|
|
let resp = http_client()
|
|
.client()
|
|
.get(url)
|
|
.header(
|
|
USER_AGENT,
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
|
|
)
|
|
.send()
|
|
.await
|
|
.map_err(|e| format!("HTTP error: {}", e))?;
|
|
if !resp.status().is_success() {
|
|
return Err(format!("BMKG returned HTTP {}", resp.status()));
|
|
}
|
|
let data: Value = resp
|
|
.json()
|
|
.await
|
|
.map_err(|e| format!("JSON parse: {}", e))?;
|
|
if url.contains("autogempa") {
|
|
auto = data
|
|
.pointer("/Infogempa/gempa")
|
|
.cloned()
|
|
.unwrap_or(Value::Null);
|
|
} else if url.contains("gempaterkini") {
|
|
terkini = data
|
|
.pointer("/Infogempa/gempa")
|
|
.and_then(|g| g.as_array())
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
} else {
|
|
dirasakan = data
|
|
.pointer("/Infogempa/gempa")
|
|
.and_then(|g| g.as_array())
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
}
|
|
}
|
|
Ok(json!({
|
|
"autogempa": auto,
|
|
"gempaterkini": terkini,
|
|
"gempadirasakan": dirasakan,
|
|
}))
|
|
}
|
|
|
|
/// Jadwal sholat from myquran.com.
|
|
pub async fn fetch_jadwal_sholat(kota: &str) -> Result<Value, String> {
|
|
// 1. Find city id
|
|
let search_url = format!(
|
|
"https://api.myquran.com/v2/sholat/kota/cari/{}",
|
|
urlencode(kota)
|
|
);
|
|
let city_data = get_json(&search_url).await?;
|
|
if city_data
|
|
.get("status")
|
|
.and_then(|s| s.as_bool())
|
|
.unwrap_or(false)
|
|
!= true
|
|
{
|
|
return Ok(json!({ "error": "Kota tidak ditemukan" }));
|
|
}
|
|
let id_list = city_data
|
|
.get("data")
|
|
.and_then(|d| d.as_array())
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
if id_list.is_empty() {
|
|
return Ok(json!({ "error": "Kota tidak ditemukan" }));
|
|
}
|
|
|
|
// 2. Today's date (Asia/Jakarta). Use chrono with a fixed offset approximation.
|
|
let now = chrono::Local::now();
|
|
let (year, month, day) = (
|
|
now.format("%Y").to_string(),
|
|
now.format("%m").to_string(),
|
|
now.format("%d").to_string(),
|
|
);
|
|
|
|
let mut results: Vec<Value> = Vec::new();
|
|
for city in id_list.iter().take(5) {
|
|
let cid = city.get("id").and_then(|i| i.as_str()).unwrap_or("");
|
|
let jadwal_url = format!(
|
|
"https://api.myquran.com/v2/sholat/jadwal/{}/{}/{}/{}",
|
|
cid, year, month, day
|
|
);
|
|
if let Ok(sched) = get_json(&jadwal_url).await {
|
|
if let Some(d) = sched.get("data") {
|
|
results.push(json!({
|
|
"lokasi": d.get("lokasi").cloned().unwrap_or(Value::Null),
|
|
"daerah": d.get("daerah").cloned().unwrap_or(Value::Null),
|
|
"jadwal": d.get("jadwal").cloned().unwrap_or(Value::Null),
|
|
}));
|
|
}
|
|
}
|
|
}
|
|
|
|
if results.is_empty() {
|
|
return Ok(json!({ "error": "Tidak dapat mengambil jadwal" }));
|
|
}
|
|
Ok(json!({ "total": results.len(), "schedules": results }))
|
|
}
|
|
|
|
/// OpenWeather current weather (uses env OPENWEATHER_API_KEY, falls back to source's key).
|
|
pub async fn fetch_weather(city: &str) -> Result<Value, String> {
|
|
let key = std::env::var("OPENWEATHER_API_KEY")
|
|
.unwrap_or_else(|_| "060a6bcfa19809c2cd4d97a212b19273".to_string());
|
|
let url = format!(
|
|
"https://api.openweathermap.org/data/2.5/weather?q={}&units=metric&appid={}",
|
|
urlencode(city),
|
|
key
|
|
);
|
|
get_json(&url).await
|
|
}
|
|
|
|
/// Google web search via HTML scraping.
|
|
pub async fn fetch_google(query: &str) -> Result<Value, String> {
|
|
let url = format!(
|
|
"https://www.google.com/search?q={}&safe=off&hl=en&gl=us",
|
|
urlencode(query)
|
|
);
|
|
let resp = http_client()
|
|
.client()
|
|
.get(&url)
|
|
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
|
.header("Accept-Language", "en-US")
|
|
.send()
|
|
.await
|
|
.map_err(|e| format!("HTTP error: {}", e))?;
|
|
let body = resp
|
|
.text()
|
|
.await
|
|
.map_err(|e| format!("Failed to read body: {}", e))?;
|
|
|
|
let document = Html::parse_document(&body);
|
|
let result_sel =
|
|
Selector::parse("div.g, div[data-hveid]").unwrap_or_else(|_| Selector::parse("a").unwrap());
|
|
let link_sel = Selector::parse("a").unwrap();
|
|
let h3_sel = Selector::parse("h3").unwrap();
|
|
|
|
let mut results: Vec<Value> = Vec::new();
|
|
for node in document.select(&result_sel) {
|
|
let link = node
|
|
.select(&link_sel)
|
|
.find_map(|a| a.value().attr("href"))
|
|
.and_then(|h| h.strip_prefix("/url?q="))
|
|
.map(|h| h.split('&').next().unwrap_or(h).to_string());
|
|
let title = node
|
|
.select(&h3_sel)
|
|
.next()
|
|
.map(|h| h.text().collect::<String>());
|
|
if title.is_some() && link.is_some() {
|
|
results.push(json!({
|
|
"title": title.unwrap_or_default(),
|
|
"description": "",
|
|
"link": link.unwrap_or_default(),
|
|
}));
|
|
}
|
|
if results.len() >= 10 {
|
|
break;
|
|
}
|
|
}
|
|
|
|
Ok(json!(results))
|
|
}
|
|
|
|
/// YouTube search via yt-dlp flat playlist (real results, scraping).
|
|
pub async fn fetch_yt_search(query: &str) -> Result<Value, String> {
|
|
let q = format!("ytsearch15:{}", query);
|
|
let output = run_ytdlp(&q).await?;
|
|
|
|
let mut videos: Vec<Value> = Vec::new();
|
|
let arr: Vec<Value> = serde_json::from_str(&output).unwrap_or_default();
|
|
for v in arr.iter().take(15) {
|
|
videos.push(json!({
|
|
"title": v.get("title").cloned().unwrap_or(Value::Null),
|
|
"id": v.get("id").cloned().unwrap_or(Value::Null),
|
|
"url": v.get("webpage_url").cloned().unwrap_or(Value::Null),
|
|
"thumbnail": v.get("thumbnail").cloned().unwrap_or(Value::Null),
|
|
"duration": v.get("duration").cloned().unwrap_or(Value::Null),
|
|
"views": v.get("view_count").cloned().unwrap_or(Value::Null),
|
|
"author": {
|
|
"name": v.pointer("/channel").cloned().unwrap_or(Value::Null),
|
|
"url": v.get("channel_url").cloned().unwrap_or(Value::Null),
|
|
},
|
|
}));
|
|
}
|
|
|
|
Ok(json!({ "total": videos.len(), "videos": videos }))
|
|
}
|
|
|
|
// ============================================================================
|
|
// Helpers
|
|
// ============================================================================
|
|
|
|
async fn get_json(url: &str) -> Result<Value, String> {
|
|
let resp = http_client()
|
|
.client()
|
|
.get(url)
|
|
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
|
.send()
|
|
.await
|
|
.map_err(|e| format!("HTTP error: {}", e))?;
|
|
if !resp.status().is_success() {
|
|
return Err(format!("HTTP {} from upstream", resp.status()));
|
|
}
|
|
resp.json::<Value>()
|
|
.await
|
|
.map_err(|e| format!("JSON parse: {}", e))
|
|
}
|
|
|
|
/// Run yt-dlp with `--dump-json --flat-playlist` for the given search/query arg.
|
|
async fn run_ytdlp(query_arg: &str) -> Result<String, String> {
|
|
let bins = [
|
|
"/usr/local/bin/yt-dlp-native",
|
|
"/home/code/hermes-agent/.venv/bin/yt-dlp",
|
|
"yt-dlp",
|
|
];
|
|
let bin = bins
|
|
.iter()
|
|
.find(|b| std::path::Path::new(b).exists() || **b == "yt-dlp")
|
|
.copied()
|
|
.unwrap_or("yt-dlp");
|
|
|
|
let child = tokio::process::Command::new(bin)
|
|
.args(["--dump-json", "--flat-playlist", "--no-warnings", query_arg])
|
|
.stdout(Stdio::piped())
|
|
.stderr(Stdio::null())
|
|
.spawn()
|
|
.map_err(|e| format!("Failed to spawn yt-dlp: {}", e))?;
|
|
|
|
let output = child
|
|
.wait_with_output()
|
|
.await
|
|
.map_err(|e| format!("yt-dlp failed: {}", e))?;
|
|
|
|
let stdout = String::from_utf8(output.stdout).map_err(|e| format!("Invalid UTF-8: {}", e))?;
|
|
// `--flat-playlist` outputs multiple JSON lines (one per video).
|
|
let mut joined = String::new();
|
|
joined.push('[');
|
|
let mut first = true;
|
|
for line in stdout.lines() {
|
|
let line = line.trim();
|
|
if line.is_empty() {
|
|
continue;
|
|
}
|
|
if !first {
|
|
joined.push(',');
|
|
}
|
|
joined.push_str(line);
|
|
first = false;
|
|
}
|
|
joined.push(']');
|
|
Ok(joined)
|
|
}
|
|
|
|
fn urlencode(s: &str) -> String {
|
|
let mut out = String::new();
|
|
for b in s.bytes() {
|
|
match b {
|
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
|
out.push(b as char)
|
|
}
|
|
b' ' => out.push_str("%20"),
|
|
_ => out.push_str(&format!("%{:02X}", b)),
|
|
}
|
|
}
|
|
out
|
|
}
|