GitHub - MikeLuu99/searxng-rust: a metadata search engine similar to searxng in rust

A SearXNG-style metadata search engine written in Rust. Fans out queries to multiple search engines concurrently, scrapes their HTML results, deduplicates by normalized URL, and ranks using Reciprocal Rank Fusion (RRF). How it works A search request arrives at GET /search?q=<query> The query is sent concurrently to DuckDuckGo, Brave, Startpage, and Yahoo via reqwest Each engine parses the HTML response with scraper (CSS selectors over Mozilla's html5ever) Results are deduplicated by normalized URL (tracking params stripped, locale prefixes removed, query params sorted) Duplicate URLs are merged and scored with RRF (score = Σ 1/(60 + rank) across engines) — pages returned by multiple engines rank higher The top results are returned as JSON Requirements Rust 1.75+ Cargo Installation As a library cargo add metadata-search-engine-rs Or add manually to Cargo.toml: [dependencies] metadata-search-engine-rs = "0.1" As a server (from source) git clone https://github.com/MikeLuu99/searxng-rust cd metadata-search-engine-rs cargo build --release Running cargo run PORT=8080 MAX_RESULTS=20 cargo run --release Enable debug logging: RUST_LOG=debug cargo run Examples Add to your Cargo.toml: [dependencies] metadata-search-engine-rs = "0.1" tokio = { version = "1", features = ["full"] } Query a single engine use std::sync::Arc; use metadata_search_engine_rs::engines::{DuckDuckGoEngine, SearchEngine, build_http_client}; #[tokio::main] async fn main() -> anyhow::Result<()> { let client = Arc::new(build_http_client()?); let engine = DuckDuckGoEngine { client }; let results = engine.search("rust programming", 5).await?; for r in results { println!("{}\n {}", r.title, r.url); } Ok(()) } Fan out to all engines and get RRF-ranked results use std::sync::Arc; use metadata_search_engine_rs::{ aggregator::{aggregate, query_all_engines}, engines::{BraveEngine, DuckDuckGoEngine, SearchEngine, StartpageEngine, YahooEngine, build_http_client}, }; #[tokio::main] async fn main() -> anyhow::Result<()> { let client = Arc::new(build_http_client()?); let engines: Vec<Arc<dyn SearchEngine>> = vec![ Arc::new(DuckDuckGoEngine { client: Arc::clone(&client) }), Arc::new(BraveEngine { client: Arc::clone(&client) }), Arc::new(StartpageEngine { client: Arc::clone(&client) }), Arc::new(YahooEngine { client: Arc::clone(&client) }), ]; let (successes, failures) = query_all_engines(&engines, "rust programming", 10).await; for (name, err) in &failures { eprintln!("engine {name} failed: {err}"); } let results = aggregate(successes, 10); for r in &results { println!("[{:.3}] ({}) {}", r.score, r.engines.join(", "), r.title); println!(" {}", r.url); } Ok(()) } Use only specific engines use std::sync::Arc; use metadata_search_engine_rs::{ aggregator::{aggregate, query_all_engines}, engines::{BraveEngine, DuckDuckGoEngine, SearchEngine, build_http_client}, }; #[tokio::main] async fn main() -> anyhow::Result<()> { let client = Arc::new(build_http_client()?); let engines: Vec<Arc<dyn SearchEngine>> = vec![ Arc::new(DuckDuckGoEngine { client: Arc::clone(&client) }), Arc::new(BraveEngine { client: Arc::clone(&client) }), ]; let (successes, _) = query_all_engines(&engines, "tokio async rust", 5).await; for r in aggregate(successes, 5) { println!("{} — {}", r.title, r.url); if let Some(snippet) = r.snippet { println!(" {snippet}"); } } Ok(()) } API GET /health curl http://localhost:3000/health { "status": "ok" } GET /search?q=<query> curl "http://localhost:3000/search?q=rust" { "query": "rust", "results": [ { "title": "Rust Programming Language", "url": "https://rust-lang.org/", "snippet": "A language empowering everyone to build reliable and efficient software.", "engines": ["duckduckgo", "brave", "startpage", "yahoo"], "score": 0.049 } ], "engines_queried": ["duckduckgo", "brave", "startpage", "yahoo"], "engines_failed": [] } Error responses: Case Status Body Missing q 400 {"error": "query parameter 'q' is required"} Empty q 400 {"error": "query parameter 'q' cannot be empty"} All engines fail 503 {"error": "all engines failed to respond"} Tests # All unit tests cargo test # Specific module cargo test normalizer cargo test aggregator cargo test engines::duckduckgo cargo test engines::brave cargo test engines::startpage cargo test engines::yahoo cargo test server::handlers # Live tests (hit real search engines — requires internet) cargo test -- --ignored test_live Live tests are marked #[ignore] so they don't run in CI by default. Run them manually to verify HTML selectors still work against the real sites. Terminal UI A ratatui-based TUI is available as a crate to install here. Access the code via github Adding a new search engine Create src/engines/<name>.rs Define a struct holding Arc<reqwest::Client> Implement the SearchEngine trait: impl SearchEngine for MyEngine { fn name(&self) -> &'static str { "myengine" } fn search<'a>( &'a self, query: &'a str, max_results: usize, ) -> BoxFuture<'a, Result<Vec<SearchResult>, EngineError>> { Box::pin(async move { // fetch HTML, parse with scraper, return Vec<SearchResult> }) } } Add it to engines/mod.rs and wire it in main.rs