commit 80c96eaa429291962b58cd3bc720c638251fb8bb Author: asepharyana Date: Thu Jul 9 22:07:03 2026 +0700 chore: initial commit for asepharyana-hub-scraper diff --git a/.github/workflows/notify-parent.yml b/.github/workflows/notify-parent.yml new file mode 100644 index 0000000..cdbf361 --- /dev/null +++ b/.github/workflows/notify-parent.yml @@ -0,0 +1,24 @@ +name: Notify Parent Repo + +on: + push: + branches: + - main + +jobs: + dispatch: + runs-on: ubuntu-latest + steps: + - name: Trigger root monorepo build + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.DISPATCH_TOKEN }} + repository: MythEclipse/ultimate-asepharyana.tech + event-type: submodule-updated + client-payload: | + { + "service": "scraper-api", + "ref": "${{ github.ref }}", + "sha": "${{ github.sha }}", + "actor": "${{ github.actor }}" + } diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c9eb931 --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +apps/gmw/ +# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files for more about ignoring files. + +# compiled output +dist +tmp +out-tsc +error.log +# dependencies +node_modules +.bun/** +.turbo/ +# IDEs and editors +/.idea +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +**/target/** +# IDE - VSCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +# misc +/.sass-cache +/connect.lock +/coverage +/libpeerconnection.log +npm-debug.log +yarn-error.log +testem.log +/typings + +# System Files +.DS_Store +Thumbs.db +.claude +# Next.js +.next +out +**/.codegraph/** +**/.claude/** +test-output +**/**.env +**/**.env.** +vite.config.*.timestamp* +vitest.config.*.timestamp* + + +storybook-static +~/.bun/** +docs/dependency-map.md +docs/handoff-log.jsonl +docs/observability.md +docs/quality-gates.json +docs/workflow-state.json +docs/todo.md +**/vendor/ + +# moonrepo +.moon/cache +.~moon** \ No newline at end of file diff --git a/AGENT.md b/AGENT.md new file mode 100644 index 0000000..2f12b61 --- /dev/null +++ b/AGENT.md @@ -0,0 +1,27 @@ +# AGENT.md - Universal AI Entry Point + +This document serves as the unified entry point for all AI agents (Gemini, Claude, GPT, etc.) interacting with the **Scraping & CDN Service**. + +## 🚀 Mission Statement + +To provide a specialized, zero-bloat backend engine for high-concurrency web scraping and image persistent caching. + +## 🛑 Global AI Protocols + +As an AI agent, you **MUST** adhere to the following when working on this codebase: + +1. **Professionalism**: Maintain a factual, technical, and objective tone. +2. **No Hyperbole**: Prohibited from using marketing-speak or exaggerated praise (e.g., "amazing", "powerful"). +3. **Minimalism**: Prioritize the **Zero-Bloat Policy**. If a request adds unnecessary complexity or dependencies, challenge the user and suggest a leaner alternative. +4. **Zero Suppression**: Never use suppression flags (`#[allow]`, `@ts-ignore`) to bypass warnings. Fix the underlying logic or types. +5. **Observability**: Ensure critical business flows emit structured logs and request context where needed. + +## 🔗 Technical Context + +- **Architecture**: [GEMINI.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/GEMINI.md) (Logic flows, tech stack). +- **Maintenance**: [Development Guide](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/docs/development.md) (Coding standards). +- **Observability**: [Metrics Guide](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/docs/observability.md) (Standard telemetry). + +--- + +*If you are an AI assistant, start by reading [GEMINI.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/GEMINI.md) for the full architectural context.* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..0c31d36 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,177 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Scraper service — a Rust/Axum backend for web scraping (anime/komik data extraction) and image proxy/CDN caching. Serves as the backend engine consumed by the `apps/solidjs` frontend. + +## Commands + +```bash +# Development +cargo run # Start server (binds 0.0.0.0:4091) +cargo test # Run all tests +cargo clippy -- -D warnings # Lint (warnings are errors) +cargo fmt # Auto-format all source files + +# Release build (full LTO, single CGU, stripped) +cargo build --release + +# PM2 production +pm2 start ecosystem.config.cjs --env production # Uses target/release/scraper +``` + +## Architecture + +### Modular MVC + Service + Repository + +Setiap module mengikuti arsitektur layered yang identik: + +``` +Request → Router (route.rs) → Controller → Service → Repository → Parser + │ + ├── Redis (L1 cache) + ├── SeaORM/MySQL (L2, image_cache) + └── External HTTP (alqanime.si, picser CDN) +``` + +### Directory Layout + +``` +src/ +├── main.rs # Entry point: builds Application, calls run() +├── lib.rs # Public module declarations +├── app.rs # Router assembly: modules + metrics + swagger + middleware layers +├── bootstrap/mod.rs # Application::build(): tracing, Redis, browser pool, DB, AppState +├── modules/ # Feature modules (vertical slices) +│ ├── anime/ # Otakudesu anime scraper +│ ├── anime2/ # Alqanime.si anime scraper +│ ├── komik/ # Komik scraper +│ └── proxy/ # Image proxy/cache/audit endpoints +└── shared/ # Cross-cutting infrastructure + ├── config/ # Lazy-static AppConfig from env vars (fail-fast at startup) + ├── state/ # AppState (redis_pool, db, semaphore, event_bus) + ├── database/ + │ ├── traits/ # ScrapingRepository, ImageCacheRepository (async_trait) + │ ├── repositories/ # SeaOrmImageCacheRepository (impl ImageCacheRepository) + │ └── persistence/ # SeaORM entities (image_cache) + ├── services/images/ # ImageCache service + apply_cached_posters helper + ├── errors/ # AppError enum → axum IntoResponse (500/404 by variant) + ├── observability/ # Utoipa/Swagger OpenAPI doc + ├── scheduler/ # Cron jobs (daily cache cleanup at 2 AM) + ├── browser/ # Headless Chrome pool for JS-rendered scraping + ├── scrapers/ # Site-specific scrapers (otakudesu) + ├── utils/ # Cache helper, HTTP client, scraping helpers, retry, conversions + ├── middlewares/ # Logging, rate limiting + ├── events/ # EventBus for repair state updates + └── types/ # ApiResponse, shared entity types (HasPoster trait, Pagination) +``` + +### Module Structure (identik untuk setiap module) + +Setiap `src/modules//`: + +| File | Peran | Pola | +|---|---|---| +| `route.rs` | Daftar endpoint, mapping URL → controller | `Router>`, tidak ada logic | +| `controller.rs` | Extract State/Path/Query/Body, panggil service | `Result, AppError>` | +| `service.rs` | Business logic, caching, delegasi ke repository + parser | Struct dengan repo di-inject via constructor `new(repo: XRepository)` | +| `repository.rs` | HTTP fetching, URL builders, DB queries | Struct + `impl ScrapingRepository` trait | +| `parser.rs` | HTML parsing dengan `scraper` crate | Free functions → `Result`, via `spawn_blocking` | +| `schema.rs` | Validasi query/path/body params | Struct `Deserialize` + `ToSchema` | +| `types.rs` | Response structs | `Serialize` + `ToSchema`, `impl HasPoster` jika punya poster | + +### Dependency Injection + +Semua service menerima dependency via constructor: + +```rust +// Controller creates and injects dependencies +let repo = AnimeRepository::new(); +let service = AnimeService::new(repo); +service.get_anime_index(app_state).await.map(Json) + +// Service stores injected repo +pub struct AnimeService { + repository: AnimeRepository, +} +impl AnimeService { + pub fn new(repository: AnimeRepository) -> Self { Self { repository } } +} +``` + +### Image Caching Architecture + +Single unified image cache system: + +1. **Trait**: `ImageCacheRepository` (`shared/database/traits/image_cache.rs`) — Redis ops, DB ops, locks, cache invalidation +2. **Impl**: `SeaOrmImageCacheRepository` (`shared/database/repositories/image_cache.rs`) +3. **Service**: `ImageCache` struct (`shared/services/images/cache.rs`) — download, MIME-verify dengan `infer`, upload ke Picser CDN, verifikasi CDN URL (10 retry dengan backoff) +4. **Concurrency**: `Semaphore` (default 5 concurrent uploads) + request coalescing via `DashMap` +5. **Lazy batch helper**: `cache_image_urls_batch_lazy()` — Redis batch check → DB batch check → background spawn untuk misses + +### HasPoster Trait & apply_cached_posters + +`HasPoster` trait di `shared/types/entities/anime.rs` memungkinkan generic poster caching: + +```rust +pub trait HasPoster { + fn poster(&self) -> &str; + fn set_poster(&mut self, url: String); +} +``` + +Semua item type dengan field `poster` mengimplementasikan trait ini (`OngoingAnimeItem`, `KomikItem`, `FilterAnimeItem`, `Recommendation`, dll). + +`apply_cached_posters()` di `shared/services/images/cache.rs` menerima `&mut [T]` where `T: HasPoster`, menggantikan pola manual ~15 baris yang sebelumnya berulang di setiap service method. + +### ScrapingRepository Trait + +```rust +#[async_trait] +pub trait ScrapingRepository: Send + Sync { + async fn fetch_html(&self, url: &str) -> Result; +} +``` + +Semua module repository (`AnimeRepository`, `Anime2Repository`, `KomikRepository`, `ProxyRepository`) mengimplementasikan trait ini. + +### Error Handling + +`AppError` enum di `src/shared/errors/app_error.rs` — derives `thiserror::Error` dan implements `IntoResponse` (404 untuk `NotFound`, 500 untuk lainnya). + +**Kontrak error per layer:** +- **Parser** → `Result` +- **Repository** → `Result` (via `ScrapingRepository` trait) +- **Service** → `Result` (tidak ada `Result` atau `Box`) +- **Controller** → `Result, AppError>` (kecuali proxy yang return raw `Response`) + +### Configuration + +`src/shared/config/mod.rs` — global `CONFIG` lazy-static loaded from: +1. `.env` file (dotenvy) +2. `config/default.toml` / `config/{RUN_MODE}.toml` +3. Environment variables (`APP__` prefix or legacy `DATABASE_URL`/`JWT_SECRET`/`REDIS_URL`) + +Panics at startup if required config is missing — intentional fail-fast design. + +## Constraints + +- **No suppression flags**: `#[allow(...)]`, `#[ignore]`, `@ts-ignore` are prohibited. Fix the underlying issue. +- **Lint strictness**: `unsafe_code = "forbid"`, `panic = "deny"`, `todo = "deny"`, `unimplemented = "deny"`, `unwrap_used = "warn"`, `expect_used = "warn"` +- **Minimal dependencies**: Before adding a crate, evaluate if existing deps or std can handle it. +- **Dead code**: Remove unused functions, types, modules rather than leaving them. +- **Performance**: Use `spawn_blocking` for CPU-heavy work (HTML parsing). +- **No duplicate infrastructure**: Satu trait, satu impl. Jangan membuat trait/repository duplikat seperti `ImageRepository` dan `ImageCacheRepository` yang berbeda. +- **No thin wrappers**: Hindari wrapper tipis seperti `CacheImageUseCase` yang hanya meneruskan panggilan ke service lain. + +## Useful Endpoints + +- `GET /docs` — Swagger UI +- `GET /api-docs/openapi.json` — OpenAPI spec +- `POST /api/proxy/image-cache` — Cache an image URL +- `POST /api/proxy/image-cache/audit` — Audit/repair cached images +- `GET /api/anime/*` — Otakudesu scraping endpoints +- `GET /api/anime2/*` — Alqanime scraping endpoints +- `GET /api/komik/*` — Komik scraping endpoints diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c0729af --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5564 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core 0.5.6", + "axum-macros", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "multer", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.17", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "compression-codecs" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "config" +version = "0.15.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68cfe19cd7d23ffde002c24ffa5cda73931913ef394d5eaaa32037dc940c0c" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "pathdiff", + "ron", + "rust-ini", + "serde-untagged", + "serde_core", + "serde_json", + "toml", + "winnow", + "yaml-rust2", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croner" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum 0.27.2", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + +[[package]] +name = "data-url" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "serde", + "tokio", +] + +[[package]] +name = "deadpool-redis" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4d00bce7a9cfd07ded19530621a7df17c4c3b1c4e8fc2262471de404ce539a" +dependencies = [ + "deadpool", + "redis", + "serde", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.2", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "html5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e" +dependencies = [ + "log", + "markup5ever", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.3", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "infer" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +dependencies = [ + "cfb", +] + +[[package]] +name = "inherent" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c727f80bfa4a6c6e2508d2f05b6f4bfce242030bd88ed15ae5331c5b5d30fba7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.7.3", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "opentelemetry" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "pgvector" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc58e2d255979a31caa7cabfa7aac654af0354220719ab7a68520ae7a91e8c0b" +dependencies = [ + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared 0.13.1", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", + "yansi", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redis" +version = "0.32.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44" +dependencies = [ + "bytes", + "cfg-if", + "combine", + "futures-util", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "ryu", + "sha1_smol", + "socket2 0.6.3", + "tokio", + "tokio-rustls", + "tokio-util", + "url", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ron" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" +dependencies = [ + "bitflags", + "once_cell", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.117", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust_decimal" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", + "wasm-bindgen", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93cecd86d6259499c844440546d02f55f3e17bd286e529e48d1f9f67e92315cb" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "scraper-service" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.8.8", + "backoff", + "base64", + "bytes", + "chrono", + "clap", + "config", + "dashmap", + "data-url", + "deadpool-redis", + "dotenvy", + "flate2", + "futures", + "governor", + "hex", + "hmac", + "http", + "infer", + "itertools", + "log", + "mime_guess", + "once_cell", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "rand 0.8.5", + "rayon", + "redis", + "regex", + "reqwest", + "scraper", + "sea-orm", + "serde", + "serde_json", + "sha1", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tl", + "tokio", + "tokio-cron-scheduler", + "tokio-util", + "tower 0.5.3", + "tower-http", + "tracing", + "tracing-subscriber", + "url", + "urlencoding", + "utoipa", + "utoipa-axum", + "utoipa-swagger-ui", + "uuid", + "walkdir", +] + +[[package]] +name = "sea-bae" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f694a6ab48f14bc063cfadff30ab551d3c7e46d8f81836c51989d548f44a2a25" +dependencies = [ + "heck 0.4.1", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "sea-orm" +version = "1.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d945f62558fac19e5988680d2fdf747b734c2dbc6ce2cb81ba33ed8dde5b103" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "chrono", + "derive_more", + "futures-util", + "log", + "ouroboros", + "pgvector", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-query-binder", + "serde", + "serde_json", + "sqlx", + "strum 0.26.3", + "thiserror 2.0.18", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sea-orm-macros" +version = "1.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c2e64a50a9cc8339f10a27577e10062c7f995488e469f2c95762c5ee847832" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "sea-bae", + "syn 2.0.117", + "unicode-ident", +] + +[[package]] +name = "sea-query" +version = "0.32.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5d1c518eaf5eda38e5773f902b26ab6d5e9e9e2bb2349ca6c64cf96f80448c" +dependencies = [ + "bigdecimal", + "chrono", + "inherent", + "ordered-float", + "rust_decimal", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "sea-query-binder" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0019f47430f7995af63deda77e238c17323359af241233ec768aba1faea7608" +dependencies = [ + "bigdecimal", + "chrono", + "rust_decimal", + "sea-query", + "serde_json", + "sqlx", + "time", + "uuid", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf 0.13.1", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-untagged" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" +dependencies = [ + "erased-serde", + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bigdecimal", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.13.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rust_decimal", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.117", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.117", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "rust_decimal", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand 0.8.5", + "rust_decimal", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.18", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.13.1", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", + "proc-macro2", + "quote", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tl" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b130bd8a58c163224b44e217b4239ca7b927d82bf6cc2fea1fc561d15056e3f7" + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.3", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-cron-scheduler" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f50e41f200fd8ed426489bd356910ede4f053e30cebfbd59ef0f856f0d7432a" +dependencies = [ + "chrono", + "chrono-tz", + "croner", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc" +dependencies = [ + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.8+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" +dependencies = [ + "indexmap 2.13.0", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +dependencies = [ + "winnow", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.9", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "iri-string", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utoipa" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-axum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c25bae5bccc842449ec0c5ddc5cbb6a3a1eaeac4503895dc105a1138f8234a0" +dependencies = [ + "axum 0.8.8", + "paste", + "tower-layer", + "tower-service", + "utoipa", +] + +[[package]] +name = "utoipa-gen" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d79d08d92ab8af4c5e8a6da20c47ae3f61a0f1dabc1997cdf2d082b757ca08b" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.117", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "9.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55" +dependencies = [ + "axum 0.8.8", + "base64", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "url", + "utoipa", + "zip", +] + +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "serde", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web_atoms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" +dependencies = [ + "phf 0.13.1", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.6", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "yaml-rust2" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.13.0", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..9bf8eb1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,132 @@ + +[package] +name = "scraper-service" +version = "0.1.0" +edition = "2021" +description = "A batteries-included, production-ready Rust web framework built on Axum" +authors = ["Asep Haryana"] +license = "MIT" +repository = "https://github.com/MythEclipse/ultimate-asepharyana.tech" +keywords = ["web", "framework", "axum", "api", "rest"] +categories = ["web-programming::http-server", "web-programming::websocket"] +default-run = "scraper" + +# Dependensi yang dibutuhkan saat aplikasi berjalan +[dependencies] +axum = { version = "0.8.8", features = ["ws", "multipart", "macros"] } +tokio = { version = "1.49.0", features = ["full"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +dotenvy = "0.15" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +anyhow = "1.0" +# sqlx removed - SeaORM uses it internally via sqlx-postgres feature +sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid"] } +uuid = { version = "1.10.0", features = ["v4", "serde"] } +chrono = { version = "0.4", features = ["serde"] } + +bytes = "1.11.0" +futures = "0.3" +reqwest = { version = "0.12.28", features = ["json", "stream", "multipart"] } +http = "1.4.0" +sha1 = "0.10.6" +data-url = "0.3.2" +base64 = "0.22.1" +tokio-util = { version = "0.7.18", features = ["codec"] } +async-trait = "0.1.89" +regex = "1.12.2" +infer = "0.19.0" + +once_cell = "1.21.3" +urlencoding = "2.1" + +url = "2.5.8" +rand = "0.8" +tempfile = "3.24.0" +mime_guess = "2.0.5" + +tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] } +backoff = { version = "0.4", features = ["futures", "tokio"] } +dashmap = "6.1" +deadpool-redis = { version = "0.22.1", features = ["serde"] } +rayon = "1.11" +tl = "0.7.8" +tower = { version = "0.5", features = ["make"] } +scraper = "0.25.0" +flate2 = "1.1" +redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"] } +thiserror = "2.0.18" +itertools = "0.14" +clap = { version = "4.5", features = ["derive"] } +config = { version = "0.15.19", features = ["toml"] } +governor = "0.10.4" +tokio-cron-scheduler = "0.15.1" +hex = "0.4.3" +hmac = "0.12.1" +sha2 = "0.10.9" +log = "0.4" +walkdir = "2.5" +utoipa = { version = "5.0", features = ["axum_extras"] } +utoipa-swagger-ui = { version = "9.0", features = ["axum"] } +utoipa-axum = "0.2.0" + +# OpenTelemetry metrics +opentelemetry = { version = "0.27", features = ["metrics"] } +opentelemetry_sdk = { version = "0.27", features = ["metrics", "rt-tokio"] } +opentelemetry-otlp = { version = "0.27", features = ["metrics"] } +opentelemetry-semantic-conventions = "0.27" + + +# Dependensi yang hanya dibutuhkan untuk build script (build.rs) + +# Dependensi yang hanya dibutuhkan untuk tes +[dev-dependencies] +sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid", "mock"] } + +# Definisi targets secara eksplisit untuk menghindari peringatan cargo-chef (edition/plugin) +[lib] +name = "scraper_service" +path = "src/lib.rs" + +[[bin]] +name = "scraper" +path = "src/main.rs" + + + +[lints.rust] +unsafe_code = "forbid" +unused_variables = "deny" +unused_imports = "deny" +unused_must_use = "deny" + +[lints.clippy] +unwrap_used = "warn" +expect_used = "warn" +panic = "deny" +todo = "deny" +unimplemented = "deny" + +[features] +# default = ["ffmpeg"] +ffmpeg = [] + +# Profile optimasi untuk production - fokus pada performa runtime maksimal +[profile.release] +opt-level = 3 # Optimasi level maksimal +lto = "fat" # Full LTO untuk inlining maksimal dan binary optimal +codegen-units = 1 # Single codegen unit untuk optimasi maksimal (lebih lambat build, binary lebih cepat) +incremental = false # Disable incremental untuk optimasi penuh +debug = false # No debug info untuk binary lebih kecil +strip = true # Strip symbols untuk binary lebih kecil +panic = "abort" # Abort on panic (binary lebih kecil dan lebih cepat) +overflow-checks = false # Non-overflowing arithmetic for speed (disable in debug) + +# Optimize dependencies too +[profile.dev] +opt-level = 0 + +[profile.bench] +opt-level = 3 +lto = "fat" diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..74f2807 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,69 @@ +# GEMINI.md - Codebase Architecture & Structure + +Internal technical overview of the **Scraping & CDN Service** (`apps/scraper`) for automated data extraction and image persistence. + +## 🌍 Context + +- **`apps/scraper`**: Specialized backend engine (Axum). +- **`apps/solidjs`**: Frontend consumer. +- **`packages/services`**: Shared logic. + +## 🤖 AI Assistant Guidelines + +AI assistants (like Claude, Gemini, GPT) interacting with this codebase **MUST** adhere to the following protocols defined in **[AGENT.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/AGENT.md)**: + +1. **Professional Tone**: Maintain a cold, technical, and objective tone. +2. **No Hyperbole**: **PROHIBITED** from using marketing-speak or exaggerated praise (e.g., "amazing", "unparalleled", "powerful", "revolutionary"). +3. **Technical Accuracy**: Focus purely on implementation facts, data structures, and performance metrics. +4. **Minimalist Adherence**: Always prioritize the **Zero-Bloat Policy**. If a request introduces unnecessary dependencies or logic, challenge the user and suggest a leaner alternative. +5. **Documentation Consistency**: Ensure any generated documentation follows the established professional and objective style of the `docs/` folder. + +## 🦀 `apps/scraper` - Backend Service + +An asynchronous service for scraping and image proxying. All secondary web framework features (Authentication, Social, GraphQL) have been removed to reduce complexity. + +### 📊 Tech Stack + +- **Framework**: [Axum](https://github.com/tokio-rs/axum) (0.8.8) - Asynchronous Rust HTTP. +- **ORM**: [SeaORM](https://www.sea-ql.org/SeaORM/) (MySQL) - Database abstraction. +- **Caching**: `deadpool-redis` & `redis` - In-memory cache mapping. +- **Observability**: Request ID tracing and structured logging. +- **Scraping**: `scraper` (CSS Selectors) & remote Chrome via HTTP. + +### 📂 Directory Structure (`apps/scraper/src`) + +Organized as a hybrid of Vertical Slice and Clean Architecture. + +| Directory | Description | +| :--- | :--- | +| **`bin/`** | Binary entry points and CLI tools. | +| **`config/`** | Strongly-typed environment configuration. | +| **`entities/`** | **SeaORM Entities**. Database schema mapping. | +| **`routes/`** | **API Handlers**. Automatic routing system. | +| **`services/`** | Business logic (e.g., `ImageCache` service). | +| **`scraping/`** | Data extraction engines and parsers. | +| **`helpers/`** | Shared utilities and cache helpers. | +| **`middleware/`** | Axum layers (CORS, Compression). | +| **`events/`** | Internal event bus for repair state updates. | +| **`jobs/`** | Background task processing. | +| **`scheduler/`** | Periodic tasks (Daily CDN audit). | +| **`observability/`** | OpenAPI documentation, request ID tracing, and structured logging. | + +### 🔑 Logic Flows + +1. **Scraping**: + `Request` -> `Router` -> `Handler` -> `Scraper Engine` -> `Redis` -> `Response`. +2. **Image Proxy/CDN**: + `Request` -> `ImageCache` -> `Cache Lookup` -> `Picser Upload (on miss)` -> `CDN URL`. + +### 📜 Commands + +- **Standard Run**: `cargo run` +- **Optimized Build**: `cargo build --release` +- **External Audit**: `POST /api/proxy/image-cache/audit` + +## 🏗 Maintenance Constraints + +- **Minimalist Approach**: New dependencies require impact evaluation. +- **Lint Compliance**: Suppression flags (`#[allow]`) are prohibited. +- **Performance-First**: Use `spawn_blocking` for CPU-heavy work (HTML parsing). diff --git a/README.md b/README.md new file mode 100644 index 0000000..9151224 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Scraper API + +Backend service berbasis Axum untuk scraping, image proxy/cache, dan endpoint API inti. + +## Stack + +- Rust + Axum +- SeaORM (MySQL) +- Redis (deadpool-redis) +- Utoipa + Swagger UI + +## Quick Start + +```bash +cargo run +``` + +Server bind ke `0.0.0.0:${PORT}` dengan default port `4091`. + +## Required Environment Variables + +```env +DATABASE_URL=mysql://asephs:hunterz@localhost:3306/sosmed +JWT_SECRET=change-me +REDIS_URL=redis://localhost:6379 +``` + +Optional yang sering dipakai: + +```env +RUST_LOG=info +EXTERNAL_BROWSERLESS_WS= +MINIO_ENDPOINT= +MINIO_BUCKET_NAME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +``` + +## Useful Endpoints + +- `GET /docs` - Swagger UI +- `GET /api-docs/openapi.json` - OpenAPI JSON +- `GET /api/anime2/*` +- `GET /api/komik/*` +- `POST /api/proxy/image-cache` +- `POST /api/proxy/image-cache/audit` + +## Notes + +- Database akan dicek/dibuat saat startup jika `DATABASE_URL` bertipe MySQL. +- Service ini juga menginisialisasi browser pool dan scheduler saat boot. + +## License + +MIT diff --git a/check_image_cache_db.sh b/check_image_cache_db.sh new file mode 100755 index 0000000..41b2510 --- /dev/null +++ b/check_image_cache_db.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Simple script to check image_cache database content + +DATABASE_URL="mysql://asephs:hunterz@127.0.0.1:3306/sosmed" + +echo "=== Checking Image Cache Database ===" +echo "" +echo "Attempting to connect via Rust binary..." +echo "" + +cd "$(dirname "$0")" + +# Create temporary Rust script +cat > /tmp/check_img_cache.rs << 'EOF' +use sea_orm::{Database, ConnectionTrait, Statement}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let db = Database::connect("mysql://asephs:hunterz@127.0.0.1:3306/sosmed").await?; + + println!("✓ Connected to database\n"); + + // Show all tables + println!("=== ALL TABLES ==="); + let tables = db.query_all(Statement::from_string( + db.get_database_backend(), + "SHOW TABLES".to_owned() + )).await?; + for table in &tables { + println!(" - {:?}", table); + } + + // Count image_cache records + println!("\n=== IMAGE_CACHE TABLE ==="); + let count = db.query_one(Statement::from_string( + db.get_database_backend(), + "SELECT COUNT(*) as total FROM ImageCache".to_owned() + )).await?; + println!("Total records: {:?}", count); + + // Show recent 10 records + println!("\n=== RECENT RECORDS (Last 10) ==="); + let records = db.query_all(Statement::from_string( + db.get_database_backend(), + "SELECT id, originalUrl, cdnUrl, createdAt FROM ImageCache ORDER BY createdAt DESC LIMIT 10".to_owned() + )).await?; + + for (i, record) in records.iter().enumerate() { + println!("\n[{}]", i + 1); + println!(" ID: {:?}", record.try_get::("", "id")); + println!(" Original: {:?}", record.try_get::("", "originalUrl")); + println!(" CDN URL: {:?}", record.try_get::("", "cdnUrl")); + println!(" Created: {:?}", record.try_get::>("", "createdAt")); + } + + Ok(()) +} +EOF + +echo "Running database check..." +rustc --edition 2021 /tmp/check_img_cache.rs -o /tmp/check_img_cache \ + -L dependency=target/debug/deps \ + --extern sea_orm=target/debug/deps/libsea_orm.rlib \ + --extern tokio=target/debug/deps/libtokio.rlib \ + --extern chrono=target/debug/deps/libchrono.rlib \ + 2>/dev/null + +if [ $? -eq 0 ]; then + /tmp/check_img_cache +else + echo "Rust compilation failed, using cargo script instead..." + cargo script /tmp/check_img_cache.rs +fi diff --git a/docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md b/docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md new file mode 100644 index 0000000..5aee98a --- /dev/null +++ b/docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md @@ -0,0 +1,167 @@ +# Clean-Modular Architecture Refactor Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Refactor the codebase into a rigid Clean-Modular architecture to improve maintainability and strictly enforce separation of concerns. + +**Architecture:** A three-tier modular approach consisting of Presentation (API Handlers/DTOs), Core (Domain Models/Traits/Use Cases), and Infrastructure (Adapters/Repositories/Scrapers). + +**Tech Stack:** Rust, Axum, SeaORM, Redis, reqwest. + +--- + +### Task 1: Initialize Core Domain Models & Shared Errors + +**Files:** +- Create: `src/shared/errors/mod.rs` +- Create: `src/core/models/image.rs` +- Create: `src/core/models/mod.rs` +- Create: `src/shared/mod.rs` + +- [ ] **Step 1: Define shared application errors** +- [ ] **Step 2: Define pure domain models for ImageCache** +- [ ] **Step 3: Setup core and shared modules in `lib.rs`** + +```rust +// src/shared/errors/mod.rs +use axum::{response::{IntoResponse, Response}, Json, http::StatusCode}; +use serde_json::json; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AppError { + #[error("Not found: {0}")] + NotFound(String), + #[error("Internal error: {0}")] + Internal(String), + #[error("Validation error: {0}")] + Validation(String), +} + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + let (status, message) = match self { + AppError::NotFound(m) => (StatusCode::NOT_FOUND, m), + AppError::Internal(m) => (StatusCode::INTERNAL_SERVER_ERROR, m), + AppError::Validation(m) => (StatusCode::BAD_REQUEST, m), + }; + (status, Json(json!({ "error": message }))).into_response() + } +} +``` + +- [ ] **Step 4: Commit changes** +```bash +git add src/shared/errors/mod.rs src/core/models/image.rs +git commit -m "feat: init core models and shared errors" +``` + +### Task 2: Define Core Repository Traits + +**Files:** +- Create: `src/core/repositories/image_repository.rs` +- Create: `src/core/repositories/mod.rs` + +- [ ] **Step 1: Define ImageRepository trait in `core`** + +```rust +// src/core/repositories/image_repository.rs +use async_trait::async_trait; +use crate::core::models::image::ImageCache; +use crate::shared::errors::AppError; + +#[async_trait] +pub trait ImageRepository: Send + Sync { + async fn find_by_original_url(&self, url: &str) -> Result, AppError>; + async fn save(&self, image: ImageCache) -> Result<(), AppError>; + async fn delete_by_original_url(&self, url: &str) -> Result<(), AppError>; +} +``` + +- [ ] **Step 2: Commit changes** +```bash +git add src/core/repositories/ +git commit -m "feat: define core repository traits" +``` + +### Task 3: Migrate SeaORM Entities to Infrastructure + +**Files:** +- Modify: `src/infra/mod.rs` +- Create: `src/infra/repositories/mysql_image_repository.rs` + +- [ ] **Step 1: Implement ImageRepository for MySQL using SeaORM** +- [ ] **Step 2: Move `src/entities/image_cache.rs` logic into the new repository implementation** +- [ ] **Step 3: Update `src/infra/mod.rs` to expose repositories** + +- [ ] **Step 4: Commit changes** +```bash +git add src/infra/repositories/ +git commit -m "feat: implement mysql image repository in infra" +``` + +### Task 4: Implement Core Use Cases (Image Caching) + +**Files:** +- Create: `src/core/use_cases/cache_image.rs` +- Create: `src/core/use_cases/mod.rs` + +- [ ] **Step 1: Implement `CacheImageUseCase`** +- [ ] **Step 2: Orchestrate logic between repository, redis, and scraper/uploader** + +- [ ] **Step 3: Commit changes** +```bash +git add src/core/use_cases/ +git commit -m "feat: implement image caching use cases" +``` + +### Task 5: Refactor Scrapers into Infrastructure + +**Files:** +- Create: `src/core/repositories/scraping_repository.rs` +- Create: `src/infra/scrapers/otakudesu.rs` + +- [ ] **Step 1: Define Scraping traits in `core`** +- [ ] **Step 2: Implement site-specific scrapers in `infra`** +- [ ] **Step 3: Migrate existing logic from `src/scraping/`** + +- [ ] **Step 4: Commit changes** +```bash +git add src/infra/scrapers/ +git commit -m "feat: migrate scrapers to infra adapters" +``` + +### Task 6: Refactor Presentation Layer (API Handlers) + +**Files:** +- Create: `src/presentation/api/anime_handler.rs` +- Create: `src/presentation/api/mod.rs` +- Create: `src/presentation/mod.rs` + +- [ ] **Step 1: Migrate handlers from `src/routes/` to `presentation/api/`** +- [ ] **Step 2: Update handlers to use Use Cases instead of direct service/helper calls** +- [ ] **Step 3: Update global router in `src/main.rs` or `src/lib.rs`** + +- [ ] **Step 4: Commit changes** +```bash +git add src/presentation/api/ +git commit -m "feat: refactor presentation layer api handlers" +``` + +### Task 7: Global Cleanup & Verification + +**Files:** +- Modify: `src/lib.rs` +- Delete: `src/helpers/` (partially merged into shared/infra) +- Delete: `src/services/` (merged into core/use_cases) +- Delete: `src/routes/` (merged into presentation) + +- [ ] **Step 1: Update `lib.rs` to reflect new module structure** +- [ ] **Step 2: Remove old redundant directories** +- [ ] **Step 3: Run full test suite** +- [ ] **Step 4: Verify metrics endpoint** + +- [ ] **Step 5: Final Commit** +```bash +git commit -m "refactor: complete clean-modular architecture overhaul" +``` diff --git a/docs/superpowers/specs/2026-05-08-clean-architecture-design.md b/docs/superpowers/specs/2026-05-08-clean-architecture-design.md new file mode 100644 index 0000000..918bac9 --- /dev/null +++ b/docs/superpowers/specs/2026-05-08-clean-architecture-design.md @@ -0,0 +1,49 @@ +# Design Spec: Clean-Modular Architecture Refactor + +**Date**: 2026-05-08 +**Topic**: Refactor `apps/rust` from Hybrid to Clean-Modular Architecture. + +## 1. Purpose +Standardize codebase structure for rigidity, maintainability, and clear separation of concerns (SOC) without violating the **Zero-Bloat Policy**. + +## 2. Target Architecture +Moving from current structure to a three-tier modular design: + +### A. Presentation Layer (`src/presentation/`) +- **API Handlers**: Pure Axum handlers. +- **DTOs**: Request/Response models for external communication. +- **Middleware**: Cross-cutting concerns (CORS, Metrics, Logging). + +### B. Core Layer (`src/core/`) - The Domain +- **Models**: Pure data structures (Plain Rust Objects). +- **Repository Traits**: Abstract interfaces for data persistence. +- **Use Cases**: Orchestration of business logic (e.g., `ScrapeAnime`, `ProcessImage`). +- **Dependencies**: None (or minimal shared utils). + +### C. Infrastructure Layer (`src/infra/`) - The Adapters +- **Repositories**: SeaORM & Redis implementations of Core traits. +- **Scrapers**: Site-specific parsing logic implementing Core scraping traits. +- **External Clients**: HTTP Client (reqwest), Browser Pool. + +### D. Shared Layer (`src/shared/`) +- **Utils**: Low-level helpers (Date, JSON, String). +- **Config**: Application configuration. +- **Errors**: Centralized error handling. + +## 3. Implementation Strategy +1. **Phase 1**: Scaffold new directory structure. +2. **Phase 2**: Migrate `models` and `entities` to `core/models` and `infra/repositories`. +3. **Phase 3**: Refactor `scraping` logic into `infra/scrapers` and define traits in `core`. +4. **Phase 4**: Move Axum handlers to `presentation/api` and update routing. +5. **Phase 5**: Cleanup `helpers` into `shared/utils`. + +## 4. Constraints +- **Zero-Bloat**: No new heavy dependencies for the sake of abstraction. +- **Performance**: Maintain latency metrics as defined in `docs/development.md`. +- **SeaORM**: Entities stay in `infra/` to keep `core/` pure. + +## 5. Success Criteria +- All tests pass. +- `/metrics` show no latency regression. +- Circular dependencies are eliminated. +- Folder structure matches this spec. diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs new file mode 100644 index 0000000..94830f4 --- /dev/null +++ b/ecosystem.config.cjs @@ -0,0 +1,26 @@ +// PM2 Ecosystem Configuration for Rust +module.exports = { + apps: [ + { + name: 'ultimate-rust', + script: 'target/release/scraper', + cwd: process.env.VPS_TARGET_DIR + ? `${process.env.VPS_TARGET_DIR}/apps/scraper` + : '/home/asephs/ultimate-asepharyana.cloud/apps/scraper', + interpreter: 'none', + exec_mode: 'fork', + autorestart: true, + watch: false, + max_memory_restart: '1G', + env_production: { + NODE_ENV: 'production', + PORT: 4091, + }, + // Logging configuration + error_file: './logs/error.log', + out_file: './logs/out.log', + log_date_format: 'YYYY-MM-DD HH:mm:ss Z', + merge_logs: true, + }, + ], +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..ebfba57 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "scraper", + "version": "0.1.0", + "private": true, + "scripts": { + "start": "./target/release/scraper" + } +} diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..3255b47 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,15 @@ +# rustfmt configuration (stable features only) + +max_width = 100 +hard_tabs = false +tab_spaces = 4 +newline_style = "Unix" +use_small_heuristics = "Default" +reorder_imports = true +reorder_modules = true +remove_nested_parens = true +edition = "2021" +merge_derives = true +use_try_shorthand = true +use_field_init_shorthand = true +force_explicit_abi = true diff --git a/scripts/__pycache__/compare-openapi.cpython-312.pyc b/scripts/__pycache__/compare-openapi.cpython-312.pyc new file mode 100644 index 0000000..2064cad Binary files /dev/null and b/scripts/__pycache__/compare-openapi.cpython-312.pyc differ diff --git a/scripts/auto-lint.sh b/scripts/auto-lint.sh new file mode 100755 index 0000000..e526714 --- /dev/null +++ b/scripts/auto-lint.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Auto-fix warnings and lint Rust code + +set -e + +echo "🔧 Auto-fixing Rust warnings and linting..." + +# Format code +echo "📝 Running rustfmt..." +cargo fmt + +# Fix clippy warnings automatically +echo "🔍 Running clippy auto-fix..." +cargo clippy --fix --allow-dirty --allow-staged --all-targets + +# Check for remaining issues +echo "✅ Running final check..." +cargo clippy --all-targets -- -D warnings + +echo "✨ Done! Code is formatted and linted." diff --git a/scripts/compare-openapi.py b/scripts/compare-openapi.py new file mode 100755 index 0000000..f6e3f3c --- /dev/null +++ b/scripts/compare-openapi.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Compare OpenAPI specifications for endpoint compatibility.""" + +import json +import sys +from pathlib import Path + + +def main(): + if len(sys.argv) != 3: + print("Usage: compare-openapi.py ") + sys.exit(1) + + ref_path = Path(sys.argv[1]) + local_path = Path(sys.argv[2]) + + # Load reference OpenAPI + try: + ref_data = json.loads(ref_path.read_text()) + except FileNotFoundError: + print(f"Error: Reference file not found: {ref_path}") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in reference file: {e}") + sys.exit(1) + + # Load local OpenAPI + try: + local_data = json.loads(local_path.read_text()) + except FileNotFoundError: + print(f"Error: Local file not found: {local_path}") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in local file: {e}") + sys.exit(1) + + ref_paths = ref_data.get("paths", {}) + local_paths = local_data.get("paths", {}) + + http_methods = {"get", "put", "post", "delete", "options", "head", "patch", "trace"} + + has_differences = False + + # Check for missing paths + missing_paths = set(ref_paths.keys()) - set(local_paths.keys()) + if missing_paths: + has_differences = True + for path in sorted(missing_paths): + print(f"Missing path: {path}") + + # Check for extra paths + extra_paths = set(local_paths.keys()) - set(ref_paths.keys()) + if extra_paths: + has_differences = True + for path in sorted(extra_paths): + print(f"Extra path: {path}") + + # Check for method mismatches in common paths + common_paths = set(ref_paths.keys()) & set(local_paths.keys()) + for path in sorted(common_paths): + ref_methods = set(method.lower() for method in ref_paths[path].keys() if method.lower() in http_methods) + local_methods = set(method.lower() for method in local_paths[path].keys() if method.lower() in http_methods) + + missing_methods = ref_methods - local_methods + if missing_methods: + has_differences = True + for method in sorted(missing_methods): + print(f"Missing method {method.upper()} for path: {path}") + + extra_methods = local_methods - ref_methods + if extra_methods: + has_differences = True + for method in sorted(extra_methods): + print(f"Extra method {method.upper()} for path: {path}") + + if has_differences: + sys.exit(1) + + print(f"OpenAPI paths/methods match: {len(ref_paths)} paths") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..d559926 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Script to run database migrations manually + +echo "Running database migrations..." +cd "$(dirname "$0")/.." + +# Build and run the migration binary +cargo run --bin migrate + +if [ $? -eq 0 ]; then + echo "✅ Migration completed successfully." +else + echo "❌ Migration failed." + exit 1 +fi diff --git a/scripts/pre-build-lint.sh b/scripts/pre-build-lint.sh new file mode 100755 index 0000000..aea3c87 --- /dev/null +++ b/scripts/pre-build-lint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Pre-build hook to enforce best practices + +set -e + +echo "🔍 Running pre-build lint checks..." + +# Format check +echo "📝 Checking code formatting..." +if ! cargo fmt -- --check; then + echo "❌ Code not formatted. Run: cargo fmt" + exit 1 +fi + +# Clippy check with denials +echo "🔧 Running clippy..." +cargo clippy --all-targets -- -D warnings + +echo "✅ All lint checks passed!" diff --git a/src/app.rs b/src/app.rs new file mode 100644 index 0000000..39fefa9 --- /dev/null +++ b/src/app.rs @@ -0,0 +1,51 @@ +use std::sync::Arc; + +use axum::Router; +use sea_orm::DatabaseConnection; +use tower_http::compression::{CompressionLayer, CompressionLevel}; +use tower_http::cors::CorsLayer; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; + +use crate::shared::observability::openapi::ApiDoc; +use crate::shared::state::AppState; + +pub async fn build_router( + app_state: Arc, + db: Arc, +) -> anyhow::Result { + init_scheduler(db).await?; + + let mut openapi = ApiDoc::openapi(); + openapi.merge(crate::shared::observability::openapi_modules::ModuleApiDoc::openapi()); + + let app = crate::modules::routes(Router::new()) + .merge(SwaggerUi::new("/docs").url("/api-docs/openapi.json", openapi)) + .with_state(app_state) + .layer(axum::middleware::from_fn( + crate::shared::observability::metrics::otel_metrics_middleware, + )) + .layer(CompressionLayer::new().quality(CompressionLevel::Fastest)) + .layer(CorsLayer::permissive()); + + Ok(app) +} + +async fn init_scheduler(db: Arc) -> anyhow::Result<()> { + let scheduler = crate::shared::scheduler::Scheduler::new() + .await + .map_err(|e| anyhow::anyhow!("Failed to create scheduler: {}", e))?; + + let cache_cleanup = crate::shared::scheduler::CleanupOldCache::new(db); + scheduler + .add(cache_cleanup) + .await + .map_err(|e| anyhow::anyhow!("Failed to add cache cleanup: {}", e))?; + + scheduler + .start() + .await + .map_err(|e| anyhow::anyhow!("Failed to start scheduler: {}", e))?; + tracing::info!("✓ Scheduler started"); + Ok(()) +} diff --git a/src/bin/capture_warning.rs b/src/bin/capture_warning.rs new file mode 100644 index 0000000..e5019c4 --- /dev/null +++ b/src/bin/capture_warning.rs @@ -0,0 +1,117 @@ +use scraper::Selector; +use scraper_service::shared::utils::parse_html; +/// Capture html5ever tree_builder warning evidence by parsing problematic HTML. +/// +/// Build with: cargo build --bin capture_warning +/// Run with: RUST_LOG=warn cargo run --bin capture_warning 2>&1 +/// +/// Evidence of the warning is captured through: +/// 1. HTML that triggers foster_parenting in html5ever::tree_builder +/// 2. Observable parsing behavior showing tree reconstruction +/// 3. The call path from src/helpers::parse_html () to html5ever +/// 4. Real endpoint context from /api/anime2/latest/{slug} route +use std::fs; +use tracing_subscriber::EnvFilter; + +fn main() { + // Initialize logging to capture WARN output from html5ever + let env_filter = EnvFilter::from_default_env() + .add_directive("warn".parse().expect("valid directive")) + .add_directive("html5ever=warn".parse().expect("valid directive")); + + tracing_subscriber::fmt() + .with_env_filter(env_filter) + .with_writer(std::io::stderr) + .init(); + + println!("=== HTML5ever Tree Builder Foster Parenting Evidence ===\n"); + println!("Real Endpoint: GET /api/anime2/latest/{{slug}}"); + println!("Handler: src/routes/api/anime2/latest/[slug].rs:124"); + println!("Helper Path: src/helpers/web/scraping.rs::parse_html() -> Html::parse_document()\n"); + + // Load HTML fixture from shared test file + let fixture_path = "src/bin/test_fixtures/foster_parenting_minimal.html"; + let test_html = fs::read_to_string(fixture_path) + .expect(&format!("Failed to read fixture from {}", fixture_path)); + + println!("Input Request:"); + println!(" GET /api/anime2/latest/some-anime"); + println!(" Body: HTML containing misplaced text in "); + println!(" Fixture: {}", fixture_path); + println!(" Test HTML: {}\n", test_html); + + println!("Parsing through src/helpers::parse_html()...\n"); + println!("--- BEGIN STDERR (logging output) ---"); + + // This parse_html() call routes through: + // src/helpers::parse_html() + // -> scraper crate Html::parse_document() + // -> html5ever::parse() [version 0.36.1] + // -> TreeBuilder::process_token() + // -> TreeBuilder::foster_parent_in_body() which emits: + // warn!("foster parenting not implemented") + let document = parse_html(&test_html); + + println!("--- END STDERR (logging output) ---\n"); + + // Analyze the result + println!("Parse Output Evidence:\n"); + + // Check table structure + let table_sel = Selector::parse("table").expect("Valid CSS selector"); + let tr_sel = Selector::parse("tr").expect("Valid CSS selector"); + let td_sel = Selector::parse("td").expect("Valid CSS selector"); + + let tables: Vec<_> = document.select(&table_sel).collect(); + println!(" ✓ Tables parsed: {}", tables.len()); + + let trs: Vec<_> = document.select(&tr_sel).collect(); + println!(" ✓ Table rows found: {}", trs.len()); + + let tds: Vec<_> = document.select(&td_sel).collect(); + println!(" ✓ Table cells found: {}", tds.len()); + + let body_sel = Selector::parse("body").expect("Valid CSS selector"); + if let Some(body) = document.select(&body_sel).next() { + let body_text: String = body.text().collect(); + let trimmed = body_text.trim(); + println!("\n Body element text content:"); + println!(" '{}'", trimmed); + + if trimmed.contains("orphaned text") { + println!("\n ✓ EVIDENCE: 'orphaned text' moved OUT of
"); + println!(" This proves foster_parenting occurred!"); + } + if trimmed.contains("more text") { + println!(" ✓ EVIDENCE: 'more text' moved OUT of
"); + println!(" This confirms the adoption agency algorithm ran!"); + } + } + + println!("\n=== Proven Call Path ==="); + println!("Route: GET /api/anime2/latest/{{slug}}"); + println!("Request Handler: src/routes/api/anime2/latest/[slug].rs"); + println!(" -> latest() handler"); + println!(" -> fetch_latest_anime()"); + println!(" -> parse_latest_page(html, page)"); + println!(" -> crate::shared::utils::parse_html(html) [line 124]\n"); + + println!("Helper Function: src/helpers/web/scraping.rs"); + println!(" pub fn parse_html(html: &str) -> Html {{"); + println!(" Html::parse_document(html) // Line 34"); + println!(" }}\n"); + + println!("Call Stack to Warning:"); + println!(" 1. crate::shared::utils::parse_html() [src/helpers/web/scraping.rs:34]"); + println!(" 2. Html::parse_document() [scraper crate wrapper]"); + println!(" 3. html5ever::parse() [Cargo.toml: version 0.36.1]"); + println!(" 4. TreeBuilder::process_token()"); + println!(" 5. TreeBuilder::process_chars_in_table()"); + println!(" 6. TreeBuilder::foster_parent_in_body() [src/tree_builder/mod.rs:1227]"); + println!(" 7. warn!(\"foster parenting not implemented\") ← EMITTED ABOVE\n"); + + println!("=== Fixture Source ==="); + println!("Shared File: {}", fixture_path); + println!("HTML Content: {}", test_html); + println!("Expected Parsing Behavior: Text nodes are fostered out of table"); +} diff --git a/src/bin/foster_parenting_assertion.rs b/src/bin/foster_parenting_assertion.rs new file mode 100644 index 0000000..dd38962 --- /dev/null +++ b/src/bin/foster_parenting_assertion.rs @@ -0,0 +1,132 @@ +/// Test: Verify parsed output for foster_parenting_minimal.html fixture +/// +/// This binary contains tests and assertions that validate the expected behavior +/// when parsing HTML that triggers the html5ever::tree_builder::foster_parent_in_body() warning. +/// +/// Uses shared fixture: src/bin/test_fixtures/foster_parenting_minimal.html +/// Uses shared parser: src/helpers::parse_html() +use scraper::Selector; +use scraper_service::shared::utils::parse_html; +use std::fs; + +fn main() { + println!("Running foster parenting regression tests...\n"); + + // Load HTML fixture from shared test file + let fixture_path = "src/bin/test_fixtures/foster_parenting_minimal.html"; + let foster_parenting_html = fs::read_to_string(fixture_path) + .expect(&format!("Failed to read fixture from {}", fixture_path)); + + test_foster_parenting_text_extraction(&foster_parenting_html); + println!("✓ test_foster_parenting_text_extraction passed"); + + test_foster_parenting_table_structure(&foster_parenting_html); + println!("✓ test_foster_parenting_table_structure passed"); + + test_expected_parsed_output_assertion(&foster_parenting_html); + println!("✓ test_expected_parsed_output_assertion passed"); + + println!("\n✓ All assertions passed (3/3)"); + println!("\nFixture source: {}", fixture_path); + println!("Parser source: src/helpers/web/scraping.rs::parse_html()"); +} + +/// Test: Text nodes in
are foster-parented to body +fn test_foster_parenting_text_extraction(html: &str) { + let document = parse_html(html); + let body_sel = Selector::parse("body").expect("Valid CSS selector"); + + let body_text: String = document + .select(&body_sel) + .next() + .map(|el| el.text().collect()) + .unwrap_or_default(); + + assert!( + body_text.contains("orphaned text"), + "Text 'orphaned text' should be present in body (fostered from table)" + ); + assert!( + body_text.contains("more text"), + "Text 'more text' should be present in body (fostered from table)" + ); + assert!( + body_text.contains("cell content"), + "Cell content should still be present" + ); +} + +fn test_foster_parenting_table_structure(html: &str) { + let document = parse_html(html); + + let table_sel = Selector::parse("table").expect("Valid CSS selector"); + let tr_sel = Selector::parse("tr").expect("Valid CSS selector"); + let td_sel = Selector::parse("td").expect("Valid CSS selector"); + + let tables: Vec<_> = document.select(&table_sel).collect(); + assert_eq!(tables.len(), 1, "Should have exactly 1 table"); + + let rows: Vec<_> = document.select(&tr_sel).collect(); + assert_eq!(rows.len(), 1, "Should have exactly 1 row"); + + let cells: Vec<_> = document.select(&td_sel).collect(); + assert_eq!(cells.len(), 1, "Should have exactly 1 cell"); + + if let Some(cell) = cells.first() { + let cell_text: String = cell.text().collect(); + assert_eq!( + cell_text.trim(), + "cell content", + "Cell content should be preserved" + ); + } +} + +fn test_expected_parsed_output_assertion(html: &str) { + let document = parse_html(html); + + let body_sel = Selector::parse("body").expect("Valid CSS selector"); + let body = document + .select(&body_sel) + .next() + .expect("body should exist"); + let full_text: String = body.text().collect(); + + let expected_pattern = "orphaned textmore textcell content"; + assert!( + full_text.contains(&expected_pattern) + || (full_text.contains("orphaned text") + && full_text.contains("more text") + && full_text.contains("cell content")), + "Parsed output should contain all text content in fostered form. Got: '{}'", + full_text + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn load_fixture() -> String { + fs::read_to_string("src/bin/test_fixtures/foster_parenting_minimal.html") + .expect("Failed to load fixture") + } + + #[test] + fn test_foster_parenting_text_extraction_test() { + let html = load_fixture(); + test_foster_parenting_text_extraction(&html); + } + + #[test] + fn test_foster_parenting_table_structure_test() { + let html = load_fixture(); + test_foster_parenting_table_structure(&html); + } + + #[test] + fn test_expected_parsed_output_assertion_test() { + let html = load_fixture(); + test_expected_parsed_output_assertion(&html); + } +} diff --git a/src/bin/scaffold_enhanced/generators/api.rs b/src/bin/scaffold_enhanced/generators/api.rs new file mode 100644 index 0000000..1b75a01 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/api.rs @@ -0,0 +1,83 @@ +//! Complete API generator - combines model, migration, controller, service, repository + +use anyhow::Result; + +pub fn generate_full_api(name: &str, full: bool) -> Result<()> { + println!("📦 Generating model..."); + let model_name = singularize(name); + super::model::generate_model(&model_name, true, true, false)?; + + if full { + println!("🔧 Generating service..."); + super::service::generate_service(name, Some(&model_name))?; + + println!("💾 Generating repository..."); + super::repository::generate_repository(name, &model_name)?; + } + + println!("🎮 Generating CRUD controller..."); + super::controller::generate_controller(name, true, Some(&model_name))?; + + println!("\n✅ Complete API generated!"); + println!("\n📋 Generated files:"); + println!( + " - src/entities/{}.rs (SeaORM model)", + model_name.to_lowercase() + ); + println!( + " - migrations/m*_create_{}.rs (migration)", + super::model::pluralize(&model_name) + ); + + if full { + println!(" - src/services/{}_service.rs (service layer)", name); + println!(" - src/repositories/{}_repository.rs (repository)", name); + } + + println!(" - src/routes/api/{}/index.rs (list)", name); + println!(" - src/routes/api/{}/[id].rs (get)", name); + println!(" - src/routes/api/{}/create.rs (create)", name); + println!(" - src/routes/api/{}/[id]/update.rs (update)", name); + println!(" - src/routes/api/{}/[id]/delete.rs (delete)", name); + + println!("\n🚀 Next steps:"); + println!(" 1. Run 'cargo build' to compile"); + println!(" 2. Run migrations: cargo run -- migration up"); + println!(" 3. Start server: cargo run"); + + println!("\n📡 Available endpoints:"); + println!(" GET /api/{} - List all", name); + println!(" GET /api/{}/{{id}} - Get one", name); + println!(" POST /api/{} - Create", name); + println!(" PUT /api/{}/{{id}} - Update", name); + println!(" DELETE /api/{}/{{id}} - Delete", name); + + Ok(()) +} + +fn singularize(word: &str) -> String { + let lower = word.to_lowercase(); + + if lower.ends_with("ies") { + format!("{}y", &lower[..lower.len() - 3]) + } else if lower.ends_with("es") { + lower[..lower.len() - 2].to_string() + } else if lower.ends_with('s') { + lower[..lower.len() - 1].to_string() + } else { + lower + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_singularize() { + assert_eq!(singularize("users"), "user"); + assert_eq!(singularize("categories"), "category"); + assert_eq!(singularize("posts"), "post"); + assert_eq!(singularize("boxes"), "box"); + } +} diff --git a/src/bin/scaffold_enhanced/generators/controller.rs b/src/bin/scaffold_enhanced/generators/controller.rs new file mode 100644 index 0000000..bfa7e7a --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/controller.rs @@ -0,0 +1,287 @@ +//! API controller generator with CRUD operations + +use anyhow::Result; +use std::fs; +use std::path::Path; + +pub fn generate_controller(name: &str, crud: bool, model: Option<&str>) -> Result<()> { + let api_dir = Path::new("src/routes/api").join(name); + fs::create_dir_all(&api_dir)?; + + let model_name = model.map(|s| s.to_string()).unwrap_or_else(|| { + // Singularize the resource name + let singular = name.trim_end_matches('s'); + format!("{}{}", &singular[..1].to_uppercase(), &singular[1..]) + }); + + if crud { + generate_crud_routes(&api_dir, name, &model_name)?; + } else { + generate_basic_controller(&api_dir, name); + } + + Ok(()) +} + +fn generate_crud_routes(api_dir: &Path, resource: &str, model: &str) -> Result<()> { + // List all + let index_content = generate_list_handler(resource, model); + fs::write(api_dir.join("index.rs"), index_content)?; + + // Get by ID + let show_content = generate_show_handler(resource, model); + fs::write(api_dir.join("[id].rs"), show_content)?; + + // Create + let create_content = generate_create_handler(resource, model); + fs::write(api_dir.join("create.rs"), create_content)?; + + // Update & Delete in [id] subdirectory + fs::create_dir_all(api_dir.join("[id]"))?; + + let update_content = generate_update_handler(resource, model); + fs::write(api_dir.join("[id]/update.rs"), update_content)?; + + let delete_content = generate_delete_handler(resource, model); + fs::write(api_dir.join("[id]/delete.rs"), delete_content)?; + + Ok(()) +} + +fn generate_list_handler(resource: &str, model: &str) -> String { + format!( + r#"//! List all {resource} + +use axum::{{Extension, Json, response::IntoResponse, Router}}; +use sea_orm::{{DatabaseConnection, EntityTrait}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{Entity as {model}, Model}}; + +pub async fn list( + Extension(db): Extension, +) -> impl IntoResponse {{ + match {model}.find().all(&db).await {{ + Ok(items) => Json(items).into_response(), + Err(e) => {{ + eprintln!("Error listing {resource}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to list {resource}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + resource = resource, + model_low = model.to_lowercase(), + model = model + ) +} + +fn generate_show_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Get {singular} by ID + +use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}}; +use sea_orm::{{DatabaseConnection, EntityTrait}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{Entity as {model}, Model}}; + +pub async fn show( + Path(id): Path, + Extension(db): Extension, +) -> impl IntoResponse {{ + match {model}.find_by_id(id).one(&db).await {{ + Ok(Some(item)) => Json(item).into_response(), + Ok(None) => (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(), + Err(e) => {{ + eprintln!("Error getting {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to get {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_create_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Create new {singular} + +use axum::{{Extension, Json, response::IntoResponse, Router}}; +use sea_orm::{{ActiveModelTrait, DatabaseConnection, Set}}; +use serde::{{Deserialize, Serialize}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{ActiveModel, Model}}; + +#[derive(Serialize, Deserialize)] +pub struct Create{model}Dto {{ + pub name: String, + // Add your fields +}} + +pub async fn create( + Extension(db): Extension, + Json(data): Json, +) -> impl IntoResponse {{ + let new_item = ActiveModel {{ + name: Set(data.name), + ..Default::default() + }}; + + match new_item.insert(&db).await {{ + Ok(item) => (axum::http::StatusCode::CREATED, Json(item)).into_response(), + Err(e) => {{ + eprintln!("Error creating {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to create {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_update_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Update {singular} + +use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}}; +use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, Set}}; +use serde::{{Deserialize, Serialize}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{ActiveModel, Entity as {model}, Model}}; + +#[derive(Serialize, Deserialize)] +pub struct Update{model}Dto {{ + pub name: Option, + // Add your fields +}} + +pub async fn update( + Path(id): Path, + Extension(db): Extension, + Json(data): Json, +) -> impl IntoResponse {{ + let item = match {model}.find_by_id(id).one(&db).await {{ + Ok(Some(item)) => item, + Ok(None) => return (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(), + Err(e) => {{ + eprintln!("Error finding {singular}: {{}}", e); + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to find {singular}").into_response(); + }} + }}; + + let mut active_model: ActiveModel = item.into(); + if let Some(name) = data.name {{ + active_model.name = Set(name); + }} + + match active_model.update(&db).await {{ + Ok(updated) => Json(updated).into_response(), + Err(e) => {{ + eprintln!("Error updating {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to update {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_delete_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Delete {singular} + +use axum::{{Extension, extract::Path, response::IntoResponse, Router}}; +use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, IntoActiveModel}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{Entity as {model}}}; + +pub async fn destroy( + Path(id): Path, + Extension(db): Extension, +) -> impl IntoResponse {{ + let item = match {model}.find_by_id(id).one(&db).await {{ + Ok(Some(item)) => item, + Ok(None) => return (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(), + Err(e) => {{ + eprintln!("Error finding {singular}: {{}}", e); + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to find {singular}").into_response(); + }} + }}; + + match item.into_active_model().delete(&db).await {{ + Ok(_) => axum::http::StatusCode::NO_CONTENT.into_response(), + Err(e) => {{ + eprintln!("Error deleting {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to delete {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_basic_controller(api_dir: &Path, resource: &str) { + let content = format!( + r#"//! {resource} controller + +use axum::Router; +use std::sync::Arc; +use crate::shared::state::AppState; + +pub async fn index() -> &'static str {{ + "{resource} endpoint" +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + resource = resource + ); + + let _ = fs::write(api_dir.join("index.rs"), content); +} diff --git a/src/bin/scaffold_enhanced/generators/migration.rs b/src/bin/scaffold_enhanced/generators/migration.rs new file mode 100644 index 0000000..7a3f76e --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/migration.rs @@ -0,0 +1,262 @@ +//! Database migration generator + +use anyhow::{Context, Result}; +use chrono::Local; +use std::fs; +use std::path::Path; + +pub fn generate_migration(name: &str, table: Option<&str>) -> Result<()> { + let timestamp = Local::now().format("%Y%m%d%H%M%S"); + let file_name = format!("m{}_{}.rs", timestamp, name); + + let migrations_dir = Path::new("migrations"); + fs::create_dir_all(migrations_dir)?; + + let content = if let Some(table_name) = table { + generate_create_table_migration(table_name) + } else { + generate_empty_migration() + }; + + let migration_path = migrations_dir.join(&file_name); + fs::write(&migration_path, content) + .with_context(|| format!("Failed to write migration: {:?}", migration_path))?; + + update_migrations_mod(&file_name)?; + + Ok(()) +} + +pub fn generate_model_migration(table: &str, timestamps: bool, soft_delete: bool) -> Result<()> { + let timestamp = Local::now().format("%Y%m%d%H%M%S"); + let name = format!("create_{}_table", table); + let file_name = format!("m{}_{}.rs", timestamp, name); + + let migrations_dir = Path::new("migrations"); + fs::create_dir_all(migrations_dir)?; + + let content = generate_model_table_migration(table, timestamps, soft_delete); + + let migration_path = migrations_dir.join(&file_name); + fs::write(&migration_path, content)?; + + update_migrations_mod(&file_name)?; + + Ok(()) +} + +fn generate_create_table_migration(table: &str) -> String { + let struct_name = table + .split('_') + .map(|s| { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().chain(c).collect(), + } + }) + .collect::(); + + format!( + r#"use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration {{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .create_table( + Table::create() + .table({table}::Table) + .if_not_exists() + .col( + ColumnDef::new({table}::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new({table}::Name).string().not_null()) + .to_owned(), + ) + .await + }} + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .drop_table(Table::drop().table({table}::Table).to_owned()) + .await + }} +}} + +#[derive(DeriveIden)] +enum {table} {{ + Table, + Id, + Name, +}} +"#, + table = struct_name + ) +} + +fn generate_model_table_migration(table: &str, timestamps: bool, soft_delete: bool) -> String { + let table_pascal = table + .split('_') + .map(|s| { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().chain(c).collect(), + } + }) + .collect::(); + + let timestamp_cols = if timestamps { + format!( + r#" + .col(ColumnDef::new({}::CreatedAt).timestamp().null()) + .col(ColumnDef::new({}::UpdatedAt).timestamp().null())"#, + table_pascal, table_pascal + ) + } else { + String::new() + }; + + let soft_delete_col = if soft_delete { + format!( + r#" + .col(ColumnDef::new({}::DeletedAt).timestamp().null())"#, + table_pascal + ) + } else { + String::new() + }; + + let enum_fields = if timestamps && soft_delete { + format!(" CreatedAt,\n UpdatedAt,\n DeletedAt,") + } else if timestamps { + format!(" CreatedAt,\n UpdatedAt,") + } else if soft_delete { + format!(" DeletedAt,") + } else { + String::new() + }; + + format!( + r#"use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration {{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .create_table( + Table::create() + .table({table}::Table) + .if_not_exists() + .col( + ColumnDef::new({table}::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new({table}::Name).string().not_null()){timestamps}{soft_delete} + .to_owned(), + ) + .await + }} + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .drop_table(Table::drop().table({table}::Table).to_owned()) + .await + }} +}} + +#[derive(DeriveIden)] +enum {table} {{ + Table, + Id, + Name, +{enum_fields} +}} +"#, + table = table_pascal, + timestamps = timestamp_cols, + soft_delete = soft_delete_col, + enum_fields = enum_fields + ) +} + +fn generate_empty_migration() -> String { + format!( + r#"use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration {{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + // Add your migration logic here + Ok(()) + }} + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + // Add your rollback logic here + Ok(()) + }} +}} +"# + ) +} + +fn update_migrations_mod(file_name: &str) -> Result<()> { + let mod_path = Path::new("migrations/mod.rs"); + let module_name = file_name.trim_end_matches(".rs"); + let module_line = format!("mod {};", module_name); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + // Find the vec![] and add migration + let new_content = if content.contains("vec![") { + content.replace( + "vec![", + &format!("vec![\n Box::new({}::Migration),", module_name), + ) + } else { + format!("{}\n{}", content.trim(), module_line) + }; + fs::write(mod_path, new_content)?; + } + } else { + let initial_content = format!( + r#"pub use sea_orm_migration::prelude::*; + +{} + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator {{ + fn migrations() -> Vec> {{ + vec![ + Box::new({}::Migration), + ] + }} +}} +"#, + module_line, module_name + ); + fs::write(mod_path, initial_content)?; + } + + Ok(()) +} diff --git a/src/bin/scaffold_enhanced/generators/mod.rs b/src/bin/scaffold_enhanced/generators/mod.rs new file mode 100644 index 0000000..9faff31 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/mod.rs @@ -0,0 +1,7 @@ +// Module declarations for generators +pub mod api; +pub mod controller; +pub mod migration; +pub mod model; +pub mod repository; +pub mod service; diff --git a/src/bin/scaffold_enhanced/generators/model.rs b/src/bin/scaffold_enhanced/generators/model.rs new file mode 100644 index 0000000..55b2c36 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/model.rs @@ -0,0 +1,125 @@ +//! SeaORM model generator + +use anyhow::{Context, Result}; +use std::fs; +use std::path::Path; + +pub fn generate_model( + name: &str, + with_migration: bool, + timestamps: bool, + soft_delete: bool, +) -> Result<()> { + let table_name = pluralize(name); + + // Create entities directory + let entities_dir = Path::new("src/entities"); + fs::create_dir_all(entities_dir)?; + + // Generate model file + let model_content = generate_model_content(name, &table_name, timestamps, soft_delete); + let model_path = entities_dir.join(format!("{}.rs", name.to_lowercase())); + fs::write(&model_path, model_content) + .with_context(|| format!("Failed to write model file: {:?}", model_path))?; + + // Update entities/mod.rs + update_entities_mod(name)?; + + // Generate migration if requested + if with_migration { + super::migration::generate_model_migration(&table_name, timestamps, soft_delete)?; + } + + Ok(()) +} + +fn generate_model_content( + name: &str, + table_name: &str, + timestamps: bool, + soft_delete: bool, +) -> String { + let timestamp_fields = if timestamps { + r#" + #[sea_orm(nullable)] + pub created_at: Option, + #[sea_orm(nullable)] + pub updated_at: Option,"# + } else { + "" + }; + + let soft_delete_field = if soft_delete { + r#" + #[sea_orm(nullable)] + pub deleted_at: Option,"# + } else { + "" + }; + + format!( + r#"//! {} entity + +use sea_orm::entity::prelude::*; +use serde::{{Deserialize, Serialize}}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "{}")] +pub struct Model {{ + #[sea_orm(primary_key)] + pub id: i32, + + // Add your fields here + pub name: String,{}{} +}} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {{}} + +impl ActiveModelBehavior for ActiveModel {{}} +"#, + name, table_name, timestamp_fields, soft_delete_field + ) +} + +fn update_entities_mod(name: &str) -> Result<()> { + let mod_path = Path::new("src/entities/mod.rs"); + let module_line = format!("pub mod {};", name.to_lowercase()); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + let new_content = format!("{}\n{}", content.trim(), module_line); + fs::write(mod_path, new_content)?; + } + } else { + fs::write(mod_path, format!("{}\n", module_line))?; + } + + Ok(()) +} + +pub fn pluralize(word: &str) -> String { + let lower = word.to_lowercase(); + + if lower.ends_with('y') { + format!("{}ies", &lower[..lower.len() - 1]) + } else if lower.ends_with('s') || lower.ends_with("ch") || lower.ends_with("sh") || lower.ends_with('x') { + format!("{}es", lower) + } else { + format!("{}s", lower) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pluralize() { + assert_eq!(pluralize("User"), "users"); + assert_eq!(pluralize("Category"), "categories"); + assert_eq!(pluralize("Post"), "posts"); + assert_eq!(pluralize("Box"), "boxes"); + } +} diff --git a/src/bin/scaffold_enhanced/generators/repository.rs b/src/bin/scaffold_enhanced/generators/repository.rs new file mode 100644 index 0000000..e4094bf --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/repository.rs @@ -0,0 +1,112 @@ +//! Repository pattern generator + +use anyhow::{Context, Result}; +use std::fs; +use std::path::Path; + +pub fn generate_repository(name: &str, model: &str) -> Result<()> { + let repos_dir = Path::new("src/repositories"); + fs::create_dir_all(repos_dir)?; + + let repo_content = generate_repository_content(name, model); + + let repo_path = repos_dir.join(format!("{}_repository.rs", name.to_lowercase())); + fs::write(&repo_path, repo_content) + .with_context(|| format!("Failed to write repository: {:?}", repo_path))?; + + update_repositories_mod(name)?; + + Ok(()) +} + +fn generate_repository_content(name: &str, model: &str) -> String { + format!( + r#"//! {} repository + +use sea_orm::*; +use crate::entities::{}::{{Entity as {}, Model, ActiveModel, Column}}; + +#[derive(Clone)] +pub struct {}Repository {{ + db: DatabaseConnection, +}} + +impl {}Repository {{ + pub fn new(db: DatabaseConnection) -> Self {{ + Self {{ db }} + }} + + /// Find all records + pub async fn find_all(&self) -> Result, DbErr> {{ + {}.find().all(&self.db).await + }} + + /// Find by ID + pub async fn find_by_id(&self, id: i32) -> Result, DbErr> {{ + {}.find_by_id(id).one(&self.db).await + }} + + /// Find with pagination + pub async fn paginate(&self, page: u64, per_page: u64) -> Result<(Vec, u64), DbErr> {{ + let paginator = {}.find() + .paginate(&self.db, per_page); + + let total = paginator.num_items().await?; + let items = paginator.fetch_page(page).await?; + + Ok((items, total)) + }} + + /// Create new record + pub async fn create(&self, data: ActiveModel) -> Result {{ + data.insert(&self.db).await + }} + + /// Update existing record + pub async fn update(&self, data: ActiveModel) -> Result {{ + data.update(&self.db).await + }} + + /// Delete by ID + pub async fn delete(&self, id: i32) -> Result {{ + {}.delete_by_id(id).exec(&self.db).await + }} + + /// Find by custom condition + pub async fn find_by_name(&self, name: &str) -> Result, DbErr> {{ + {}.find() + .filter(Column::Name.contains(name)) + .all(&self.db) + .await + }} +}} +"#, + name, + model.to_lowercase(), + model, + model, + model, + model, + model, + model, + model, + model + ) +} + +fn update_repositories_mod(name: &str) -> Result<()> { + let mod_path = Path::new("src/repositories/mod.rs"); + let module_line = format!("pub mod {}_repository;", name.to_lowercase()); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + let new_content = format!("{}\n{}", content.trim(), module_line); + fs::write(mod_path, new_content)?; + } + } else { + fs::write(mod_path, format!("{}\n", module_line))?; + } + + Ok(()) +} diff --git a/src/bin/scaffold_enhanced/generators/service.rs b/src/bin/scaffold_enhanced/generators/service.rs new file mode 100644 index 0000000..adc4587 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/service.rs @@ -0,0 +1,86 @@ +//! Service layer generator + +use anyhow::{Context, Result}; +use std::fs; +use std::path::Path; + +pub fn generate_service(name: &str, model: Option<&str>) -> Result<()> { + let services_dir = Path::new("src/services"); + fs::create_dir_all(services_dir)?; + + let model_name = model.unwrap_or(name); + let service_content = generate_service_content(name, model_name); + + let service_path = services_dir.join(format!("{}_service.rs", name.to_lowercase())); + fs::write(&service_path, service_content) + .with_context(|| format!("Failed to write service: {:?}", service_path))?; + + update_services_mod(name)?; + + Ok(()) +} + +fn generate_service_content(name: &str, model: &str) -> String { + format!( + r#"//! {} service layer + +use sea_orm::*; +use crate::entities::{}::{{Entity as {}, Model, ActiveModel}}; + +pub struct {}Service {{ + db: DatabaseConnection, +}} + +impl {}Service {{ + pub fn new(db: DatabaseConnection) -> Self {{ + Self {{ db }} + }} + + pub async fn find_all(&self) -> Result, DbErr> {{ + {}.find().all(&self.db).await + }} + + pub async fn find_by_id(&self, id: i32) -> Result, DbErr> {{ + {}.find_by_id(id).one(&self.db).await + }} + + pub async fn create(&self, data: ActiveModel) -> Result {{ + data.insert(&self.db).await + }} + + pub async fn update(&self, id: i32, data: ActiveModel) -> Result {{ + data.update(&self.db).await + }} + + pub async fn delete(&self, id: i32) -> Result {{ + {}.delete_by_id(id).exec(&self.db).await + }} +}} +"#, + name, + model.to_lowercase(), + model, + model, + model, + model, + model, + model + ) +} + +fn update_services_mod(name: &str) -> Result<()> { + let mod_path = Path::new("src/services/mod.rs"); + let module_line = format!("pub mod {}_service;", name.to_lowercase()); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + let new_content = format!("{}\n{}", content.trim(), module_line); + fs::write(mod_path, new_content)?; + } + } else { + fs::write(mod_path, format!("{}\n", module_line))?; + } + + Ok(()) +} diff --git a/src/bin/test_fixtures/foster_parenting_minimal.html b/src/bin/test_fixtures/foster_parenting_minimal.html new file mode 100644 index 0000000..7b9b5ea --- /dev/null +++ b/src/bin/test_fixtures/foster_parenting_minimal.html @@ -0,0 +1 @@ +
orphaned textmore text
cell content
diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs new file mode 100644 index 0000000..d5d931b --- /dev/null +++ b/src/bootstrap/mod.rs @@ -0,0 +1,118 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::net::TcpListener; + +use axum::Router; +use sea_orm::Database; +use tracing_subscriber::EnvFilter; + +use crate::shared::config::CONFIG; +use crate::shared::database::get_redis_conn; +use crate::shared::state::AppState; + +pub struct Application { + pub port: u16, + router: Router, + listener: TcpListener, +} + +impl Application { + pub async fn build() -> anyhow::Result { + // Initialize tracing. Default to warn/error globally unless RUST_LOG is explicitly set. + let env_filter = match std::env::var("RUST_LOG") { + Ok(filter) => EnvFilter::new(filter).add_directive("html5ever=error".parse()?), + Err(_) => EnvFilter::new("warn,html5ever=error"), + }; + + tracing_subscriber::fmt().with_env_filter(env_filter).init(); + + // Initialize OpenTelemetry metrics + crate::shared::observability::metrics::init_otel_metrics(); + + tracing::info!("🚀 Scraper starting up..."); + tracing::info!(" Environment: {}", CONFIG.environment); + + // Log thread configuration + let worker_threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + tracing::info!( + " Tokio Worker Threads: (Defaulting to CPU cores: {})", + worker_threads + ); + + // Redis + let _ = get_redis_conn().await; + + // Browser Pool + tracing::info!("Initializing browser pool..."); + let browser_config = crate::shared::browser::BrowserPoolConfig::default(); + match crate::shared::browser::pool::init_browser_pool(browser_config).await { + Ok(_) => tracing::info!("✓ Browser pool initialized"), + Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e), + } + + // Database + let mut opt = sea_orm::ConnectOptions::new(CONFIG.database_url.clone()); + opt.max_connections(20) + .min_connections(1) + .connect_timeout(std::time::Duration::from_secs( + CONFIG.db.connect_timeout_seconds, + )) + .idle_timeout(std::time::Duration::from_secs( + CONFIG.db.idle_timeout_seconds, + )) + .acquire_timeout(std::time::Duration::from_secs( + CONFIG.db.acquire_timeout_seconds, + )) + .max_lifetime(std::time::Duration::from_secs( + CONFIG.db.max_lifetime_seconds, + )) + .sqlx_logging(CONFIG.log_level == "debug"); + + let db = Database::connect(opt) + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?; + tracing::info!("✓ SeaORM database connection established"); + + // Schema & Seeding + if let Err(e) = crate::shared::database::setup::init(&db).await { + tracing::error!("Failed to init DB schema: {}", e); + } + + // App State components + let db_arc = Arc::new(db); + let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new( + CONFIG.image_processing_concurrency, + )); + let event_bus = Arc::new(crate::shared::events::bus::EventBus::new()); + + let redis_pool = crate::shared::database::redis_pool() + .map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?; + + let app_state = Arc::new(AppState { + redis_pool, + db: db_arc.clone(), + image_processing_semaphore, + event_bus: event_bus.clone(), + }); + + let app = crate::app::build_router(app_state, db_arc.clone()).await?; + + // Listener + let port = CONFIG.server_port; + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + let listener = TcpListener::bind(&addr).await?; + tracing::info!("Server listening on {}", listener.local_addr()?); + + Ok(Self { + port, + router: app, + listener, + }) + } + + pub async fn run(self) -> std::io::Result<()> { + axum::serve(self.listener, self.router.into_make_service()).await + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..278f6ea --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,10 @@ +// Library root - clean organized module structure +// All modules organized into logical folders + +// ============================================================================ +// Core Framework +// ============================================================================ +pub mod app; +pub mod bootstrap; +pub mod modules; +pub mod shared; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..e84c72a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,12 @@ +#![doc = "Logging Setup"] +use scraper_service::bootstrap::Application; + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + // Application setup and server logic is now encapsulated in `startup.rs` + // This makes the main function clean and the app easier to integration test. + let app = Application::build().await?; + app.run().await?; + + Ok(()) +} diff --git a/src/modules/anime/controller.rs b/src/modules/anime/controller.rs new file mode 100644 index 0000000..9efb932 --- /dev/null +++ b/src/modules/anime/controller.rs @@ -0,0 +1,236 @@ +use crate::modules::anime::repository::AnimeRepository; +use crate::modules::anime::service::AnimeService; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; +use axum::extract::{Path, State}; +use axum::Json; +use std::sync::Arc; +use tracing::info; + +#[utoipa::path( + get, + path = "/api/anime", + tag = "anime", + operation_id = "anime_index", + responses( + (status = 200, description = "Handles GET requests for the /api/anime endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn anime_index( + State(app_state): State>, +) -> Result, AppError> { + info!("Handling request for anime index"); + let service = AnimeService::new(AnimeRepository::new()); + service.get_anime_index(app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/genre_list", + tag = "anime", + operation_id = "anime_genre_list", + responses( + (status = 200, description = "Handles GET requests for the /api/anime/genre_list endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genres( + State(app_state): State>, +) -> Result, AppError> { + info!("Handling request for anime genres"); + let service = AnimeService::new(AnimeRepository::new()); + service.get_genres(app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/detail/{slug}", + tag = "anime", + operation_id = "anime_detail_slug", + responses( + (status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn detail_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for detail slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service.get_anime_detail(app_state, slug).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/complete_anime/{slug}", + tag = "anime", + operation_id = "anime_complete_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific complete_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn complete_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for complete_anime slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_complete_anime_page(app_state, slug) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/full/{slug}", + tag = "anime", + operation_id = "anime_full_slug", + responses( + (status = 200, description = "Retrieves full episode details for a specific episode by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn full_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for full slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service.get_anime_full(app_state, slug).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/ongoing_anime/{slug}", + tag = "anime", + operation_id = "anime_ongoing_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific ongoing_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn ongoing_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for ongoing_anime slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_ongoing_anime_page(app_state, slug) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/latest/{slug}", + tag = "anime", + operation_id = "anime_latest_slug", + responses( + (status = 200, description = "Retrieves details for a specific latest by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn latest_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for latest slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_latest_anime_page(app_state, slug) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/search/{slug}", + tag = "anime", + operation_id = "anime_search_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific search by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for search slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_search_anime_page(app_state, slug, "1".to_string()) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/search/{slug}/{page}", + tag = "anime", + operation_id = "anime_search_slug_page", + responses( + (status = 200, description = "Retrieves details for a specific search by slug and page.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, String)>, +) -> Result, AppError> { + info!("Starting request for search slug: {} page: {}", slug, page); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_search_anime_page(app_state, slug, page) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/genre/{slug}", + tag = "anime", + operation_id = "anime_genre_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for genre slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_genre_anime_page(app_state, slug, "1".to_string()) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/genre/{slug}/{page}", + tag = "anime", + operation_id = "anime_genre_slug_page", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug and page.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, String)>, +) -> Result, AppError> { + info!("Starting request for genre slug: {} page: {}", slug, page); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_genre_anime_page(app_state, slug, page) + .await + .map(Json) +} diff --git a/src/modules/anime/mod.rs b/src/modules/anime/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/anime/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/anime/parser.rs b/src/modules/anime/parser.rs new file mode 100644 index 0000000..62418cf --- /dev/null +++ b/src/modules/anime/parser.rs @@ -0,0 +1,632 @@ +use crate::modules::anime::types::*; +use crate::shared::errors::AppError; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{ + attr, attr_from, attr_from_or, extract_slug, selector, text, text_from_or, +}; + +pub fn parse_ongoing_anime(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut ongoing_anime = Vec::new(); + + let venz_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + + for element in document.select(&venz_selector) { + let title = text_from_or(&element, &title_selector, ""); + let href = attr_from(&element, &link_selector, "href").unwrap_or_default(); + let slug = extract_slug(&href); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let current_episode = text_from_or(&element, &episode_selector, "N/A"); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + + if !title.is_empty() { + ongoing_anime.push(OngoingAnimeItem { + title, + slug, + poster, + current_episode, + anime_url, + }); + } + } + Ok(ongoing_anime) +} + +pub fn parse_complete_anime(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut complete_anime = Vec::new(); + + let venz_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + + for element in document.select(&venz_selector) { + let title = text_from_or(&element, &title_selector, ""); + let href = attr_from(&element, &link_selector, "href").unwrap_or_default(); + let slug = extract_slug(&href); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode_count = text_from_or(&element, &episode_selector, "N/A"); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + + if !title.is_empty() { + complete_anime.push(CompleteAnimeItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + } + Ok(complete_anime) +} + +pub fn parse_genres(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut genres = Vec::new(); + let genre_selector = selector(".genres li a, .genre-list a").unwrap(); + + for element in document.select(&genre_selector) { + let name = text(&element); + let url = attr(&element, "href").unwrap_or_default(); + let slug = extract_slug(&url); + + if !name.is_empty() && !slug.is_empty() { + genres.push(Genre { name, slug, url }); + } + } + + Ok(genres) +} + +pub fn parse_anime_detail_document(html: &str) -> Result { + let document = parse_html(html); + + let info_selector = selector(".infozingle p").unwrap(); + let poster_selector = selector(".fotoanime img").unwrap(); + let synopsis_selector = selector(".sinopc").unwrap(); + let genre_link_selector = selector("a").unwrap(); + let episode_list_selector = selector(".episodelist ul li a").unwrap(); + let recommendation_selector = selector("#recommend-anime-series .isi-anime").unwrap(); + let recommendation_title_selector = selector(".judul-anime a").unwrap(); + let recommendation_img_selector = selector("img").unwrap(); + + let mut title = String::new(); + let mut alternative_title = String::new(); + let mut r#type: Option = None; + let mut status: Option = None; + let mut release_date = String::new(); + let mut studio = String::new(); + let producers = Vec::new(); + + for element in document.select(&info_selector) { + let text = text(&element); + if text.contains("Judul:") { + title = text.replace("Judul:", "").trim().to_string(); + } else if text.contains("Japanese:") { + alternative_title = text.replace("Japanese:", "").trim().to_string(); + } else if text.contains("Type:") { + let type_str = text.replace("Type:", "").trim().to_string(); + if !type_str.is_empty() { + r#type = Some(type_str); + } + } else if text.contains("Status:") { + let status_str = text.replace("Status:", "").trim().to_string(); + if !status_str.is_empty() { + status = Some(status_str); + } + } else if text.contains("Tanggal Rilis:") { + release_date = text.replace("Tanggal Rilis:", "").trim().to_string(); + } else if text.contains("Studio:") { + studio = text.replace("Studio:", "").trim().to_string(); + } + } + + let poster = document + .select(&poster_selector) + .next() + .and_then(|e| e.value().attr("src")) + .unwrap_or("") + .to_string(); + + let synopsis = text_from_or(&document.root_element(), &synopsis_selector, ""); + + let mut genres = Vec::new(); + if let Some(genres_element) = document + .select(&info_selector) + .find(|e| text(&e).contains("Genres:")) + { + for genre_link in genres_element.select(&genre_link_selector) { + let name = text(&genre_link); + let anime_url = attr(&genre_link, "href").unwrap_or_default(); + let genre_slug = extract_slug(&anime_url); + genres.push(DetailGenre { + name, + slug: genre_slug, + anime_url, + }); + } + } + + let mut episode_lists = Vec::new(); + for element in document.select(&episode_list_selector) { + let episode = text(&element); + let href = attr(&element, "href").unwrap_or_default(); + let slug = extract_slug(&href); + episode_lists.push(EpisodeList { episode, slug }); + } + + let mut recommendations = Vec::new(); + for element in document.select(&recommendation_selector) { + let title = text_from_or(&element, &recommendation_title_selector, ""); + let poster = attr_from_or(&element, &recommendation_img_selector, "src", ""); + let href = element + .select(&genre_link_selector) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or(""); + + let slug = extract_slug(href); + + recommendations.push(Recommendation { + title, + slug, + poster, + status: None, + r#type: None, + }); + } + + Ok(AnimeDetailData { + title, + alternative_title, + poster, + r#type, + status, + release_date, + studio, + genres, + synopsis, + episode_lists, + batch: vec![], + producers, + recommendations, + }) +} + +pub fn parse_anime_page( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode_count = text_from_or(&element, &episode_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(CompleteAnimeListItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_ongoing_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let score_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let score = text_from_or(&element, &score_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(OngoingAnimeListItem { + title, + slug, + poster, + score, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_latest_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode = text_from_or(&element, &episode_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(LatestAnimeItem { + title, + slug, + poster, + episode, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_search_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let genre_selector = selector(".genre-tag").unwrap(); + let status_selector = selector(".status").unwrap(); + let rating_selector = selector(".rating").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode = text_from_or(&element, &episode_selector, "N/A"); + + let mut genres = Vec::new(); + for genre_elem in element.select(&genre_selector) { + genres.push(text(&genre_elem)); + } + + let status = text_from_or(&element, &status_selector, ""); + let rating = text_from_or(&element, &rating_selector, ""); + + if !title.is_empty() { + anime_list.push(SearchAnimeItem { + title, + slug, + poster, + episode, + anime_url, + genres, + status, + rating, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_genre_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode = text_from_or(&element, &episode_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(GenreAnimeItem { + title, + slug, + poster, + episode, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_anime_full_document(html: &str, slug: &str) -> Result { + let document = parse_html(html); + + let episode_title_selector = selector("h1.posttl").unwrap(); + let image_selector = selector(".cukder img").unwrap(); + let stream_selector = selector("#embed_holder iframe").unwrap(); + let download_item_selector = selector(".download ul li").unwrap(); + let resolution_selector = selector("strong").unwrap(); + let link_selector = selector("a").unwrap(); + let next_episode_selector = selector(".flir a[title*='Episode Selanjutnya']").unwrap(); + let previous_episode_selector = selector(".flir a[title*='Episode Sebelumnya']").unwrap(); + + let episode = document + .select(&episode_title_selector) + .next() + .map(|e| text(&e)) + .unwrap_or_default(); + + let episode_number = episode + .split("Episode") + .nth(1) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + let image_url = document + .select(&image_selector) + .next() + .and_then(|e| attr(&e, "src")) + .unwrap_or_default(); + + let stream_url = document + .select(&stream_selector) + .next() + .and_then(|e| attr(&e, "src")) + .unwrap_or_default(); + + let mut download_urls = std::collections::HashMap::new(); + + for element in document.select(&download_item_selector) { + let resolution = element + .select(&resolution_selector) + .next() + .map(|e| text(&e)) + .unwrap_or_default(); + + let mut links = Vec::new(); + for link_element in element.select(&link_selector) { + let server = text(&link_element); + let url = attr(&link_element, "href").unwrap_or_default(); + links.push(DownloadLink { server, url }); + } + + if !resolution.is_empty() && !links.is_empty() { + download_urls.insert(resolution, links); + } + } + + let next_episode_element = document.select(&next_episode_selector).next(); + let previous_episode_element = document.select(&previous_episode_selector).next(); + + let next_episode_slug = next_episode_element + .and_then(|e| attr(&e, "href")) + .and_then(|href| { + href.split('/') + .nth(href.split('/').count().saturating_sub(2)) + .map(|s| s.to_string() + "/") + }); + + let previous_episode_slug = previous_episode_element + .and_then(|e| attr(&e, "href")) + .and_then(|href| { + href.split('/') + .nth(href.split('/').count().saturating_sub(2)) + .map(|s| s.to_string() + "/") + }); + + Ok(AnimeFullData { + episode, + episode_number, + anime: AnimeInfo { + slug: slug.to_string(), + }, + has_next_episode: next_episode_slug.is_some(), + next_episode: next_episode_slug.map(|s| EpisodeInfo { slug: s }), + has_previous_episode: previous_episode_slug.is_some(), + previous_episode: previous_episode_slug.map(|s| EpisodeInfo { slug: s }), + stream_url, + download_urls, + image_url, + }) +} diff --git a/src/modules/anime/repository.rs b/src/modules/anime/repository.rs new file mode 100644 index 0000000..423df2f --- /dev/null +++ b/src/modules/anime/repository.rs @@ -0,0 +1,199 @@ +use crate::modules::anime::parser; +use crate::modules::anime::types::*; +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::fetch_with_proxy; +use crate::shared::utils::web::scraping_urls::{get_otakudesu_url, OTAKUDESU_BASE_URL}; +use crate::shared::utils::{default_backoff, fetch_html_with_retry, transient}; +use async_trait::async_trait; +use backoff::future::retry; +use tracing::{info, warn}; + +pub struct AnimeRepository; + +impl Default for AnimeRepository { + fn default() -> Self { + Self::new() + } +} + +impl AnimeRepository { + pub fn new() -> Self { + Self + } +} + +#[async_trait] +impl ScrapingRepository for AnimeRepository { + async fn fetch_html(&self, url: &str) -> Result { + fetch_html_with_retry(url).await + } +} + +impl AnimeRepository { + pub fn base_url(&self) -> String { + get_otakudesu_url() + } + + pub fn index_urls(&self) -> (String, String) { + let base = self.base_url(); + ( + format!("{}/ongoing-anime/", base), + format!("{}/complete-anime/", base), + ) + } + + pub fn genres_url(&self) -> String { + format!("{}/genre-list/", self.base_url()) + } + + pub fn detail_url(&self, slug: &str) -> String { + format!("{}/anime/{}", OTAKUDESU_BASE_URL, slug) + } + + pub fn page_url(&self, category: &str, page: &str) -> String { + format!("{}/{}/page/{}/", OTAKUDESU_BASE_URL, category, page) + } + + pub fn search_url(&self, query: &str, page: &str) -> String { + if page == "1" { + format!("{}/search/{}/", self.base_url(), query) + } else { + format!("{}/search/{}/page/{}/", self.base_url(), query, page) + } + } + + pub fn genre_page_url(&self, genre_slug: &str, page: &str) -> String { + format!("{}/genre/{}/page/{}/", self.base_url(), genre_slug, page) + } + + pub fn full_episode_url(&self, slug: &str) -> String { + format!("{}/episode/{}", OTAKUDESU_BASE_URL, slug) + } + + pub async fn fetch_anime_index(&self) -> Result { + let (ongoing_url, complete_url) = self.index_urls(); + + let (ongoing_html, complete_html) = tokio::join!( + self.fetch_html(&ongoing_url), + self.fetch_html(&complete_url) + ); + + let ongoing_html = ongoing_html?; + let complete_html = complete_html?; + + let ongoing_anime = + tokio::task::spawn_blocking(move || parser::parse_ongoing_anime(&ongoing_html)) + .await??; + let complete_anime = + tokio::task::spawn_blocking(move || parser::parse_complete_anime(&complete_html)) + .await??; + + Ok(AnimeData { + ongoing_anime, + complete_anime, + }) + } + + pub async fn fetch_genres(&self) -> Result, AppError> { + let html = self.fetch_html(&self.genres_url()).await?; + tokio::task::spawn_blocking(move || parser::parse_genres(&html)).await? + } + + pub async fn fetch_anime_detail(&self, slug: &str) -> Result { + let url = self.detail_url(slug); + let html = self + .fetch_with_proxy_retry(&url) + .await + .map_err(|e| AppError::ScraperError(e.to_string()))?; + + tokio::task::spawn_blocking(move || parser::parse_anime_detail_document(&html)).await? + } + + pub async fn fetch_complete_anime_page( + &self, + slug: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.page_url("complete-anime", slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || parser::parse_anime_page(&html, &slug_owned)).await? + } + + pub async fn fetch_ongoing_anime_page( + &self, + slug: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.page_url("ongoing-anime", slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || { + parser::parse_ongoing_anime_document(&html, &slug_owned) + }) + .await? + } + + pub async fn fetch_latest_anime_page( + &self, + slug: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.page_url("latest-anime", slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || parser::parse_latest_anime_document(&html, &slug_owned)) + .await? + } + + pub async fn fetch_search_anime_page( + &self, + slug: &str, + page: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.search_url(slug, page); + let html = self.fetch_html(&url).await?; + let page_owned = page.to_string(); + tokio::task::spawn_blocking(move || parser::parse_search_anime_document(&html, &page_owned)) + .await? + } + + pub async fn fetch_genre_anime_page( + &self, + genre_slug: &str, + page: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.genre_page_url(genre_slug, page); + let html = self.fetch_html(&url).await?; + let page_owned = page.to_string(); + tokio::task::spawn_blocking(move || parser::parse_genre_anime_document(&html, &page_owned)) + .await? + } + + pub async fn fetch_anime_full(&self, slug: &str) -> Result { + let url = self.full_episode_url(slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || parser::parse_anime_full_document(&html, &slug_owned)) + .await? + } + + async fn fetch_with_proxy_retry(&self, url: &str) -> Result { + let backoff = default_backoff(); + let url_owned = url.to_string(); + let fetch_op = || async { + info!("Fetching URL: {}", url_owned); + match fetch_with_proxy(&url_owned).await { + Ok(response) => { + info!("Successfully fetched URL: {}", url_owned); + Ok(response.data) + } + Err(e) => { + warn!("Failed to fetch URL: {}, error: {:?}", url_owned, e); + Err(transient(e)) + } + } + }; + retry(backoff, fetch_op) + .await + .map_err(|e| AppError::ScraperError(e.to_string())) + } +} diff --git a/src/modules/anime/route.rs b/src/modules/anime/route.rs new file mode 100644 index 0000000..9bee8a6 --- /dev/null +++ b/src/modules/anime/route.rs @@ -0,0 +1,49 @@ +use crate::modules::anime::controller; +use crate::shared::state::AppState; +use axum::Router; +use std::sync::Arc; + +pub fn routes(router: Router>) -> Router> { + router + .route("/api/anime", axum::routing::get(controller::anime_index)) + .route( + "/api/anime/genre_list", + axum::routing::get(controller::genres), + ) + .route( + "/api/anime/detail/{slug}", + axum::routing::get(controller::detail_slug), + ) + .route( + "/api/anime/complete_anime/{slug}", + axum::routing::get(controller::complete_anime_slug), + ) + .route( + "/api/anime/full/{slug}", + axum::routing::get(controller::full_slug), + ) + .route( + "/api/anime/ongoing_anime/{slug}", + axum::routing::get(controller::ongoing_anime_slug), + ) + .route( + "/api/anime/latest/{slug}", + axum::routing::get(controller::latest_slug), + ) + .route( + "/api/anime/search/{slug}", + axum::routing::get(controller::search_slug_index), + ) + .route( + "/api/anime/search/{slug}/{page}", + axum::routing::get(controller::search_slug_page), + ) + .route( + "/api/anime/genre/{slug}", + axum::routing::get(controller::genre_slug_index), + ) + .route( + "/api/anime/genre/{slug}/{page}", + axum::routing::get(controller::genre_slug_page), + ) +} diff --git a/src/modules/anime/schema.rs b/src/modules/anime/schema.rs new file mode 100644 index 0000000..2321901 --- /dev/null +++ b/src/modules/anime/schema.rs @@ -0,0 +1,18 @@ +use serde::Deserialize; +use utoipa::ToSchema; + +#[derive(Deserialize, ToSchema)] +pub struct SearchQuery { + pub q: Option, +} + +#[derive(Deserialize, ToSchema)] +pub struct SlugPath { + pub slug: String, +} + +#[derive(Deserialize, ToSchema)] +pub struct SlugPagePath { + pub slug: String, + pub page: String, +} diff --git a/src/modules/anime/scraping/cache.rs b/src/modules/anime/scraping/cache.rs new file mode 100644 index 0000000..7fe182e --- /dev/null +++ b/src/modules/anime/scraping/cache.rs @@ -0,0 +1,81 @@ +use crate::shared::types::entities::anime::HasPoster; +use crate::shared::state::AppState; +use std::sync::Arc; + +/// Cache poster URLs for a collection of anime items +/// This is a fire-and-forget operation that triggers lazy background caching +pub async fn cache_posters(app_state: &Arc, items: &[T]) { + let posters: Vec = items.iter().map(|item| item.poster().to_string()).collect(); + + if posters.is_empty() { + return; + } + + let db = app_state.db.clone(); + let redis = app_state.redis_pool.clone(); + + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + db, + &redis, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; +} + +/// Cache poster URLs and update items with cached URLs +/// Returns the updated items with CDN URLs +pub async fn cache_and_update_posters( + app_state: &Arc, + mut items: Vec, +) -> Vec { + let posters: Vec = items.iter().map(|item| item.poster().to_string()).collect(); + + if posters.is_empty() { + return items; + } + + let db = app_state.db.clone(); + let redis = app_state.redis_pool.clone(); + + let cached_posters = crate::shared::services::images::cache::cache_image_urls_batch_lazy( + db, + &redis, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + // Update items with cached URLs + for (i, item) in items.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(i) { + item.set_poster(url.clone()); + } + } + + items +} + +/// Cache multiple collections of posters and update them +/// Useful when you have different types of items (e.g., ongoing and complete anime) +pub async fn cache_multiple_collections( + app_state: &Arc, + collections: Vec>, +) -> Vec { + let all_posters: Vec = collections.into_iter().flatten().collect(); + + if all_posters.is_empty() { + return Vec::new(); + } + + let db = app_state.db.clone(); + let redis = app_state.redis_pool.clone(); + + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + db, + &redis, + all_posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await +} diff --git a/src/modules/anime/service.rs b/src/modules/anime/service.rs new file mode 100644 index 0000000..4eb3007 --- /dev/null +++ b/src/modules/anime/service.rs @@ -0,0 +1,295 @@ +use std::sync::Arc; + +use crate::modules::anime::repository::AnimeRepository; +use crate::modules::anime::types::*; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; +use crate::shared::utils::Cache; + +const INDEX_CACHE_TTL: u64 = 10; +const GENRE_LIST_CACHE_TTL: u64 = 3600; +const DEFAULT_CACHE_TTL: u64 = 300; + +pub struct AnimeService { + repository: AnimeRepository, +} + +impl AnimeService { + pub fn new(repository: AnimeRepository) -> Self { + Self { repository } + } + + pub async fn get_anime_index(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async { + let mut data = self + .repository + .fetch_anime_index() + .await + .map_err(|e| e.to_string())?; + + if data.ongoing_anime.is_empty() && data.complete_anime.is_empty() { + return Err("Empty anime index — refusing to cache".to_string()); + } + + let mut posters: Vec = data + .ongoing_anime + .iter() + .map(|item| item.poster.clone()) + .collect(); + posters.extend(data.complete_anime.iter().map(|item| item.poster.clone())); + + let cached_posters = + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + let ongoing_len = data.ongoing_anime.len(); + for (i, item) in data.ongoing_anime.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(i) { + item.poster = url.clone(); + } + } + for (i, item) in data.complete_anime.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(ongoing_len + i) { + item.poster = url.clone(); + } + } + + Ok(data) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_genres(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime:genres:list", GENRE_LIST_CACHE_TTL, || async { + let genres = self + .repository + .fetch_genres() + .await + .map_err(|e| e.to_string())?; + Ok(GenresResponse { + status: "Ok".to_string(), + data: genres, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_anime_detail( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:detail:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let mut data = self + .repository + .fetch_anime_detail(&slug) + .await + .map_err(|e| e.to_string())?; + + data.poster = crate::shared::services::images::cache::get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + let rec_posters: Vec = data + .recommendations + .iter() + .map(|r| r.poster.clone()) + .collect(); + let cached_rec_posters = + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + rec_posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + for (i, rec) in data.recommendations.iter_mut().enumerate() { + if let Some(url) = cached_rec_posters.get(i) { + rec.poster = url.clone(); + } + } + + Ok(DetailResponse { + status: Some("Ok".to_string()), + data, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_complete_anime_page( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:complete:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_complete_anime_page(&slug) + .await + .map_err(|e| e.to_string())?; + let total = anime_list.len() as i64; + Ok(ListResponse { + message: "Success".to_string(), + data: anime_list, + total: Some(total), + pagination: Some(pagination), + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_ongoing_anime_page( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:ongoing:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_ongoing_anime_page(&slug) + .await + .map_err(|e| e.to_string())?; + Ok(OngoingAnimeResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_latest_anime_page( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:latest:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_latest_anime_page(&slug) + .await + .map_err(|e| e.to_string())?; + Ok(LatestAnimeResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_search_anime_page( + &self, + app_state: Arc, + slug: String, + page: String, + ) -> Result { + let cache_key = format!("anime:search:{}:{}", slug, page); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_search_anime_page(&slug, &page) + .await + .map_err(|e| e.to_string())?; + Ok(SearchResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_genre_anime_page( + &self, + app_state: Arc, + genre_slug: String, + page: String, + ) -> Result { + let cache_key = format!("anime:genre:{}:{}", genre_slug, page); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_genre_anime_page(&genre_slug, &page) + .await + .map_err(|e| e.to_string())?; + Ok(GenreListResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_anime_full( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:full:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let data = self + .repository + .fetch_anime_full(&slug) + .await + .map_err(|e| e.to_string())?; + Ok(FullResponse { + status: "Ok".to_string(), + data, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } +} diff --git a/src/modules/anime/types.rs b/src/modules/anime/types.rs new file mode 100644 index 0000000..aecb906 --- /dev/null +++ b/src/modules/anime/types.rs @@ -0,0 +1,231 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +// Index endpoint types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub current_episode: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct CompleteAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode_count: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeData { + pub ongoing_anime: Vec, + pub complete_anime: Vec, +} + +// Genre list types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Genre { + pub name: String, + pub slug: String, + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenresResponse { + pub status: String, + pub data: Vec, +} + +// Detail endpoint types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailGenre { + pub name: String, + pub slug: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct EpisodeList { + pub episode: String, + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Recommendation { + pub title: String, + pub slug: String, + pub poster: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeDetailData { + pub title: String, + pub alternative_title: String, + pub poster: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + pub release_date: String, + pub studio: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub genres: Vec, + pub synopsis: String, + pub episode_lists: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub batch: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub producers: Vec, + pub recommendations: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailResponse { + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + pub data: AnimeDetailData, +} + +// Complete anime list types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct CompleteAnimeListItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode_count: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Pagination { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct ListResponse { + pub message: String, + pub data: Vec, + pub total: Option, + pub pagination: Option, +} + +// Full episode types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeInfo { + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct EpisodeInfo { + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DownloadLink { + pub server: String, + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeFullData { + pub episode: String, + pub episode_number: String, + pub anime: AnimeInfo, + pub has_next_episode: bool, + pub next_episode: Option, + pub has_previous_episode: bool, + pub previous_episode: Option, + pub stream_url: String, + pub download_urls: std::collections::HashMap>, + pub image_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FullResponse { + pub status: String, + pub data: AnimeFullData, +} + +// Ongoing anime list types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeListItem { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} + +// Latest anime types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct LatestAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct LatestAnimeResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} + +// Search types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode: String, + pub anime_url: String, + pub genres: Vec, + pub status: String, + pub rating: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} + +// Genre list by slug types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreListResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} diff --git a/src/modules/anime2/controller.rs b/src/modules/anime2/controller.rs new file mode 100644 index 0000000..1245c04 --- /dev/null +++ b/src/modules/anime2/controller.rs @@ -0,0 +1,276 @@ +use std::sync::Arc; + +use axum::{ + extract::{Path, Query, State}, + Json, +}; + +use crate::modules::anime2::repository::Anime2Repository; +use crate::modules::anime2::schema::FilterQuery; +use crate::modules::anime2::service::Anime2Service; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; + +#[utoipa::path( + get, + path = "/api/anime2", + tag = "anime2", + operation_id = "anime2_index", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2 endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn index( + State(app_state): State>, +) -> Result, AppError> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.index(app_state).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/genre_list", + tag = "anime2", + operation_id = "anime2_genre_list", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/genre_list endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_list( + State(app_state): State>, +) -> Result, AppError> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.genre_list(app_state).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/filter", + tag = "anime2", + operation_id = "anime2_filter", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/filter endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn filter( + State(app_state): State>, + Query(params): Query, +) -> Result, AppError> { + let page = params.page.unwrap_or(1); + let genre = params.genre.clone(); + let status = params.status.clone(); + let anime_type = params.r#type.clone(); + let order = params.order.clone().unwrap_or("update".to_string()); + + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json( + service + .filter(app_state, page, genre, status, anime_type, order) + .await?, + )) +} + +#[utoipa::path( + get, + path = "/api/anime2/detail/{slug}", + tag = "anime2", + operation_id = "anime2_detail_slug", + responses( + (status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn detail_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.detail(app_state, slug).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/genre/{slug}", + tag = "anime2", + operation_id = "anime2_genre_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.genre_slug(app_state, slug, 1).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/genre/{slug}/{page}", + tag = "anime2", + operation_id = "anime2_genre_slug_page", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/genre/{slug}/{page} endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, u32)>, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.genre_slug(app_state, slug, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/search/{slug}", + tag = "anime2", + operation_id = "anime2_search_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific search by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.search(app_state, slug, 1).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/search/{slug}/{page}", + tag = "anime2", + operation_id = "anime2_search_slug_page", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/search/{slug}/{page} endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, u32)>, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.search(app_state, slug, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/latest/{slug}", + tag = "anime2", + operation_id = "anime2_latest_slug", + responses( + (status = 200, description = "Retrieves details for a specific latest by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn latest_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let page = slug + .parse::() + .map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?; + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.latest(app_state, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/ongoing_anime/{slug}", + tag = "anime2", + operation_id = "anime2_ongoing_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific ongoing_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn ongoing_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let page = slug + .parse::() + .map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?; + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.ongoing_anime(app_state, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/complete_anime/{slug}", + tag = "anime2", + operation_id = "anime2_complete_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific complete_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn complete_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let page = slug + .parse::() + .map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?; + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.complete_anime(app_state, page).await?)) +} diff --git a/src/modules/anime2/mod.rs b/src/modules/anime2/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/anime2/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/anime2/parser.rs b/src/modules/anime2/parser.rs new file mode 100644 index 0000000..ab06c44 --- /dev/null +++ b/src/modules/anime2/parser.rs @@ -0,0 +1,804 @@ +use crate::shared::errors::AppError; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{attr, extract_slug, selector, text, text_from_or}; +use once_cell::sync::Lazy; +use regex::Regex; +use scraper::Selector; + +static ITEM_SELECTOR: Lazy = Lazy::new(|| Selector::parse("article.bs").unwrap()); +static TITLE_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".tt h2").unwrap()); +static IMG_SELECTOR: Lazy = Lazy::new(|| Selector::parse("img").unwrap()); +static SCORE_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".numscore").unwrap()); +static STATUS_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".status").unwrap()); +static TYPE_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".type").unwrap()); +static LINK_SELECTOR: Lazy = Lazy::new(|| Selector::parse("a").unwrap()); +static PAGINATION_SELECTOR: Lazy = + Lazy::new(|| Selector::parse(".pagination .page-numbers:not(.next)").unwrap()); +static NEXT_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".pagination .next").unwrap()); +static SLUG_REGEX: Lazy = Lazy::new(|| Regex::new(r"/([^/]+)/?$").unwrap()); +static GENRE_SLUG_REGEX: Lazy = Lazy::new(|| Regex::new(r"genre-(.+)$").unwrap()); +pub fn parse_ongoing_anime( + html: &str, +) -> Result, AppError> { + let items = parse_ongoing_anime_with_score(html)?; + Ok(items + .into_iter() + .map( + |item| crate::shared::types::entities::anime::OngoingAnimeItem { + title: item.title, + slug: item.slug, + poster: item.poster, + current_episode: item.score, + anime_url: item.anime_url, + }, + ) + .collect()) +} + +pub fn parse_complete_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut complete_anime = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + let episode_count = text_from_or(&element, &STATUS_SELECTOR, "N/A"); + + if !title.is_empty() { + complete_anime.push(crate::shared::types::entities::anime::CompleteAnimeItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + } + + Ok(complete_anime) +} + +pub fn parse_genres(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut genres = Vec::new(); + let genre_label_selector = selector("label[for^=\"genre-\"]").ok_or_else(|| { + AppError::ScraperError("Invalid selector: label[for^=\"genre-\"]".to_string()) + })?; + + for element in document.select(&genre_label_selector) { + let name = text(&element).trim().to_string(); + let for_attr = attr(&element, "for").unwrap_or_default(); + + let slug = GENRE_SLUG_REGEX + .captures(&for_attr) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !name.is_empty() && !slug.is_empty() { + genres.push(crate::modules::anime2::types::Genre { name, slug }); + } + } + + Ok(genres) +} + +pub fn parse_filter_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let status = element + .select(&STATUS_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("Unknown".to_string()); + + let anime_type = element + .select(&TYPE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("Unknown".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::FilterAnimeItem { + title, + slug, + poster, + score, + status, + r#type: anime_type, + anime_url, + }); + } + } + + let last_visible_page = document + .select(&PAGINATION_SELECTOR) + .next_back() + .map(|e| { + e.text() + .collect::() + .trim() + .parse::() + .unwrap_or(1) + }) + .unwrap_or(1); + + let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); + let pagination = crate::shared::types::entities::anime::Pagination { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some(current_page + 1) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some(current_page - 1) + } else { + None + }, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_genre_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let status = element + .select(&STATUS_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("Unknown".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::GenreAnimeItem { + title, + slug, + poster, + score, + status, + anime_url, + }); + } + } + + Ok(anime_list) +} + +pub fn parse_search_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::SearchAnimeItem { + title, + slug, + poster, + description: String::new(), + anime_url, + genres: Vec::new(), + rating: "N/A".to_string(), + r#type: "Unknown".to_string(), + season: "Unknown".to_string(), + }); + } + } + + Ok(anime_list) +} + +pub fn parse_latest_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::LatestAnimeItem { + title, + slug, + poster, + current_episode: "N/A".to_string(), + score, + anime_url, + }); + } + } + + Ok(anime_list) +} + +pub fn parse_ongoing_anime_with_score( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push( + crate::shared::types::entities::anime::OngoingAnimeItemWithScore { + title, + slug, + poster, + score, + anime_url, + }, + ); + } + } + + Ok(anime_list) +} + +pub fn parse_pagination( + document: &scraper::Html, + current_page: u32, +) -> Result { + let last_visible_page = document + .select(&PAGINATION_SELECTOR) + .next_back() + .map(|e| { + e.text() + .collect::() + .trim() + .parse::() + .unwrap_or(1) + }) + .unwrap_or(1); + + let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); + let pagination = crate::shared::types::entities::anime::Pagination { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some(current_page + 1) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some(current_page - 1) + } else { + None + }, + }; + + Ok(pagination) +} + +pub fn parse_pagination_with_string( + document: &scraper::Html, + current_page: u32, +) -> Result { + let last_visible_page = document + .select(&PAGINATION_SELECTOR) + .next_back() + .map(|e| { + e.text() + .collect::() + .trim() + .parse::() + .unwrap_or(1) + }) + .unwrap_or(1); + + let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); + let pagination = crate::shared::types::entities::anime::PaginationWithStringPages { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some((current_page + 1).to_string()) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some((current_page - 1).to_string()) + } else { + None + }, + }; + + Ok(pagination) +} + +pub fn parse_anime_detail( + html: &str, +) -> Result { + let document = parse_html(html); + + let title_selector = selector(".entry-title") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-title".to_string()))?; + let alt_title_selector = selector(".alter") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .alter".to_string()))?; + let poster_selector = selector(".thumb img, .thumbook img, .wp-post-image, .ts-post-image") + .ok_or_else(|| { + AppError::ScraperError( + "Invalid selector: .thumb img, .thumbook img, .wp-post-image, .ts-post-image" + .to_string(), + ) + })?; + let poster2_selector = selector(".bigcover img, .bixbox.animefull .bigcover .ime img") + .ok_or_else(|| { + AppError::ScraperError( + "Invalid selector: .bigcover img, .bixbox.animefull .bigcover .ime img".to_string(), + ) + })?; + let spe_span_selector = selector(".info-content .spe span").ok_or_else(|| { + AppError::ScraperError("Invalid selector: .info-content .spe span".to_string()) + })?; + let a_selector = + selector("a").ok_or_else(|| AppError::ScraperError("Invalid selector: a".to_string()))?; + let synopsis_selector = selector(".entry-content p") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-content p".to_string()))?; + let genre_selector = selector(".genxed a") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .genxed a".to_string()))?; + let download_container_selector = selector(".soraddl.dlone") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .soraddl.dlone".to_string()))?; + let resolution_selector = selector(".res") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .res".to_string()))?; + let link_selector = selector(".slink a") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .slink a".to_string()))?; + let h3_selector = + selector("h3").ok_or_else(|| AppError::ScraperError("Invalid selector: h3".to_string()))?; + let recommendation_selector = selector(".listupd .bs") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .listupd .bs".to_string()))?; + let rec_title_selector = selector(".ntitle") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .ntitle".to_string()))?; + let rec_img_selector = selector("img") + .ok_or_else(|| AppError::ScraperError("Invalid selector: img".to_string()))?; + let status_selector = selector(".status") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .status".to_string()))?; + let type_selector = selector(".typez") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .typez".to_string()))?; + + let title = text_from_or(&document.root_element(), &title_selector, ""); + let alternative_title = text_from_or(&document.root_element(), &alt_title_selector, ""); + + let poster = document + .select(&poster_selector) + .next() + .and_then(|e| { + attr(&e, "src") + .or_else(|| attr(&e, "data-src")) + .or_else(|| attr(&e, "data-lazy-src")) + }) + .unwrap_or_default(); + + let poster2 = document + .select(&poster2_selector) + .next() + .and_then(|e| { + attr(&e, "src") + .or_else(|| attr(&e, "data-src")) + .or_else(|| attr(&e, "data-lazy-src")) + }) + .unwrap_or_default(); + + let r#type = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Tipe:")) + .and_then(|span| span.select(&a_selector).next()) + .map(|e| text(&e)) + .unwrap_or_default(); + + let release_date = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Dirilis:")) + .map(|e| text(&e)) + .unwrap_or_default(); + + let status = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Status:")) + .map(|e| text(&e)) + .unwrap_or_default(); + + let synopsis = text_from_or(&document.root_element(), &synopsis_selector, ""); + + let studio = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Studio:")) + .and_then(|span| span.select(&a_selector).next()) + .map(|e| text(&e)) + .unwrap_or_default(); + + let mut genres = Vec::new(); + for element in document.select(&genre_selector) { + let name = text(&element); + let anime_url = attr(&element, "href").unwrap_or_default(); + let genre_slug = extract_slug(&anime_url); + genres.push(crate::modules::anime2::types::DetailGenre { + name, + slug: genre_slug, + anime_url, + }); + } + + let mut batch = Vec::new(); + let mut ova = Vec::new(); + let mut downloads = Vec::new(); + + for element in document.select(&download_container_selector) { + let title = element + .select(&h3_selector) + .next() + .map(|e| text(&e)) + .unwrap_or_else(|| "Unknown".to_string()); + + let category = title.to_lowercase(); + let is_batch = category.contains("batch"); + let is_ova = category.contains("ova"); + + let mut all_links = Vec::new(); + + let row_selector = selector("table tr") + .ok_or_else(|| AppError::ScraperError("Invalid selector: table tr".to_string()))?; + for row in element.select(&row_selector) { + let resolution = text_from_or(&row, &resolution_selector, ""); + + for link_element in row.select(&link_selector) { + let provider = text(&link_element); + let url = attr(&link_element, "href").unwrap_or_default(); + + let name = if !resolution.is_empty() { + format!("{} - {}", resolution, provider) + } else { + provider + }; + + all_links.push(crate::modules::anime2::types::Link { name, url }); + } + } + + let download_item = crate::modules::anime2::types::DownloadItem { + resolution: title, + links: all_links, + }; + + if is_batch { + batch.push(download_item); + } else if is_ova { + ova.push(download_item); + } else { + downloads.push(download_item); + } + } + + let mut recommendations = Vec::new(); + for element in document.select(&recommendation_selector) { + let title = text_from_or(&element, &rec_title_selector, ""); + + let anime_url = element + .select(&a_selector) + .next() + .and_then(|e| attr(&e, "href")) + .unwrap_or_default(); + + let rec_slug = extract_slug(&anime_url); + + let poster = element + .select(&rec_img_selector) + .next() + .and_then(|e| attr(&e, "data-src").or_else(|| attr(&e, "src"))) + .unwrap_or_default(); + + let status = text_from_or(&element, &status_selector, ""); + + let r#type = text_from_or(&element, &type_selector, ""); + + recommendations.push(crate::modules::anime2::types::Recommendation { + title, + slug: rec_slug, + poster, + status, + r#type, + }); + } + + Ok(crate::modules::anime2::types::AnimeDetailData { + title, + alternative_title, + poster, + poster2, + r#type, + release_date, + status, + synopsis, + studio, + genres, + producers: vec![], + recommendations, + batch, + ova, + downloads, + }) +} + +pub fn parse_genre_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_genre_anime(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} + +pub fn parse_search_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::PaginationWithStringPages, + ), + AppError, +> { + let document = parse_html(html); + let data = parse_search_anime(html)?; + let pagination = parse_pagination_with_string(&document, current_page)?; + Ok((data, pagination)) +} + +pub fn parse_latest_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_latest_anime(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} + +pub fn parse_ongoing_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_ongoing_anime_with_score(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} + +pub fn parse_complete_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_complete_anime(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} diff --git a/src/modules/anime2/repository.rs b/src/modules/anime2/repository.rs new file mode 100644 index 0000000..9b21931 --- /dev/null +++ b/src/modules/anime2/repository.rs @@ -0,0 +1,102 @@ +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::fetch_with_proxy_only; +use async_trait::async_trait; +use tracing::warn; + +const BASE_URL: &str = "https://alqanime.si"; +const BASE_DETAIL_URL: &str = "https://alqanime.net"; + +pub struct Anime2Repository; + +impl Default for Anime2Repository { + fn default() -> Self { + Self::new() + } +} + +impl Anime2Repository { + pub fn new() -> Self { + Self + } + + pub fn index_ongoing_url(&self) -> String { + format!("{}/anime/?status=ongoing&type=&order=update", BASE_URL) + } + + pub fn index_complete_url(&self) -> String { + format!("{}/anime/?status=completed&type=&order=update", BASE_URL) + } + + pub fn genre_list_url(&self) -> String { + format!("{}/anime/", BASE_URL) + } + + pub fn filter_url(&self, page: u32, order: &str) -> String { + if page > 1 { + format!("{}/anime/page/{}/?order={}", BASE_URL, page, order) + } else { + format!("{}/anime/?order={}", BASE_URL, order) + } + } + + pub fn detail_url(&self, slug: &str) -> String { + format!("{}/{}/", BASE_DETAIL_URL, slug) + } + + pub fn detail_image_url(&self, slug: &str) -> String { + format!("{}/anime/{}/", BASE_URL, slug) + } + + pub fn genre_page_url(&self, genre_slug: &str, page: u32) -> String { + if page > 1 { + format!( + "{}/anime/page/{}/?genre[]={}&order=update", + BASE_URL, page, genre_slug + ) + } else { + format!("{}/anime/?genre[]={}&order=update", BASE_URL, genre_slug) + } + } + + pub fn search_url(&self, query: &str, page: u32) -> String { + let encoded = urlencoding::encode(query); + if page == 1 { + format!("{}/?s={}", BASE_URL, encoded) + } else { + format!("{}/page/{}/?s={}", BASE_URL, page, encoded) + } + } + + pub fn latest_url(&self, page: u32) -> String { + format!( + "{}/anime/page/{}/?status=&type=&order=latest", + BASE_URL, page + ) + } + + pub fn ongoing_url(&self, page: u32) -> String { + format!( + "{}/anime/page/{}/?status=ongoing&type=&order=update", + BASE_URL, page + ) + } + + pub fn complete_url(&self, page: u32) -> String { + format!( + "{}/anime/page/{}/?status=completed&order=update", + BASE_URL, page + ) + } +} + +#[async_trait] +impl ScrapingRepository for Anime2Repository { + async fn fetch_html(&self, url: &str) -> Result { + let response = fetch_with_proxy_only(url).await?; + if response.data.trim().is_empty() { + warn!("Anime2 browserless fetch returned empty body for {}", url); + } + Ok(response.data) + } +} diff --git a/src/modules/anime2/route.rs b/src/modules/anime2/route.rs new file mode 100644 index 0000000..279b4f9 --- /dev/null +++ b/src/modules/anime2/route.rs @@ -0,0 +1,39 @@ +use std::sync::Arc; + +use axum::{routing::get, Router}; + +use crate::modules::anime2::controller; +use crate::shared::state::AppState; + +pub fn routes(router: Router>) -> Router> { + router + .route("/api/anime2", get(controller::index)) + .route( + "/api/anime2/complete_anime/{slug}", + get(controller::complete_anime_slug), + ) + .route("/api/anime2/detail/{slug}", get(controller::detail_slug)) + .route("/api/anime2/filter", get(controller::filter)) + .route("/api/anime2/genre_list", get(controller::genre_list)) + .route( + "/api/anime2/genre/{slug}", + get(controller::genre_slug_index), + ) + .route( + "/api/anime2/genre/{slug}/{page}", + get(controller::genre_slug_page), + ) + .route("/api/anime2/latest/{slug}", get(controller::latest_slug)) + .route( + "/api/anime2/ongoing_anime/{slug}", + get(controller::ongoing_anime_slug), + ) + .route( + "/api/anime2/search/{slug}", + get(controller::search_slug_index), + ) + .route( + "/api/anime2/search/{slug}/{page}", + get(controller::search_slug_page), + ) +} diff --git a/src/modules/anime2/schema.rs b/src/modules/anime2/schema.rs new file mode 100644 index 0000000..aeb3d8c --- /dev/null +++ b/src/modules/anime2/schema.rs @@ -0,0 +1,34 @@ +use serde::Deserialize; +use utoipa::ToSchema; + +#[derive(Debug, Clone, Deserialize, ToSchema)] +pub struct SlugPath { + pub slug: String, +} + +#[derive(Debug, Clone, Deserialize, ToSchema)] +pub struct SlugPagePath { + pub slug: String, + pub page: u32, +} + +#[derive(Deserialize, ToSchema)] +pub struct FilterQuery { + pub page: Option, + pub genre: Option, + pub status: Option, + pub r#type: Option, + pub order: Option, +} + +#[derive(Deserialize, ToSchema)] +pub struct GenreQuery { + pub page: Option, + pub status: Option, + pub order: Option, +} + +#[derive(Deserialize, ToSchema)] +pub struct SearchQuery { + pub q: Option, +} diff --git a/src/modules/anime2/scraping.rs b/src/modules/anime2/scraping.rs new file mode 100644 index 0000000..70eb326 --- /dev/null +++ b/src/modules/anime2/scraping.rs @@ -0,0 +1,359 @@ +use crate::shared::types::entities::anime::*; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{ + attr, attr_from, attr_from_or, extract_slug, selector, text, text_from_or, +}; +use scraper::{Html, Selector}; + +// ============================================================================ +// SELECTORS +// ============================================================================ + +/// Common selectors used across anime parsing +pub struct AnimeSelectors { + pub item: Selector, + pub title: Selector, + pub link: Selector, + pub img: Selector, + pub episode: Selector, + pub score: Selector, + pub status: Selector, + pub genre: Selector, + pub rating: Selector, + pub type_sel: Selector, + pub season: Selector, + pub desc: Selector, +} + +impl AnimeSelectors { + pub fn new() -> Result { + Ok(Self { + item: selector("article.bs").ok_or("Invalid selector: article.bs")?, + title: selector(".tt h2").ok_or("Invalid selector: .tt h2")?, + link: selector("a").ok_or("Invalid selector: a")?, + img: selector("img").ok_or("Invalid selector: img")?, + episode: selector(".epx").ok_or("Invalid selector: .epx")?, + score: selector(".numscore").ok_or("Invalid selector: .numscore")?, + status: selector(".status").ok_or("Invalid selector: .status")?, + genre: selector(".genres a").ok_or("Invalid selector: .genres a")?, + rating: selector(".score").ok_or("Invalid selector: .score")?, + type_sel: selector(".typez").ok_or("Invalid selector: .typez")?, + season: selector(".season").ok_or("Invalid selector: .season")?, + desc: selector(".data .typez").ok_or("Invalid selector: .data .typez")?, + }) + } +} + +impl Default for AnimeSelectors { + fn default() -> Self { + Self::new().expect("Valid CSS selectors") + } +} + +// Global lazy static instance for selectors to avoid reallocation per parse +use once_cell::sync::Lazy; +static ANIME_SELECTORS: Lazy> = + Lazy::new(|| AnimeSelectors::new().map_err(|e| format!("Failed to create selectors: {}", e))); + +// ============================================================================ +// HELPER FUNCTIONS +// ============================================================================ + +/// Extract poster URL from an element, checking both src and data-src attributes +pub fn extract_poster(element: &scraper::ElementRef, img_selector: &Selector) -> String { + element + .select(img_selector) + .next() + .and_then(|e| attr(&e, "src").or(attr(&e, "data-src"))) + .unwrap_or_default() +} + +// ============================================================================ +// ANIME PARSERS +// ============================================================================ + +/// Parse ongoing anime items from HTML +pub fn parse_ongoing_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let current_episode = text_from_or(&element, &selectors.episode, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + + items.push(OngoingAnimeItem { + title, + slug, + poster, + current_episode, + anime_url, + }); + } + + Ok(items) +} + +/// Parse ongoing anime items with score from HTML +pub fn parse_ongoing_anime_with_score( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let poster = extract_poster(&element, &selectors.img); + let score = text_from_or(&element, &selectors.score, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&anime_url); + + items.push(OngoingAnimeItemWithScore { + title, + slug, + poster, + score, + anime_url, + }); + } + + Ok(items) +} + +/// Parse complete anime items from HTML +pub fn parse_complete_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let episode_count = text_from_or(&element, &selectors.episode, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + + items.push(CompleteAnimeItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + + Ok(items) +} + +/// Parse latest anime items from HTML +pub fn parse_latest_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let poster = extract_poster(&element, &selectors.img); + let current_episode = text_from_or(&element, &selectors.episode, "N/A"); + let score = text_from_or(&element, &selectors.score, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&anime_url); + + items.push(LatestAnimeItem { + title, + slug, + poster, + current_episode, + score, + anime_url, + }); + } + + Ok(items) +} + +/// Parse search results from HTML +pub fn parse_search_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from(&element, &selectors.link, "href").unwrap_or_default(); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let description = text_from_or(&element, &selectors.desc, ""); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let genres = element.select(&selectors.genre).map(|e| text(&e)).collect(); + let rating = text_from_or(&element, &selectors.rating, ""); + let r#type = text_from_or(&element, &selectors.type_sel, ""); + let season = text_from_or(&element, &selectors.season, ""); + + items.push(SearchAnimeItem { + title, + slug, + poster, + description, + anime_url, + genres, + rating, + r#type, + season, + }); + } + + Ok(items) +} + +/// Parse genre-filtered anime items from HTML +pub fn parse_genre_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let poster = extract_poster(&element, &selectors.img); + let score = text_from_or(&element, &selectors.score, "N/A"); + let status = text_from_or(&element, &selectors.status, "Unknown"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&anime_url); + + items.push(GenreAnimeItem { + title, + slug, + poster, + score, + status, + anime_url, + }); + } + + Ok(items) +} + +// ============================================================================ +// PAGINATION PARSERS +// ============================================================================ + +/// Parse pagination from HTML document +pub fn parse_pagination(document: &Html, current_page: u32) -> Result { + let pagination_selector = + selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?; + let next_selector = selector(".pagination .next").ok_or("Invalid selector")?; + + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| text(&e).trim().parse::().ok()) + .unwrap_or(current_page); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + Ok(Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }) +} + +/// Parse pagination with string-based page numbers (for search results) +pub fn parse_pagination_with_string( + document: &Html, + current_page: u32, +) -> Result { + let pagination_selector = + selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?; + let next_selector = selector(".pagination .next").ok_or("Invalid selector")?; + + let last_visible_page = document + .select(&pagination_selector) + .last() + .and_then(|e| text(&e).trim().parse::().ok()) + .unwrap_or(current_page); + + let has_next_page = document.select(&next_selector).next().is_some(); + + let next_page = if has_next_page { + document + .select(&next_selector) + .next() + .and_then(|e| attr(&e, "href")) + .and_then(|href| href.split("/page/").nth(1).map(|s| s.to_string())) + .and_then(|s| s.split('/').next().map(|s| s.to_string())) + } else { + None + }; + + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some((current_page - 1).to_string()) + } else { + None + }; + + Ok(PaginationWithStringPages { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }) +} diff --git a/src/modules/anime2/service.rs b/src/modules/anime2/service.rs new file mode 100644 index 0000000..fc7548a --- /dev/null +++ b/src/modules/anime2/service.rs @@ -0,0 +1,453 @@ +use std::sync::Arc; + +use crate::modules::anime2::parser; +use crate::modules::anime2::repository::Anime2Repository; +use crate::modules::anime2::types::{DetailResponse, GenresResponse}; +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::services::images::cache::{ + apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original, +}; +use crate::shared::state::AppState; +use crate::shared::types::ApiResponse; +use crate::shared::utils::Cache; + +const INDEX_CACHE_TTL: u64 = 300; +const GENRE_LIST_CACHE_TTL: u64 = 3600; +const FILTER_CACHE_TTL: u64 = 300; +const DETAIL_CACHE_TTL: u64 = 300; +const GENRE_CACHE_TTL: u64 = 300; +const SEARCH_CACHE_TTL: u64 = 300; +const LATEST_CACHE_TTL: u64 = 120; +const ONGOING_CACHE_TTL: u64 = 300; +const COMPLETE_CACHE_TTL: u64 = 300; + +pub struct Anime2Service { + repository: Anime2Repository, +} + +impl Anime2Service { + pub fn new(repository: Anime2Repository) -> Self { + Self { repository } + } + + pub async fn index( + &self, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime2:index", INDEX_CACHE_TTL, || async { + let ongoing_html = self + .repository + .fetch_html(&self.repository.index_ongoing_url()) + .await + .map_err(|e| e.to_string())?; + let complete_html = self + .repository + .fetch_html(&self.repository.index_complete_url()) + .await + .map_err(|e| e.to_string())?; + + let mut data = tokio::task::spawn_blocking(move || { + Ok::<_, String>(( + parser::parse_ongoing_anime(&ongoing_html).map_err(|e| e.to_string())?, + parser::parse_complete_anime(&complete_html).map_err(|e| e.to_string())?, + )) + }) + .await + .map_err(|e| e.to_string())??; + + let mut posters: Vec = + data.0.iter().map(|item| item.poster.clone()).collect(); + posters.extend(data.1.iter().map(|item| item.poster.clone())); + + let cached_posters = cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + let ongoing_len = data.0.len(); + for (i, item) in data.0.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(i) { + item.poster = url.clone(); + } + } + for (i, item) in data.1.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(ongoing_len + i) { + item.poster = url.clone(); + } + } + + Ok(crate::modules::anime2::types::Anime2Response { + status: "Ok".to_string(), + data: crate::modules::anime2::types::Anime2Data { + ongoing_anime: data.0, + complete_anime: data.1, + }, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_list(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime2:genres:list:v3", GENRE_LIST_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.genre_list_url()) + .await + .map_err(|e| e.to_string())?; + + let genres = tokio::task::spawn_blocking(move || { + parser::parse_genres(&html).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + Ok(GenresResponse { + status: "Ok".to_string(), + data: genres, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn filter( + &self, + app_state: Arc, + page: u32, + genre: Option, + status: Option, + anime_type: Option, + order: String, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!( + "anime2:filter:{}:{:?}:{:?}:{:?}:{}", + page, genre, status, anime_type, order + ); + let genre_clone = genre.clone(); + let status_clone = status.clone(); + let anime_type_clone = anime_type.clone(); + + cache + .get_or_set(&cache_key, FILTER_CACHE_TTL, || async { + let mut url = self.repository.filter_url(page, &order); + + if let Some(g) = &genre { + for genre_item in g.split(',') { + url.push_str(&format!("&genre[]={}", genre_item.trim())); + } + } + if let Some(s) = &status { + url.push_str(&format!("&status={}", s)); + } + if let Some(t) = &anime_type { + url.push_str(&format!("&type={}", t)); + } + + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + let (data, pagination) = tokio::task::spawn_blocking(move || { + parser::parse_filter_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(crate::modules::anime2::types::FilterResponse { + success: true, + data: final_data, + pagination, + filters_applied: crate::modules::anime2::types::FiltersApplied { + genre: genre_clone, + status: status_clone, + r#type: anime_type_clone, + order: order.clone(), + }, + status: "Ok".to_string(), + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn detail( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:detail:{}", slug); + + cache + .get_or_set(&cache_key, DETAIL_CACHE_TTL, || async { + let detail_url = self.repository.detail_url(&slug); + let image_url = self.repository.detail_image_url(&slug); + let (detail_html, image_html) = tokio::join!( + self.repository.fetch_html(&detail_url), + self.repository.fetch_html(&image_url) + ); + let detail_html = detail_html.map_err(|e| e.to_string())?; + let image_html = image_html.ok(); + + let mut data = tokio::task::spawn_blocking(move || { + let mut data = + parser::parse_anime_detail(&detail_html).map_err(|e| e.to_string())?; + if let Some(image_html) = image_html { + if let Ok(image_data) = parser::parse_anime_detail(&image_html) { + if !image_data.poster.is_empty() { + data.poster = image_data.poster; + } + if !image_data.poster2.is_empty() { + data.poster2 = image_data.poster2; + } + data.recommendations = image_data.recommendations; + } + } + Ok::<_, String>(data) + }) + .await + .map_err(|e| e.to_string())??; + + data.poster = get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + data.poster2 = get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster2, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + apply_cached_posters( + &mut data.recommendations, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(DetailResponse { + status: "Ok".to_string(), + data, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_slug( + &self, + app_state: Arc, + genre_slug: String, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:genre:{}:{}", genre_slug, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.genre_page_url(&genre_slug, page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_genre_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn search( + &self, + app_state: Arc, + query: String, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:search:{}:{}", query, page); + + cache + .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.search_url(&query, page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_search_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn latest( + &self, + app_state: Arc, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:latest:{}", page); + + cache + .get_or_set(&cache_key, LATEST_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.latest_url(page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_latest_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn ongoing_anime( + &self, + app_state: Arc, + page: u32, + ) -> Result< + ApiResponse>, + AppError, + > { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:ongoing:{}", page); + + cache + .get_or_set(&cache_key, ONGOING_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.ongoing_url(page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn complete_anime( + &self, + app_state: Arc, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:complete:{}", page); + + cache + .get_or_set(&cache_key, COMPLETE_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.complete_url(page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_complete_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } +} diff --git a/src/modules/anime2/types.rs b/src/modules/anime2/types.rs new file mode 100644 index 0000000..2523d1e --- /dev/null +++ b/src/modules/anime2/types.rs @@ -0,0 +1,106 @@ +use crate::shared::types::entities::anime::HasPoster; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Anime2Data { + pub ongoing_anime: Vec, + pub complete_anime: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Anime2Response { + pub status: String, + pub data: Anime2Data, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Genre { + pub name: String, + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenresResponse { + pub status: String, + pub data: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FiltersApplied { + pub genre: Option, + pub status: Option, + pub r#type: Option, + pub order: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FilterResponse { + pub success: bool, + pub data: Vec, + pub pagination: crate::shared::types::entities::anime::Pagination, + pub filters_applied: FiltersApplied, + pub status: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeDetailData { + pub title: String, + pub alternative_title: String, + pub poster: String, + pub poster2: String, + pub r#type: String, + pub release_date: String, + pub status: String, + pub synopsis: String, + pub studio: String, + pub genres: Vec, + pub producers: Vec, + pub recommendations: Vec, + pub batch: Vec, + pub ova: Vec, + pub downloads: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailGenre { + pub name: String, + pub slug: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Link { + pub name: String, + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DownloadItem { + pub resolution: String, + pub links: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Recommendation { + pub title: String, + pub slug: String, + pub poster: String, + pub status: String, + pub r#type: String, +} + +impl HasPoster for Recommendation { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailResponse { + pub status: String, + pub data: AnimeDetailData, +} diff --git a/src/modules/komik/controller.rs b/src/modules/komik/controller.rs new file mode 100644 index 0000000..cec37e8 --- /dev/null +++ b/src/modules/komik/controller.rs @@ -0,0 +1,217 @@ +use std::sync::Arc; + +use axum::{extract::State, Json}; + +use crate::modules::komik::repository::KomikRepository; +use crate::modules::komik::service::KomikService; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; + +#[utoipa::path( + get, + path = "/api/komik/genre_list", + tag = "komik", + operation_id = "komik_genre_list", + responses( + (status = 200, description = "Handles GET requests for the /api/komik/genre_list endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_list( + State(app_state): State>, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.genre_list(app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/chapter/{slug}", + tag = "komik", + operation_id = "komik_chapter_slug", + responses( + (status = 200, description = "Retrieves details for a specific chapter by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn chapter_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.chapter_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/detail/{slug}", + tag = "komik", + operation_id = "komik_detail_slug", + responses( + (status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn detail_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.detail_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/genre/{slug}", + tag = "komik", + operation_id = "komik_genre_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.genre_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/genre/{slug}/{page}", + tag = "komik", + operation_id = "komik_genre_slug_page", + responses( + (status = 200, description = "Retrieves paginated genre results by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_page( + State(app_state): State>, + axum::extract::Path((slug, page)): axum::extract::Path<(String, String)>, +) -> Result, AppError> { + let page_num = page + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + let service = KomikService::new(KomikRepository::new()); + service + .genre_slug_page(slug, page_num, app_state) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/manga/{slug}", + tag = "komik", + operation_id = "komik_manga_slug", + responses( + (status = 200, description = "Retrieves manga details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn manga_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.manga_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/manhua/{slug}", + tag = "komik", + operation_id = "komik_manhua_slug", + responses( + (status = 200, description = "Retrieves manhua details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn manhua_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.manhua_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/manhwa/{slug}", + tag = "komik", + operation_id = "komik_manhwa_slug", + responses( + (status = 200, description = "Retrieves manhwa details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn manhwa_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.manhwa_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/popular/{slug}", + tag = "komik", + operation_id = "komik_popular_slug", + responses( + (status = 200, description = "Retrieves popular komik details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn popular_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.popular_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/search/{slug}", + tag = "komik", + operation_id = "komik_search_slug_index", + responses( + (status = 200, description = "Retrieves search results by query slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.search_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/search/{slug}/{page}", + tag = "komik", + operation_id = "komik_search_slug_page", + responses( + (status = 200, description = "Retrieves paginated search results by query slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_page( + State(app_state): State>, + axum::extract::Path((slug, page)): axum::extract::Path<(String, String)>, +) -> Result, AppError> { + let page_num = page + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + let service = KomikService::new(KomikRepository::new()); + service + .search_slug_page(slug, page_num, app_state) + .await + .map(Json) +} diff --git a/src/modules/komik/mod.rs b/src/modules/komik/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/komik/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/komik/parser.rs b/src/modules/komik/parser.rs new file mode 100644 index 0000000..0b4c45a --- /dev/null +++ b/src/modules/komik/parser.rs @@ -0,0 +1,592 @@ +use crate::modules::komik::types::{ChapterData, DetailData, Genre, KomikItem, Pagination}; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{attr, attr_from, attr_from_or, selector, text, text_from_or}; +use once_cell::sync::Lazy; +use rayon::prelude::*; +use regex::Regex; +use tracing::info; + +static TD_LAST_SELECTOR: Lazy = Lazy::new(|| selector("td:last-child").unwrap()); +static TITLE_SELECTOR: Lazy = + Lazy::new(|| selector("div#Judul h1 span[itemprop=\"name\"]").unwrap()); +static H1_SELECTOR: Lazy = Lazy::new(|| selector("h1").unwrap()); +static TITLE_TAG_SELECTOR: Lazy = Lazy::new(|| selector("title").unwrap()); +static INFO_ROW_SELECTOR: Lazy = + Lazy::new(|| selector("table.inftable tr").unwrap()); +static POSTER_SELECTOR: Lazy = + Lazy::new(|| selector("section#Informasi .ims img").unwrap()); +static DESC_SELECTOR: Lazy = Lazy::new(|| selector("p.desc").unwrap()); +static CHAPTER_LIST_SELECTOR: Lazy = + Lazy::new(|| selector("#Daftar_Chapter tr, tbody#daftarChapter tr").unwrap()); +static DATE_LINK_SELECTOR: Lazy = + Lazy::new(|| selector("td.tanggalseries, .tanggalseries").unwrap()); +static JUDUL2_SELECTOR: Lazy = Lazy::new(|| selector("div.judul2").unwrap()); +static GENRE_SELECTOR: Lazy = Lazy::new(|| selector("ul.genre li a").unwrap()); +static CHAPTER_LINK_SELECTOR: Lazy = + Lazy::new(|| selector("td.judulseries a").unwrap()); +static CHAPTER_TITLE_REGEX: Lazy = + Lazy::new(|| Regex::new(r"(?i)(?:chapter|ch\.?)\s*([\d\.]+)").unwrap()); +static CHAPTER_NUMBER_REGEX: Lazy = Lazy::new(|| Regex::new(r"([\d\.]+)").unwrap()); + +pub fn parse_genres(html: &str) -> Result, String> { + let document = parse_html(html); + let mut genres = Vec::new(); + + let genre_selector = + selector("#Genre .ls3, section#Genre .ls3, .ls3").ok_or("Selector error".to_string())?; + let genre_name_selector = selector(".ls3p h4, h4").ok_or("Selector error".to_string())?; + let genre_link_selector = selector("a[href*='/genre/']").ok_or("Selector error".to_string())?; + let slug_regex = Regex::new(r"/genre/([^/]+)").unwrap(); + + for element in document.select(&genre_selector) { + let name = text_from_or(&element, &genre_name_selector, ""); + let href = attr_from(&element, &genre_link_selector, "href").unwrap_or_default(); + + let slug = slug_regex + .captures(&href) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !name.is_empty() && !slug.is_empty() { + genres.push(Genre { + name, + slug, + count: None, + }); + } + } + + info!("Parsed {} genres", genres.len()); + Ok(genres) +} + +pub fn parse_komik_chapter_document(html: &str, chapter_url: &str) -> Result { + let document = parse_html(html); + let _start_time = std::time::Instant::now(); + info!("Starting to parse komik chapter document"); + + let title_selector = selector("title").ok_or("Selector error".to_string())?; + let prev_chapter_selector = selector( + "a[aria-label='Prev'][href*='chapter'], .nxpr a:not(.rl):not([href*='#Chapter']), .chprev a, a.prev", + ) + .ok_or("Selector error".to_string())?; + let next_chapter_selector = selector( + "a[aria-label='Next'][href*='chapter'], .nxpr a.rl, .nxpr a.next, .chnext a, a.next", + ) + .ok_or("Selector error".to_string())?; + let image_selector = + selector("#Baca_Komik img, img.klazy.ww").ok_or("Selector error".to_string())?; + + let title = document + .select(&title_selector) + .next() + .map(|e| { + let full_title = text(&e); + if let Some(start) = full_title.find("Komik ") { + if let Some(end) = full_title.find(" - Komiku") { + full_title[start + 6..end].trim().to_string() + } else { + full_title + } + } else { + full_title + } + }) + .unwrap_or_default(); + + let next_chapter_id = document + .select(&next_chapter_selector) + .next() + .and_then(|e| attr(&e, "href")) + .map(|href| { + href.trim_end_matches('/') + .split('/') + .filter(|s| !s.is_empty()) + .next_back() + .unwrap_or("") + .to_string() + }) + .unwrap_or_default(); + + fn get_previous_chapter_id(chapter_url: &str) -> String { + const CHAPTER_PATTERN: &str = "chapter-"; + + if let Some(pattern_pos) = chapter_url.rfind(CHAPTER_PATTERN) { + let prefix = &chapter_url[0..pattern_pos]; + let suffix = &chapter_url[pattern_pos + CHAPTER_PATTERN.len()..]; + + let chapter_num = suffix + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect::(); + + if let Ok(num) = chapter_num.parse::() { + let prev_num = num.saturating_sub(1); + + let formatted_num = if chapter_num.starts_with('0') { + format!("{:0width$}", prev_num, width = chapter_num.len()) + } else { + prev_num.to_string() + }; + + return format!("{}{}{}", prefix, CHAPTER_PATTERN, formatted_num); + } + } + + String::new() + } + + let prev_chapter_id_from_url = get_previous_chapter_id(chapter_url); + let prev_chapter_id = if !prev_chapter_id_from_url.is_empty() { + prev_chapter_id_from_url + } else { + document + .select(&prev_chapter_selector) + .next() + .and_then(|e| attr(&e, "href")) + .map(|href| { + href.trim_end_matches('/') + .split('/') + .filter(|s| !s.is_empty()) + .next_back() + .unwrap_or("") + .to_string() + }) + .unwrap_or_default() + }; + + fn get_list_chapter_from_url(chapter_url: &str) -> String { + if let Some(pos) = chapter_url.rfind("-chapter-") { + chapter_url[..pos].to_string() + } else { + chapter_url.to_string() + } + } + + let list_chapter = get_list_chapter_from_url(chapter_url); + + let mut images = Vec::new(); + let forbidden_images = [ + "https://flagcdn.com/32x24/jp.png", + "https://flagcdn.com/32x24/kr.png", + "https://flagcdn.com/32x24/cn.png", + "https://www.gstatic.com/firebasejs/ui/2.0.0/images/auth/google.svg", + "https://www.gravatar.com/avatar/?d=mp&s=80", + "/asset/img/komikuplus2.jpg", + "https://komiku.org/asset/img/Loading.gif", + ]; + for el in document.select(&image_selector) { + if let Some(src) = attr(&el, "src") + .or_else(|| attr(&el, "data-src")) + .or_else(|| attr(&el, "data-lazy-src")) + .or_else(|| { + attr(&el, "srcset").and_then(|s| s.split_whitespace().next().map(|s| s.to_string())) + }) + { + if !forbidden_images.contains(&src.as_str()) { + images.push(src); + } + } + } + + Ok(ChapterData { + title, + next_chapter_id, + prev_chapter_id, + list_chapter, + images, + }) +} + +fn clean_text(text: String) -> String { + text.replace(['\n', '\t'], " ").trim().to_string() +} + +fn extract_value_after_keyword(full_text: &str, keywords: &[&str], default_index: usize) -> String { + let lower_text = full_text.to_lowercase(); + for keyword in keywords { + if let Some(pos) = lower_text.find(&format!("{}:", keyword)) { + return clean_text(full_text[pos + keyword.len() + 1..].to_string()); + } else if let Some(pos) = lower_text.find(&format!("{} ", keyword)) { + return clean_text(full_text[pos + keyword.len() + 1..].to_string()); + } + } + if let Some(colon_pos) = full_text.find(':') { + return clean_text(full_text[colon_pos + 1..].to_string()); + } + clean_text(full_text[default_index..].to_string()) +} + +fn find_table_row_with_text<'a>( + info_rows: &[scraper::ElementRef<'a>], + text_fragments: &[&str], +) -> Option { + let lower_text_fragments: Vec = + text_fragments.iter().map(|&s| s.to_lowercase()).collect(); + let td_last_selector = &*TD_LAST_SELECTOR; + + info_rows + .iter() + .find(|row| { + let row_text = text(row).to_lowercase(); + lower_text_fragments + .iter() + .any(|fragment| row_text.contains(fragment)) + }) + .and_then(|row| { + text_from_or(row, td_last_selector, "") + .trim() + .to_string() + .into() + }) +} + +pub fn parse_komik_detail_document(html: &str) -> Result { + let start_time = std::time::Instant::now(); + info!("Starting to parse komik detail document"); + + let document = parse_html(html); + + let title_selector = &*TITLE_SELECTOR; + let h1_selector = &*H1_SELECTOR; + let title_tag_selector = &*TITLE_TAG_SELECTOR; + let info_row_selector = &*INFO_ROW_SELECTOR; + let poster_selector = &*POSTER_SELECTOR; + let desc_selector = &*DESC_SELECTOR; + let chapter_list_selector = &*CHAPTER_LIST_SELECTOR; + let date_link_selector = &*DATE_LINK_SELECTOR; + let judul2_selector = &*JUDUL2_SELECTOR; + let genre_selector = &*GENRE_SELECTOR; + let chapter_link_selector = &*CHAPTER_LINK_SELECTOR; + + let title = document + .select(&title_selector) + .next() + .map(|e| { + let text = clean_text(text(&e)); + text.replace("Komik ", "") + .replace("Manga ", "") + .replace("Manhua ", "") + .replace("Manhwa ", "") + .trim() + .to_string() + }) + .or_else(|| { + document + .select(&h1_selector) + .next() + .map(|e| clean_text(text(&e))) + }) + .or_else(|| { + document.select(&title_tag_selector).next().map(|e| { + let text = clean_text(text(&e)); + if text.contains("Komik ") { + text.replace("Komik ", "").trim().to_string() + } else { + text.trim().to_string() + } + }) + }) + .unwrap_or_default(); + + let info_rows_vec: Vec = document.select(&info_row_selector).collect(); + let info_rows = &info_rows_vec[..]; + + let status = info_rows + .iter() + .find_map(|&row| { + let full_text = text(&row); + if full_text.to_lowercase().contains("status") { + Some( + extract_value_after_keyword(&full_text, &["status"], 0) + .replace("Status", "") + .replace("Jenis Komik", "") + .replace("Type", "") + .trim() + .to_string(), + ) + } else { + None + } + }) + .unwrap_or_default(); + + let r#type = info_rows + .iter() + .find_map(|&row| { + let full_text = text(&row); + if full_text.to_lowercase().contains("jenis komik") + || full_text.to_lowercase().contains("type") + { + Some( + extract_value_after_keyword(&full_text, &["jenis komik", "type"], 0) + .replace("Jenis Komik", "") + .replace("Type", "") + .trim() + .to_string(), + ) + } else { + None + } + }) + .unwrap_or_default(); + + let author = info_rows + .iter() + .find_map(|&row| { + let full_text = text(&row); + if full_text.to_lowercase().contains("pengarang") + || full_text.to_lowercase().contains("author") + || full_text.to_lowercase().contains("artist") + { + Some( + extract_value_after_keyword(&full_text, &["pengarang", "author", "artist"], 0) + .replace("Pengarang", "") + .replace("Author", "") + .replace("pengarang", "") + .replace("author", "") + .replace("Artist", "") + .replace("artist", "") + .trim() + .to_string(), + ) + } else { + None + } + }) + .unwrap_or_default(); + + let poster = document + .select(&poster_selector) + .next() + .and_then(|e| attr(&e, "src")) + .map(|s| s.split('?').next().unwrap_or(&s).to_string()) + .unwrap_or_default(); + + let description = document + .select(&desc_selector) + .map(|e| clean_text(text(&e))) + .filter(|t| t.len() > 50) + .collect::>() + .join("\n") + .trim() + .to_string(); + + let release_date = find_table_row_with_text(info_rows, &["tanggal rilis", "release date"]) + .map(clean_text) + .unwrap_or_else(|| { + document + .select(&chapter_list_selector) + .next_back() + .and_then(|last| last.select(&date_link_selector).next()) + .map(|e| clean_text(text(&e))) + .unwrap_or_default() + }); + + let total_chapter = find_table_row_with_text(info_rows, &["total chapter", "total chapters"]) + .unwrap_or_else(|| { + let count = document.select(&chapter_list_selector).count(); + if count > 0 { + count.to_string() + } else { + String::new() + } + }); + + let updated_on = find_table_row_with_text(info_rows, &["diperbarui", "updated"]) + .or_else(|| { + document.select(&judul2_selector).next().map(|e| { + let text_str = clean_text(text(&e)); + text_str.split("• ").nth(1).unwrap_or("").trim().to_string() + }) + }) + .unwrap_or_else(|| { + document + .select(&chapter_list_selector) + .next() + .and_then(|first| first.select(&date_link_selector).next()) + .map(|e| clean_text(text(&e))) + .unwrap_or_default() + }); + + let mut genres = Vec::new(); + for element in document.select(&genre_selector) { + let genre = clean_text(text(&element)); + if !genre.is_empty() { + genres.push(genre); + } + } + + let raw_chapter_data: Vec<(String, String, String)> = document + .select(&chapter_list_selector) + .filter_map(|el| { + let chapter_link_element = el.select(&chapter_link_selector).next(); + let date_element = el.select(&date_link_selector).next(); + + let chapter_text = chapter_link_element + .as_ref() + .map(|e| clean_text(text(e))) + .unwrap_or_default(); + + let date_text = date_element + .map(|e| clean_text(text(&e))) + .unwrap_or_default(); + + let href_text = chapter_link_element + .and_then(|e| attr(&e, "href")) + .unwrap_or_default(); + + if !chapter_text.is_empty() || !date_text.is_empty() || !href_text.is_empty() { + Some((chapter_text, date_text, href_text)) + } else { + None + } + }) + .collect(); + + let chapters: Vec = raw_chapter_data + .par_iter() + .filter_map(|(chapter_text, date_text, href_text)| { + let chapter = { + let trimmed_chapter_text = chapter_text.trim(); + if let Some(captures) = CHAPTER_TITLE_REGEX.captures(trimmed_chapter_text) { + captures + .get(1) + .map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string()) + } else if let Some(captures) = CHAPTER_NUMBER_REGEX.captures(trimmed_chapter_text) { + captures + .get(1) + .map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string()) + } else { + trimmed_chapter_text.to_string() + } + }; + + let date = date_text.trim().to_string(); + + let chapter_id = href_text + .split('/') + .filter(|s| !s.is_empty()) + .next_back() + .unwrap_or("") + .to_string(); + + if !chapter_id.is_empty() { + Some(crate::modules::komik::types::Chapter { + chapter, + date, + chapter_id, + }) + } else { + None + } + }) + .collect(); + + let duration = start_time.elapsed(); + info!("Parsed komik detail document in {:?}", duration); + + Ok(DetailData { + title, + poster, + description, + status, + r#type, + release_date, + author, + total_chapter, + updated_on, + genres, + chapters, + }) +} + +pub fn parse_genre_page( + html: &str, + current_page: u32, +) -> Result<(Vec, Pagination), String> { + let document = parse_html(html); + let mut komik_list = Vec::new(); + + let item_selector = + selector(".bge, article, .ls4, .ls2").ok_or("Selector error".to_string())?; + let title_selector = selector(".kan h3, h3 a, h4 a").ok_or("Selector error".to_string())?; + let img_selector = selector(".bgei img, img.lazy, img").ok_or("Selector error".to_string())?; + let chapter_selector = selector(".new1:last-of-type a span:last-child, .new1 a:last-child span:last-child, .ls4s a, .ls24, .ls2l a") + .ok_or("Selector error".to_string())?; + let score_selector = selector(".up, .numscore, .epx").ok_or("Selector error".to_string())?; + let type_selector = selector(".tpe1_inf, .ls3p, .type").ok_or("Selector error".to_string())?; + let link_selector = + selector(".kan h3 a, .bgei a, h3 a, h4 a, a").ok_or("Selector error".to_string())?; + let next_selector = selector("span[hx-get]").ok_or("Selector error".to_string())?; + let slug_regex = Regex::new(r"/([^/]+)/?$").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + + let poster = element + .select(&img_selector) + .next() + .and_then(|e| attr(&e, "data-src").or(attr(&e, "src"))) + .unwrap_or_else(|| "".to_string()) + .to_string(); + + let chapter = text_from_or(&element, &chapter_selector, "N/A"); + + let score = text_from_or(&element, &score_selector, "N/A"); + + let komik_type = text_from_or(&element, &type_selector, "Unknown") + .split_whitespace() + .next() + .unwrap_or("Unknown") + .to_string(); + + let komik_url = attr_from_or(&element, &link_selector, "href", ""); + + let slug = slug_regex + .captures(&komik_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + komik_list.push(KomikItem { + title, + slug, + poster, + chapter, + score, + r#type: komik_type, + komik_url, + }); + } + } + + let has_next_page = document.select(&next_selector).next().is_some(); + let last_visible_page = if has_next_page { + current_page + 1 + } else { + current_page + }; + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some(current_page + 1) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some(current_page - 1) + } else { + None + }, + }; + + Ok((komik_list, pagination)) +} diff --git a/src/modules/komik/repository.rs b/src/modules/komik/repository.rs new file mode 100644 index 0000000..5e487d4 --- /dev/null +++ b/src/modules/komik/repository.rs @@ -0,0 +1,78 @@ +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::fetch_html_with_retry; +use crate::shared::utils::web::scraping_urls::{get_komik_api_url, get_komik_url}; +use async_trait::async_trait; + +pub struct KomikRepository; + +impl Default for KomikRepository { + fn default() -> Self { + Self::new() + } +} + +impl KomikRepository { + pub fn new() -> Self { + Self + } + + pub fn api_url(&self) -> String { + get_komik_api_url() + } + + pub fn base_url(&self) -> String { + get_komik_url() + } + + pub fn genre_url(&self, genre_slug: &str, page: u32) -> String { + if page == 1 { + format!("{}/genre/{}/", self.api_url(), genre_slug) + } else { + format!("{}/genre/{}/page/{}/", self.api_url(), genre_slug, page) + } + } + + pub fn search_url(&self, query: &str, page: u32) -> String { + if page == 1 { + format!("{}/search/{}/", self.api_url(), query) + } else { + format!("{}/search/{}/page/{}/", self.api_url(), query, page) + } + } + + pub fn manga_list_url(&self, page: u32) -> String { + format!("{}/manga/page/{}/?tipe=manga", self.api_url(), page) + } + + pub fn manhua_list_url(&self, page: u32) -> String { + format!("{}/manga/page/{}/?tipe=manhua", self.api_url(), page) + } + + pub fn manhwa_list_url(&self, page: u32) -> String { + format!("{}/manga/page/{}/?tipe=manhwa", self.api_url(), page) + } + + pub fn popular_list_url(&self, page: u32) -> String { + format!( + "{}/manga/page/{}/?orderby=meta_value_num", + self.api_url(), + page + ) + } + + pub fn detail_url(&self, slug: &str) -> String { + format!("{}/manga/{}/", self.base_url(), slug) + } + + pub fn chapter_url(&self, chapter_url: &str) -> String { + format!("{}/{}", self.base_url(), chapter_url) + } +} + +#[async_trait] +impl ScrapingRepository for KomikRepository { + async fn fetch_html(&self, url: &str) -> Result { + fetch_html_with_retry(url).await + } +} diff --git a/src/modules/komik/route.rs b/src/modules/komik/route.rs new file mode 100644 index 0000000..7f207aa --- /dev/null +++ b/src/modules/komik/route.rs @@ -0,0 +1,27 @@ +use std::sync::Arc; + +use axum::{routing::get, Router}; + +use crate::modules::komik::controller; +use crate::shared::state::AppState; + +pub fn routes(router: Router>) -> Router> { + router + .route("/api/komik/genre_list", get(controller::genre_list)) + .route("/api/komik/chapter/{slug}", get(controller::chapter_slug)) + .route("/api/komik/detail/{slug}", get(controller::detail_slug)) + .route("/api/komik/genre/{slug}", get(controller::genre_slug)) + .route( + "/api/komik/genre/{slug}/{page}", + get(controller::genre_slug_page), + ) + .route("/api/komik/manga/{slug}", get(controller::manga_slug)) + .route("/api/komik/manhua/{slug}", get(controller::manhua_slug)) + .route("/api/komik/manhwa/{slug}", get(controller::manhwa_slug)) + .route("/api/komik/popular/{slug}", get(controller::popular_slug)) + .route("/api/komik/search/{slug}", get(controller::search_slug)) + .route( + "/api/komik/search/{slug}/{page}", + get(controller::search_slug_page), + ) +} diff --git a/src/modules/komik/schema.rs b/src/modules/komik/schema.rs new file mode 100644 index 0000000..0a6985c --- /dev/null +++ b/src/modules/komik/schema.rs @@ -0,0 +1,18 @@ +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct SlugPath { + pub slug: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SlugPagePath { + pub slug: String, + pub page: String, +} + +#[derive(Deserialize)] +pub struct ChapterQuery { + /// URL-friendly identifier for the chapter (typically the chapter slug or URL path) + pub chapter_url: Option, +} diff --git a/src/modules/komik/service.rs b/src/modules/komik/service.rs new file mode 100644 index 0000000..e5f18a7 --- /dev/null +++ b/src/modules/komik/service.rs @@ -0,0 +1,421 @@ +use std::sync::Arc; + +use crate::modules::komik::parser; +use crate::modules::komik::repository::KomikRepository; +use crate::modules::komik::types::{ + ChapterResponse, DetailResponse, GenreKomikResponse, GenresResponse, SearchKomikResponse, +}; +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::services::images::cache::{ + apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original, +}; +use crate::shared::state::AppState; +use crate::shared::utils::Cache; + +const GENRE_LIST_CACHE_TTL: u64 = 3600; +const GENRE_CACHE_TTL: u64 = 300; +const DETAIL_CACHE_TTL: u64 = 300; +const CHAPTER_CACHE_TTL: u64 = 300; +const SEARCH_CACHE_TTL: u64 = 300; + +pub struct KomikService { + repository: KomikRepository, +} + +impl KomikService { + pub fn new(repository: KomikRepository) -> Self { + Self { repository } + } + + pub async fn genre_list(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = "komik:genres:list:v3"; + + cache + .get_or_set(cache_key, GENRE_LIST_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.api_url()) + .await + .map_err(|e| e.to_string())?; + let genres = tokio::task::spawn_blocking(move || parser::parse_genres(&html)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + Ok(GenresResponse { + status: "Ok".to_string(), + data: genres, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_slug( + &self, + genre_slug: String, + app_state: Arc, + ) -> Result { + let page = 1; + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let url = self.repository.genre_url(&genre_slug, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(GenreKomikResponse { + status: "Ok".to_string(), + genre: genre_slug.clone(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_slug_page( + &self, + genre_slug: String, + page: u32, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let url = self.repository.genre_url(&genre_slug, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(GenreKomikResponse { + status: "Ok".to_string(), + genre: genre_slug.clone(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn detail_slug( + &self, + komik_id: String, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:detail:{}", komik_id); + + cache + .get_or_set(&cache_key, DETAIL_CACHE_TTL, || async { + let url = self.repository.detail_url(&komik_id); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let mut data = + tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + if !data.poster.is_empty() { + data.poster = get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + } + + Ok(DetailResponse { status: true, data }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn chapter_slug( + &self, + chapter_url: String, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:chapter:{}", chapter_url); + + cache + .get_or_set(&cache_key, CHAPTER_CACHE_TTL, || async { + let url = self.repository.chapter_url(&chapter_url); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let mut data = tokio::task::spawn_blocking({ + let chapter_url = chapter_url.clone(); + move || parser::parse_komik_chapter_document(&html, &chapter_url) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + data.images = cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + data.images, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ChapterResponse { + message: "Ok".to_string(), + data, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn manga_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "manga", + page, + self.repository.manga_list_url(page), + app_state, + ) + .await + } + + pub async fn manhua_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "manhua", + page, + self.repository.manhua_list_url(page), + app_state, + ) + .await + } + + pub async fn manhwa_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "manhwa", + page, + self.repository.manhwa_list_url(page), + app_state, + ) + .await + } + + pub async fn popular_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "popular", + page, + self.repository.popular_list_url(page), + app_state, + ) + .await + } + + async fn list_by_url( + &self, + list_name: &str, + page: u32, + url: String, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:list:{}:{}:v2", list_name, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + if komik_list.is_empty() { + return Err(format!("Empty komik {} page {}", list_name, page)); + } + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(GenreKomikResponse { + status: "Ok".to_string(), + genre: list_name.to_string(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn search_slug( + &self, + query: String, + app_state: Arc, + ) -> Result { + let page = 1; + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:search:{}:{}", query, page); + + cache + .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { + let url = self.repository.search_url(&query, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(SearchKomikResponse { + status: "Ok".to_string(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn search_slug_page( + &self, + query: String, + page: u32, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:search:{}:{}", query, page); + + cache + .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { + let url = self.repository.search_url(&query, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(SearchKomikResponse { + status: "Ok".to_string(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } +} diff --git a/src/modules/komik/types.rs b/src/modules/komik/types.rs new file mode 100644 index 0000000..463239a --- /dev/null +++ b/src/modules/komik/types.rs @@ -0,0 +1,118 @@ +use crate::shared::types::entities::anime::HasPoster; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Genre { + pub name: String, + pub slug: String, + pub count: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenresResponse { + pub status: String, + pub data: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct ChapterData { + pub title: String, + pub next_chapter_id: String, + pub prev_chapter_id: String, + pub list_chapter: String, + pub images: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct ChapterResponse { + pub message: String, + pub data: ChapterData, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Chapter { + pub chapter: String, + pub date: String, + pub chapter_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailData { + pub title: String, + pub poster: String, + pub description: String, + pub status: String, + pub r#type: String, + pub release_date: String, + pub author: String, + pub total_chapter: String, + pub updated_on: String, + pub genres: Vec, + pub chapters: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailResponse { + pub status: bool, + pub data: DetailData, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct KomikDetailRequest { + pub komik_id: String, + pub chapter_id: Option, +} + +#[derive(Debug, Serialize, Clone, ToSchema)] +pub enum KomikDetailEvent { + Chapter(Chapter), + Detail(DetailData), + Error(String), + EndOfStream, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct KomikItem { + pub title: String, + pub slug: String, + pub poster: String, + pub chapter: String, + pub score: String, + pub r#type: String, + pub komik_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Pagination { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreKomikResponse { + pub status: String, + pub genre: String, + pub data: Vec, + pub pagination: Pagination, +} + +impl HasPoster for KomikItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchKomikResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} diff --git a/src/modules/mod.rs b/src/modules/mod.rs new file mode 100644 index 0000000..0bd4033 --- /dev/null +++ b/src/modules/mod.rs @@ -0,0 +1,17 @@ +use axum::Router; +use std::sync::Arc; + +use crate::shared::state::AppState; + +pub mod anime; +pub mod anime2; +pub mod komik; +pub mod proxy; + +pub fn routes(router: Router>) -> Router> { + let router = anime::route::routes(router); + let router = anime2::route::routes(router); + let router = komik::route::routes(router); + let router = proxy::route::routes(router); + router +} diff --git a/src/modules/proxy/controller.rs b/src/modules/proxy/controller.rs new file mode 100644 index 0000000..18780ca --- /dev/null +++ b/src/modules/proxy/controller.rs @@ -0,0 +1,75 @@ +use crate::modules::proxy::repository::ProxyRepository; +use crate::modules::proxy::schema::{AuditImageCacheRequest, ImageCacheRequest, ProxyParams}; +use crate::modules::proxy::service::ProxyService; +use crate::modules::proxy::types::{AuditImageCacheResponse, ImageCacheResponse}; +use crate::shared::database::repositories::image_cache::SeaOrmImageCacheRepository; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; +use axum::extract::{Json, Query, State}; +use axum::response::Response; +use std::sync::Arc; + +fn make_service(state: &Arc) -> ProxyService { + let repo = Arc::new(SeaOrmImageCacheRepository::new( + state.db.clone(), + state.redis_pool.clone(), + )); + ProxyService::new(ProxyRepository::new(), repo) +} + +#[utoipa::path( + get, + path = "/api/proxy/croxy", + tag = "proxy", + operation_id = "proxy_croxy", + params(ProxyParams), + responses( + (status = 200, description = "Handles GET requests for the /api/proxy/croxy endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn fetch_with_proxy_only( + State(state): State>, + Query(params): Query, +) -> Result { + make_service(&state).fetch_with_proxy_only(params).await +} + +#[utoipa::path( + post, + path = "/api/proxy/image-cache", + tag = "proxy", + operation_id = "proxy_image_cache", + request_body = ImageCacheRequest, + responses( + (status = 200, description = "Cache an image to CDN and return the cached URL", body = ImageCacheResponse), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn image_cache( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + make_service(&state).image_cache(state, req).await.map(Json) +} + +#[utoipa::path( + post, + path = "/api/proxy/image-cache/audit", + tag = "proxy", + operation_id = "proxy_image_cache_audit", + request_body = AuditImageCacheRequest, + responses( + (status = 200, description = "Audit an image cache entry", body = AuditImageCacheResponse), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn audit_image_cache( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + make_service(&state) + .audit_image_cache(state, req) + .await + .map(Json) +} diff --git a/src/modules/proxy/mod.rs b/src/modules/proxy/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/proxy/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/proxy/parser.rs b/src/modules/proxy/parser.rs new file mode 100644 index 0000000..b1f7bc0 --- /dev/null +++ b/src/modules/proxy/parser.rs @@ -0,0 +1 @@ +// Proxy endpoints do not parse HTML or structured upstream payloads. diff --git a/src/modules/proxy/repository.rs b/src/modules/proxy/repository.rs new file mode 100644 index 0000000..84f5e9a --- /dev/null +++ b/src/modules/proxy/repository.rs @@ -0,0 +1,29 @@ +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::{self, FetchResult}; +use async_trait::async_trait; + +pub struct ProxyRepository; + +impl Default for ProxyRepository { + fn default() -> Self { + Self::new() + } +} + +impl ProxyRepository { + pub fn new() -> Self { + Self + } + + pub async fn fetch_with_proxy_url(&self, url: &str) -> Result { + proxy_fetch::fetch_with_proxy(url).await + } +} + +#[async_trait] +impl ScrapingRepository for ProxyRepository { + async fn fetch_html(&self, url: &str) -> Result { + self.fetch_with_proxy_url(url).await.map(|r| r.data) + } +} diff --git a/src/modules/proxy/route.rs b/src/modules/proxy/route.rs new file mode 100644 index 0000000..78c4f8e --- /dev/null +++ b/src/modules/proxy/route.rs @@ -0,0 +1,16 @@ +use crate::modules::proxy::controller; +use crate::shared::state::AppState; +use axum::Router; +use std::sync::Arc; + +pub fn routes(router: Router>) -> Router> { + router + .route( + "/api/proxy/croxy", + axum::routing::get(controller::fetch_with_proxy_only), + ) + .route( + "/api/proxy/image-cache", + axum::routing::post(controller::image_cache), + ) +} diff --git a/src/modules/proxy/schema.rs b/src/modules/proxy/schema.rs new file mode 100644 index 0000000..4ddb84f --- /dev/null +++ b/src/modules/proxy/schema.rs @@ -0,0 +1,27 @@ +use serde::Deserialize; +use utoipa::IntoParams; +use utoipa::ToSchema; + +/// Query parameters for proxy fetch (GET) +#[derive(Debug, Deserialize, ToSchema, IntoParams)] +pub struct ProxyParams { + /// URL to fetch via proxy + pub url: String, +} + +/// Request body for image cache (POST) +#[derive(Debug, Deserialize, ToSchema)] +pub struct ImageCacheRequest { + /// Original image URL to cache + pub url: String, + /// If true, returns original URL immediately and caches in background + #[serde(default)] + pub lazy: bool, +} + +/// Request body for auditing image cache (POST) +#[derive(Debug, Deserialize, ToSchema)] +pub struct AuditImageCacheRequest { + /// Original image URL to audit + pub url: String, +} diff --git a/src/modules/proxy/service.rs b/src/modules/proxy/service.rs new file mode 100644 index 0000000..3cf39ed --- /dev/null +++ b/src/modules/proxy/service.rs @@ -0,0 +1,232 @@ +use crate::modules::proxy::repository::ProxyRepository; +use crate::modules::proxy::schema::{AuditImageCacheRequest, ImageCacheRequest, ProxyParams}; +use crate::modules::proxy::types::{AuditImageCacheResponse, ImageCacheResponse}; +use crate::shared::config::CONFIG; +use crate::shared::database::traits::image_cache::ImageCacheRepository; +use crate::shared::errors::AppError; +use crate::shared::events::bus::ImageRepaired; +use crate::shared::services::images::cache::ImageCache; +use crate::shared::state::AppState; +use axum::http::StatusCode; +use axum::response::Response; +use std::sync::Arc; +use tracing::{error, info, warn}; + +pub struct ProxyService { + repository: ProxyRepository, + image_cache_repo: Arc, +} + +impl ProxyService { + pub fn new( + repository: ProxyRepository, + image_cache_repo: Arc, + ) -> Self { + Self { + repository, + image_cache_repo, + } + } + + fn build_image_cache(&self) -> ImageCache { + ImageCache::new(self.image_cache_repo.clone()) + } + + pub async fn fetch_with_proxy_only(&self, params: ProxyParams) -> Result { + let url = params.url; + match self.repository.fetch_with_proxy_url(&url).await { + Ok(fetch_result) => { + let mut builder = Response::builder().status(StatusCode::OK); + if let Some(content_type) = fetch_result.content_type { + builder = builder.header("Content-Type", content_type); + } + Ok(builder.body(fetch_result.data.into())?) + } + Err(e) => { + error!("Proxy fetch error: {:?}", e); + Err(AppError::Other(format!( + "Failed to fetch URL via proxy: {}", + e + ))) + } + } + } + + pub async fn image_cache( + &self, + state: Arc, + req: ImageCacheRequest, + ) -> Result { + let cache = self + .build_image_cache() + .with_semaphore(state.image_processing_semaphore.clone()); + + if let Some(cdn_url) = cache.get_cdn_url(&req.url).await { + return Ok(ImageCacheResponse { + success: true, + original_url: req.url, + cdn_url, + from_cache: true, + pending: None, + }); + } + + if req.lazy { + let url = req.url.clone(); + let repo = self.image_cache_repo.clone(); + let semaphore = state.image_processing_semaphore.clone(); + tokio::spawn(async move { + let cache = ImageCache::new(repo).with_semaphore(semaphore); + match cache.get_or_cache(&url).await { + Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url, cdn), + Err(e) => warn!("[LazyCache] Failed {}: {}", url, e), + } + }); + return Ok(ImageCacheResponse { + success: true, + original_url: req.url.clone(), + cdn_url: req.url, + from_cache: false, + pending: Some(true), + }); + } + + match cache.get_or_cache(&req.url).await { + Ok(cdn_url) => Ok(ImageCacheResponse { + success: true, + original_url: req.url, + cdn_url, + from_cache: false, + pending: None, + }), + Err(e) => { + error!("ImageCache error: {}", e); + Ok(ImageCacheResponse { + success: false, + original_url: req.url.clone(), + cdn_url: req.url, + from_cache: false, + pending: None, + }) + } + } + } + + pub async fn audit_image_cache( + &self, + state: Arc, + req: AuditImageCacheRequest, + ) -> Result { + let cache = self.build_image_cache(); + + let mut cdn_opt = cache.get_cdn_url(&req.url).await; + let mut original = req.url.clone(); + if cdn_opt.is_none() { + if let Some(orig) = cache.find_original_from_cdn(&req.url).await { + info!( + "SmartAudit: {} recognized as CDN, original {}", + req.url, orig + ); + original = orig; + cdn_opt = Some(req.url.clone()); + } + } + + if let Some(cdn_url) = cdn_opt { + let client = crate::shared::utils::web::http_client::http_client().client(); + let mut accessible = false; + match client.get(cdn_url.clone()).send().await { + Ok(resp) if resp.status().is_success() => { + if let Ok(bytes) = resp.bytes().await { + if infer::get(&bytes) + .map(|k| k.mime_type().starts_with("image/")) + .unwrap_or(false) + { + accessible = true; + } else { + warn!("CDN {} returned non-image content", cdn_url); + } + } + } + Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()), + Err(e) => warn!("CDN {} fetch error {}", cdn_url, e), + } + if accessible { + return Ok(AuditImageCacheResponse { + success: true, + original_url: original, + cdn_url: Some(cdn_url), + was_accessible: true, + re_uploaded: false, + message: "CDN URL is accessible and the image is valid".to_string(), + }); + } + info!("CDN {} inaccessible, purging and reuploading", cdn_url); + let picser_delete_url = &CONFIG.urls.picser_api_url; + if let Some(filename) = cdn_url.split('/').last() { + let payload = serde_json::json!({ "filename": filename }); + match client.delete(picser_delete_url).json(&payload).send().await { + Ok(r) if r.status().is_success() => info!("Deleted {} via Picser", filename), + Ok(r) => warn!("Picser delete {} status {}", filename, r.status()), + Err(e) => warn!("Picser delete error {}: {}", filename, e), + } + } + let _ = cache.invalidate(&original).await; + match cache.get_or_cache(&original).await { + Ok(new_cdn) => { + state + .event_bus + .publish(ImageRepaired { + original_url: original.clone(), + cdn_url: new_cdn.clone(), + }) + .await; + Ok(AuditImageCacheResponse { + success: true, + original_url: original, + cdn_url: Some(new_cdn), + was_accessible: false, + re_uploaded: true, + message: "CDN URL was inaccessible, re-uploaded".to_string(), + }) + } + Err(e) => Ok(AuditImageCacheResponse { + success: false, + original_url: original, + cdn_url: None, + was_accessible: false, + re_uploaded: false, + message: format!("Re-upload failed: {}", e), + }), + } + } else { + match cache.get_or_cache(&original).await { + Ok(new_cdn) => { + state + .event_bus + .publish(ImageRepaired { + original_url: original.clone(), + cdn_url: new_cdn.clone(), + }) + .await; + Ok(AuditImageCacheResponse { + success: true, + original_url: original, + cdn_url: Some(new_cdn), + was_accessible: false, + re_uploaded: true, + message: "Cached newly".to_string(), + }) + } + Err(e) => Ok(AuditImageCacheResponse { + success: false, + original_url: original, + cdn_url: None, + was_accessible: false, + re_uploaded: false, + message: format!("Cache failed: {}", e), + }), + } + } + } +} diff --git a/src/modules/proxy/types.rs b/src/modules/proxy/types.rs new file mode 100644 index 0000000..8e619f0 --- /dev/null +++ b/src/modules/proxy/types.rs @@ -0,0 +1,24 @@ +use serde::Serialize; +use utoipa::ToSchema; + +/// Response for image cache POST +#[derive(Debug, Serialize, ToSchema)] +pub struct ImageCacheResponse { + pub success: bool, + pub original_url: String, + pub cdn_url: String, + pub from_cache: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub pending: Option, +} + +/// Response for image cache audit POST +#[derive(Debug, Serialize, ToSchema)] +pub struct AuditImageCacheResponse { + pub success: bool, + pub original_url: String, + pub cdn_url: Option, + pub was_accessible: bool, + pub re_uploaded: bool, + pub message: String, +} diff --git a/src/shared/browser/mod.rs b/src/shared/browser/mod.rs new file mode 100644 index 0000000..52bd506 --- /dev/null +++ b/src/shared/browser/mod.rs @@ -0,0 +1,3 @@ +pub mod pool; + +pub use pool::{init_browser_pool, BrowserPoolConfig}; diff --git a/src/shared/browser/pool.rs b/src/shared/browser/pool.rs new file mode 100644 index 0000000..c69d5c1 --- /dev/null +++ b/src/shared/browser/pool.rs @@ -0,0 +1,483 @@ +//! Browser pool for managing a single remote browser with multiple tabs. +//! +//! This pool maintains a connection to a remote Chrome DevTools Protocol (CDP) +//! endpoint and provides tabs on-demand for scraping. Tabs are returned to the +//! pool after use. Requires EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS to be set. + +use reqwest::StatusCode; +use serde_json::json; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::sync::{Mutex, Semaphore}; +use tracing::{debug, info, warn}; + +fn build_http_endpoint(remote_url: &str, path: &str) -> anyhow::Result { + let mut url = reqwest::Url::parse(remote_url) + .map_err(|e| anyhow::anyhow!("Invalid remote browser URL '{}': {}", remote_url, e))?; + + match url.scheme() { + "ws" => { + url.set_scheme("http") + .map_err(|_| anyhow::anyhow!("Failed to convert scheme from ws to http"))?; + } + "wss" => { + url.set_scheme("https") + .map_err(|_| anyhow::anyhow!("Failed to convert scheme from wss to https"))?; + } + "http" | "https" => {} + scheme => { + return Err(anyhow::anyhow!( + "Unsupported browser URL scheme '{}'. Expected ws:// or wss://", + scheme + )); + } + } + + let normalized = path.trim_start_matches('/'); + url.set_path(normalized); + + Ok(url.to_string()) +} + +/// Configuration for the browser pool. +#[derive(Debug, Clone)] +pub struct BrowserPoolConfig { + /// Remote Chrome DevTools Protocol WebSocket URL (required). + /// Must be set via `EXTERNAL_BROWSERLESS_WS` or `CHROME_REMOTE_WS` environment variable. + /// Takes precedence from: EXTERNAL_BROWSERLESS_WS → CHROME_REMOTE_WS → None + pub remote_websocket_url: String, + /// Maximum number of concurrent tabs + pub max_tabs: usize, +} + +impl Default for BrowserPoolConfig { + fn default() -> Self { + // Priority (highest → lowest): + // 1. EXTERNAL_BROWSERLESS_WS — explicit operator override, always wins + // 2. CHROME_REMOTE_WS — may be injected by Coolify/Docker networking; + // rejected when it resolves to the unroutable Docker alias + let external = std::env::var("EXTERNAL_BROWSERLESS_WS").ok(); + let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok(); + + let remote_websocket_url = if let Some(ext) = external { + tracing::info!("🌐 Browser: using EXTERNAL_BROWSERLESS_WS"); + ext + } else if let Some(ref cr) = chrome_remote { + if cr == "ws://browserless:3000" { + eprintln!( + "CHROME_REMOTE_WS is set to the unroutable Docker alias \"ws://browserless:3000\" \ + and EXTERNAL_BROWSERLESS_WS is unset. Remote browser connectivity is required." + ); + std::process::exit(1); + } else { + tracing::info!("🌐 Browser: using CHROME_REMOTE_WS"); + cr.clone() + } + } else { + eprintln!( + "Remote browser URL not configured. Set EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS environment variable." + ); + std::process::exit(1); + }; + + Self { + remote_websocket_url, + max_tabs: 10, + } + } +} + +/// A pool of browser tabs backed by a single remote browser instance. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::browser::{BrowserPool, BrowserPoolConfig}; +/// +/// let pool = BrowserPool::new(BrowserPoolConfig::default()).await?; +/// let tab = pool.get_tab().await?; +/// tab.goto("https://example.com").await?; +/// let html = tab.content().await?; +/// ``` +pub struct BrowserPool { + /// Available (idle) tab IDs + available_tabs: Mutex>, + /// Semaphore to limit concurrent tabs + semaphore: Arc, + /// Configuration + config: BrowserPoolConfig, + /// Counter for generating unique tab IDs + tab_counter: AtomicU64, +} + +impl BrowserPool { + /// Create a new browser pool connecting to a remote CDP endpoint. + /// + /// The pool will attempt to connect to the remote browser URL specified in + /// BrowserPoolConfig. If the connection fails, an error is returned. + pub async fn new(config: BrowserPoolConfig) -> anyhow::Result> { + info!( + "Initializing browser pool (remote: {}) with max {} tabs", + config.remote_websocket_url, config.max_tabs + ); + + // Verify connection to remote browser with a simple health check + Self::verify_remote_connection(&config).await?; + + let pool = Arc::new(Self { + available_tabs: Mutex::new(Vec::new()), + semaphore: Arc::new(Semaphore::new(config.max_tabs)), + config: config.clone(), + tab_counter: AtomicU64::new(0), + }); + + info!("Browser pool initialized (remote-only)"); + Ok(pool) + } + + /// Verify connection to the remote browser. + async fn verify_remote_connection(config: &BrowserPoolConfig) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + + // Browserless/CDP deployments vary by exposed endpoint. + // Try multiple known paths and succeed on any valid HTTP response from the host. + let check_paths = ["json/version", "health", ""]; + let mut last_non_success: Option<(String, StatusCode)> = None; + + for path in check_paths { + let endpoint = build_http_endpoint(&config.remote_websocket_url, path)?; + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + client.get(&endpoint).send(), + ) + .await; + + match response { + Ok(Ok(r)) if r.status().is_success() => { + info!("Remote browser health check passed via {}", endpoint); + return Ok(()); + } + Ok(Ok(r)) + if r.status() == StatusCode::UNAUTHORIZED + || r.status() == StatusCode::FORBIDDEN => + { + return Err(anyhow::anyhow!( + "Remote browser reachable but authentication failed at {}: {}", + endpoint, + r.status() + )); + } + Ok(Ok(r)) => { + last_non_success = Some((endpoint, r.status())); + } + Ok(Err(e)) => { + return Err(anyhow::anyhow!( + "Failed to connect to remote browser: {}", + e + )); + } + Err(_) => { + return Err(anyhow::anyhow!("Remote browser connection timeout")); + } + } + } + + if let Some((endpoint, status)) = last_non_success { + warn!( + "Remote browser reachable but no known health endpoint succeeded (last: {} -> {})", + endpoint, status + ); + } + + Ok(()) + } + + /// Get a tab from the pool. + /// + /// This will reuse an existing idle tab or create a new one. + /// The returned `PooledTab` automatically returns to the pool when dropped. + pub async fn get_tab(self: &Arc) -> anyhow::Result { + // Acquire semaphore permit (limits concurrent tabs) + let permit = self.semaphore.clone().acquire_owned().await?; + + // Try to reuse an existing tab ID, or generate a new one + let tab_id = { + let mut tabs = self.available_tabs.lock().await; + tabs.pop().unwrap_or_else(|| { + let id = self.tab_counter.fetch_add(1, Ordering::SeqCst); + format!("tab-{}", id) + }) + }; + + debug!("Allocated tab: {}", tab_id); + + Ok(PooledTab { + tab_id, + cdp_url: self.config.remote_websocket_url.clone(), + pool: Arc::clone(self), + _permit: permit, + }) + } + + /// Get the number of available (idle) tabs in the pool. + pub async fn available_count(&self) -> usize { + self.available_tabs.lock().await.len() + } + + /// Close the browser pool. + pub async fn close(&self) -> anyhow::Result<()> { + info!("Closing browser pool"); + self.available_tabs.lock().await.clear(); + Ok(()) + } +} + +/// A tab borrowed from the pool (remote CDP-based). +/// +/// This is a lightweight wrapper that communicates with a remote Chrome instance +/// via HTTP calls to the browser service. The tab is returned to the pool when dropped. +pub struct PooledTab { + /// Unique identifier for this tab + pub tab_id: String, + /// Remote CDP/browser service URL + cdp_url: String, + /// Pool reference for returning on drop + pool: Arc, + /// Semaphore permit (released on drop) + _permit: tokio::sync::OwnedSemaphorePermit, +} + +impl PooledTab { + /// Navigate to a URL. + pub async fn goto(&self, url: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "goto")?; + + let response = tokio::time::timeout( + timeout, + client.post(&endpoint).json(&json!({ "url": url })).send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout navigating to {}", url))? + .map_err(|e| anyhow::anyhow!("Failed to navigate to {}: {}", url, e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Failed to navigate to {}: {}", + url, + response.status() + )) + } + } + + /// Get the page content (HTML). + pub async fn content(&self) -> anyhow::Result { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "content")?; + + let response = tokio::time::timeout(timeout, client.post(&endpoint).send()) + .await + .map_err(|_| anyhow::anyhow!("Timeout getting page content"))? + .map_err(|e| anyhow::anyhow!("Failed to get page content: {}", e))?; + + response + .text() + .await + .map_err(|e| anyhow::anyhow!("Failed to read response body: {}", e)) + } + + /// Execute JavaScript and return the result. + pub async fn evaluate( + &self, + expression: &str, + ) -> anyhow::Result { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "evaluate")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "expression": expression })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout evaluating JS"))? + .map_err(|e| anyhow::anyhow!("Failed to evaluate JS: {}", e))?; + + let data: serde_json::Value = response + .json() + .await + .map_err(|e| anyhow::anyhow!("Failed to parse JS result: {}", e))?; + + serde_json::from_value(data).map_err(|e| anyhow::anyhow!("Invalid JS result: {}", e)) + } + + /// Wait for a selector to appear. + pub async fn wait_for_selector(&self, selector: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "waitForSelector")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "selector": selector })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout waiting for selector '{}'", selector))? + .map_err(|e| anyhow::anyhow!("Selector '{}' error: {}", selector, e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Selector '{}' not found: {}", + selector, + response.status() + )) + } + } + + /// Click an element by selector. + pub async fn click(&self, selector: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "click")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "selector": selector })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout clicking element"))? + .map_err(|e| anyhow::anyhow!("Click error: {}", e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Failed to click '{}': {}", + selector, + response.status() + )) + } + } + + /// Type text into an element. + pub async fn type_text(&self, selector: &str, text: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "type")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "selector": selector, "text": text })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout typing text"))? + .map_err(|e| anyhow::anyhow!("Type error: {}", e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Failed to type into '{}': {}", + selector, + response.status() + )) + } + } + + /// Take a screenshot as PNG bytes. + pub async fn screenshot(&self) -> anyhow::Result> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "screenshot")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "fullPage": true })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout taking screenshot"))? + .map_err(|e| anyhow::anyhow!("Failed to take screenshot: {}", e))?; + + response + .bytes() + .await + .map(|b| b.to_vec()) + .map_err(|e| anyhow::anyhow!("Failed to read screenshot bytes: {}", e)) + } + + /// Get the current URL. + pub async fn url(&self) -> anyhow::Result { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(10); + let endpoint = build_http_endpoint(&self.cdp_url, "url")?; + + let response = tokio::time::timeout(timeout, client.post(&endpoint).send()) + .await + .map_err(|_| anyhow::anyhow!("Timeout getting URL"))? + .map_err(|e| anyhow::anyhow!("Failed to get URL: {}", e))?; + + response + .text() + .await + .map_err(|e| anyhow::anyhow!("Failed to read URL: {}", e)) + } +} + +impl Drop for PooledTab { + fn drop(&mut self) { + let pool = Arc::clone(&self.pool); + let tab_id = self.tab_id.clone(); + + let rt = tokio::runtime::Handle::try_current(); + if let Ok(handle) = rt { + handle.spawn(async move { + // Return tab ID to available pool for reuse + pool.available_tabs.lock().await.push(tab_id); + }); + } else { + warn!("Cannot return tab: no tokio runtime available"); + } + } +} + +// Global browser pool instance +use once_cell::sync::OnceCell; + +static BROWSER_POOL: OnceCell> = OnceCell::new(); + +/// Initialize the global browser pool. +/// Call this once at application startup. +pub async fn init_browser_pool(config: BrowserPoolConfig) -> anyhow::Result<()> { + let pool = BrowserPool::new(config).await?; + BROWSER_POOL + .set(pool) + .map_err(|_| anyhow::anyhow!("Browser pool already initialized"))?; + Ok(()) +} + +/// Get the global browser pool. +/// Returns None if not initialized. +pub fn get_browser_pool() -> Option> { + BROWSER_POOL.get().cloned() +} diff --git a/src/shared/config/mod.rs b/src/shared/config/mod.rs new file mode 100644 index 0000000..5da118d --- /dev/null +++ b/src/shared/config/mod.rs @@ -0,0 +1,289 @@ +//! Type-safe application configuration. +//! +//! This module provides a strongly-typed configuration system that: +//! - Loads from environment variables and optional TOML files +//! - Fails fast at startup if required variables are missing +//! - Supports hierarchical configuration (default -> environment-specific) + +use config::{Config, ConfigError, Environment, File}; +use once_cell::sync::Lazy; +use serde::Deserialize; +use std::env; + +/// Application configuration loaded at startup. +/// All fields are required unless marked as `Option`. +#[derive(Debug, Clone, Deserialize)] +pub struct AppConfig { + /// Database connection URL (PostgreSQL) + pub database_url: String, + + /// Secret key for JWT signing + pub jwt_secret: String, + + /// Redis connection URL + #[serde(default)] + pub redis_url: String, + + /// Server port to bind to + #[serde(default = "default_port")] + pub server_port: u16, + + /// Environment (development, staging, production) + #[serde(default = "default_env")] + pub environment: String, + + /// Allowed CORS origins (comma-separated) + #[serde(default)] + pub cors_origins: Vec, + + /// Log level (trace, debug, info, warn, error) + #[serde(default = "default_log_level")] + pub log_level: String, + + /// SMTP configuration for emails (optional) + pub smtp: Option, + + /// Database pool configuration + #[serde(default)] + pub db: DbConfig, + + /// Max concurrent image processing tasks + #[serde(default = "default_image_processing_concurrency")] + pub image_processing_concurrency: usize, + + /// Domain/URL values that may change between deployments + #[serde(default)] + pub urls: UrlConfig, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct UrlConfig { + #[serde(default = "default_site_url")] + pub site_url: String, + #[serde(default = "default_picser_api_url")] + pub picser_api_url: String, + #[serde(default = "default_fallback_upload_api_url")] + pub fallback_upload_api_url: String, +} + +impl Default for UrlConfig { + fn default() -> Self { + Self { + site_url: default_site_url(), + picser_api_url: default_picser_api_url(), + fallback_upload_api_url: default_fallback_upload_api_url(), + } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct DbConfig { + #[serde(default = "default_db_max_connections")] + pub max_connections: u32, + #[serde(default = "default_db_min_connections")] + pub min_connections: u32, + #[serde(default = "default_db_connect_timeout")] + pub connect_timeout_seconds: u64, + #[serde(default = "default_db_idle_timeout")] + pub idle_timeout_seconds: u64, + #[serde(default = "default_db_acquire_timeout")] + pub acquire_timeout_seconds: u64, + #[serde(default = "default_db_max_lifetime")] + pub max_lifetime_seconds: u64, +} + +impl Default for DbConfig { + fn default() -> Self { + Self { + max_connections: default_db_max_connections(), + min_connections: default_db_min_connections(), + connect_timeout_seconds: default_db_connect_timeout(), + idle_timeout_seconds: default_db_idle_timeout(), + acquire_timeout_seconds: default_db_acquire_timeout(), + max_lifetime_seconds: default_db_max_lifetime(), + } + } +} + +/// SMTP configuration for sending emails +#[derive(Debug, Clone, Deserialize)] +pub struct SmtpConfig { + pub host: String, + pub port: u16, + pub username: String, + pub password: String, + pub from_email: String, + pub from_name: String, +} + +/// MinIO/S3-compatible storage configuration +#[derive(Debug, Clone)] +pub struct MinioConfig { + /// MinIO endpoint URL (e.g., "https://cdn.asepharyana.my.id") + pub endpoint: String, + /// Bucket name + pub bucket_name: String, + /// Access key / username + pub access_key: String, + /// Secret key / password + pub secret_key: String, + /// Use HTTPS (true) or HTTP (false) + pub secure: bool, + /// AWS region (default: us-east-1) + pub region: String, + /// Public URL for serving files (optional) + pub public_url: Option, + /// Prefix for avatar files (e.g., "avatars") + pub avatar_prefix: String, +} + +impl MinioConfig { + /// Load MinIO configuration from environment variables + pub fn from_env() -> Option { + let endpoint = env::var("MINIO_ENDPOINT").ok()?; + let bucket_name = env::var("MINIO_BUCKET_NAME").ok()?; + let access_key = env::var("MINIO_ACCESS_KEY").ok()?; + let secret_key = env::var("MINIO_SECRET_KEY").ok()?; + + let secure = env::var("MINIO_SECURE") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(true); + + let region = env::var("MINIO_REGION").unwrap_or_else(|_| "us-east-1".to_string()); + + let public_url = env::var("MINIO_PUBLIC_URL").ok(); + + let avatar_prefix = + env::var("MINIO_AVATAR_PREFIX").unwrap_or_else(|_| "avatars".to_string()); + + Some(Self { + endpoint, + bucket_name, + access_key, + secret_key, + secure, + region, + public_url, + avatar_prefix, + }) + } +} + +fn default_port() -> u16 { + 4091 +} + +fn default_env() -> String { + "development".to_string() +} + +fn default_log_level() -> String { + "info".to_string() +} + +fn default_image_processing_concurrency() -> usize { + 5 +} + +fn default_site_url() -> String { + "https://asepharyana.my.id".to_string() +} + +fn default_picser_api_url() -> String { + "https://picser.asepharyana.my.id/api/upload".to_string() +} + +fn default_fallback_upload_api_url() -> String { + "https://upload.asepharyana.my.id/api/upload".to_string() +} + +fn default_db_max_connections() -> u32 { + 100 +} + +fn default_db_min_connections() -> u32 { + 10 +} + +fn default_db_connect_timeout() -> u64 { + 5 +} + +fn default_db_idle_timeout() -> u64 { + 300 +} + +fn default_db_acquire_timeout() -> u64 { + 10 +} + +fn default_db_max_lifetime() -> u64 { + 1800 +} + +impl AppConfig { + /// Load configuration from environment and optional config files. + /// + /// Priority (highest to lowest): + /// 1. Environment variables (prefixed with APP_) + /// 2. `config/{environment}.toml` + /// 3. `config/default.toml` + pub fn load() -> Result { + // Load .env file first + if let Err(e) = dotenvy::dotenv() { + tracing::debug!("Could not load .env file: {}", e); + } + + let run_mode = env::var("RUN_MODE").unwrap_or_else(|_| "development".into()); + + let config = Config::builder() + // Start with default config file + .add_source(File::with_name("config/default").required(false)) + // Layer on environment-specific values + .add_source(File::with_name(&format!("config/{}", run_mode)).required(false)) + // Add environment variables (with APP_ prefix) + .add_source( + Environment::with_prefix("APP") + .separator("__") + .try_parsing(true) + .list_separator(","), + ) + // Map legacy env vars to new config structure + .set_override_option("database_url", env::var("DATABASE_URL").ok())? + .set_override_option("jwt_secret", env::var("JWT_SECRET").ok())? + .set_override_option("redis_url", env::var("REDIS_URL").ok())? + .build()?; + + config.try_deserialize() + } + + /// Check if running in production mode + pub fn is_production(&self) -> bool { + self.environment == "production" + } + + /// Check if running in development mode + pub fn is_development(&self) -> bool { + self.environment == "development" + } +} + +/// Global configuration instance, loaded once at startup. +/// Panics if configuration is invalid - this is intentional for fail-fast behavior. +pub static CONFIG: Lazy = Lazy::new(|| { + AppConfig::load().unwrap_or_else(|e| { + eprintln!("❌ Failed to load configuration: {}", e); + eprintln!(" Make sure all required environment variables are set:"); + eprintln!(" - DATABASE_URL"); + eprintln!(" - JWT_SECRET"); + eprintln!(" - REDIS_URL (or APP_REDIS_URL)"); + std::process::exit(1); + }) +}); + +/// Global MinIO configuration, loaded from environment variables. +/// Returns None if required MINIO_* variables are not set. +pub static MINIO_CONFIG: Lazy> = Lazy::new(|| { + let _ = dotenvy::dotenv(); + MinioConfig::from_env() +}); diff --git a/src/shared/database/mod.rs b/src/shared/database/mod.rs new file mode 100644 index 0000000..85f0cc4 --- /dev/null +++ b/src/shared/database/mod.rs @@ -0,0 +1,7 @@ +pub mod persistence; +pub mod redis; +pub mod repositories; +pub mod setup; +pub mod traits; + +pub use redis::{get_redis_conn, get_redis_pool, redis_pool}; diff --git a/src/shared/database/persistence/entities/image_cache.rs b/src/shared/database/persistence/entities/image_cache.rs new file mode 100644 index 0000000..073fb6a --- /dev/null +++ b/src/shared/database/persistence/entities/image_cache.rs @@ -0,0 +1,71 @@ +//! `SeaORM` Entity for ImageCache - stores URL mappings for CDN caching + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "ImageCache" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)] +pub struct Model { + pub id: String, + pub original_url: String, + pub cdn_url: String, + pub created_at: DateTimeUtc, + pub expires_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + #[sea_orm(column_name = "original_url")] + OriginalUrl, + #[sea_orm(column_name = "cdn_url")] + CdnUrl, + #[sea_orm(column_name = "created_at")] + CreatedAt, + #[sea_orm(column_name = "expires_at")] + ExpiresAt, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = String; + fn auto_increment() -> bool { + false + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::String(StringLen::N(36u32)).def(), + Self::OriginalUrl => ColumnType::String(StringLen::N(512u32)).def().unique(), + Self::CdnUrl => ColumnType::String(StringLen::N(512u32)).def(), + Self::CreatedAt => ColumnType::TimestampWithTimeZone.def(), + Self::ExpiresAt => ColumnType::TimestampWithTimeZone.def().null(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match *self {} + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/shared/database/persistence/entities/mod.rs b/src/shared/database/persistence/entities/mod.rs new file mode 100644 index 0000000..44e6ce2 --- /dev/null +++ b/src/shared/database/persistence/entities/mod.rs @@ -0,0 +1 @@ +pub mod image_cache; diff --git a/src/shared/database/persistence/mod.rs b/src/shared/database/persistence/mod.rs new file mode 100644 index 0000000..0b8f0b5 --- /dev/null +++ b/src/shared/database/persistence/mod.rs @@ -0,0 +1 @@ +pub mod entities; diff --git a/src/shared/database/redis.rs b/src/shared/database/redis.rs new file mode 100644 index 0000000..01225b7 --- /dev/null +++ b/src/shared/database/redis.rs @@ -0,0 +1,74 @@ +//! Redis connection utility with tracing for connection lifecycle and errors. +//! +//! Uses the type-safe CONFIG for Redis connection parameters. + +use crate::shared::config::CONFIG; +use crate::shared::errors::AppError; +use deadpool_redis::{Manager, Pool}; +use once_cell::sync::Lazy; +use tracing::{debug, error, info}; + +static REDIS_POOL_INIT: Lazy> = Lazy::new(|| { + let redis_url = if !CONFIG.redis_url.is_empty() { + CONFIG.redis_url.clone() + } else { + let host = std::env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); + let port = std::env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_string()); + let password = std::env::var("REDIS_PASSWORD").unwrap_or_default(); + + if password.is_empty() { + format!("redis://{}:{}", host, port) + } else { + format!("redis://:{}@{}:{}", password, host, port) + } + }; + + info!("Initializing Redis connection pool for URL: {}", redis_url); + + Manager::new(redis_url) + .map_err(|e| format!("Failed to create Redis manager: {}", e)) + .and_then(|manager| { + Pool::builder(manager) + .max_size(100) + .wait_timeout(Some(std::time::Duration::from_millis(200))) + .runtime(deadpool_redis::Runtime::Tokio1) + .build() + .map_err(|e| format!("Failed to create Redis connection pool: {}", e)) + }) +}); + +/// Get the Redis connection pool (for internal use). +pub fn get_redis_pool() -> Result<&'static Pool, String> { + REDIS_POOL_INIT.as_ref().map_err(|e| e.clone()) +} + +/// Get a cloned reference to the Redis pool. +pub fn redis_pool() -> Result { + get_redis_pool().map(|p| (*p).clone()) +} + +/// Get an async connection from the pool with retry backoff. +pub async fn get_redis_conn() -> Result { + let pool = get_redis_pool().map_err(|e| AppError::Other(e))?; + let mut retries = 5; + let mut wait = std::time::Duration::from_millis(100); + + loop { + match pool.get().await { + Ok(conn) => { + debug!("Successfully retrieved Redis connection from pool."); + return Ok(conn); + } + Err(e) => { + if retries <= 0 { + error!("Failed to get Redis connection after retries: {:?}", e); + return Err(AppError::from(e)); + } + debug!("Redis connection failed, retrying in {:?}: {:?}", wait, e); + tokio::time::sleep(wait).await; + wait = std::cmp::min(wait * 2, std::time::Duration::from_secs(5)); + retries -= 1; + } + } + } +} diff --git a/src/shared/database/repositories/image_cache.rs b/src/shared/database/repositories/image_cache.rs new file mode 100644 index 0000000..53f8443 --- /dev/null +++ b/src/shared/database/repositories/image_cache.rs @@ -0,0 +1,129 @@ +use crate::shared::database::persistence::entities::image_cache; +use crate::shared::database::traits::image_cache::ImageCacheRepository; +use crate::shared::utils::Cache; +use async_trait::async_trait; +use chrono::Utc; +use deadpool_redis::Pool as RedisPool; +use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set}; +use std::sync::Arc; + +pub struct SeaOrmImageCacheRepository { + db: Arc, + redis: RedisPool, +} + +impl SeaOrmImageCacheRepository { + pub fn new(db: Arc, redis: RedisPool) -> Self { + Self { db, redis } + } +} + +#[async_trait] +impl ImageCacheRepository for SeaOrmImageCacheRepository { + async fn get_from_redis(&self, key: &str) -> Option { + Cache::new(&self.redis).get::(key).await + } + + async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String> { + Cache::new(&self.redis) + .set_with_ttl(key, &value, ttl) + .await + .map_err(|e| e.to_string()) + } + + async fn get_from_db(&self, original_url: &str) -> Result, String> { + let entry = image_cache::Entity::find() + .filter(image_cache::Column::OriginalUrl.eq(original_url)) + .one(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(entry.map(|m| m.cdn_url)) + } + + async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String> { + let model = image_cache::ActiveModel { + id: Set(uuid::Uuid::new_v4().to_string()), + original_url: Set(original_url.to_string()), + cdn_url: Set(cdn_url.to_string()), + created_at: Set(Utc::now()), + expires_at: Set(None), + }; + + model + .insert(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(()) + } + + async fn find_original_from_cdn(&self, cdn_url: &str) -> Result, String> { + let entry = image_cache::Entity::find() + .filter(image_cache::Column::CdnUrl.eq(cdn_url)) + .one(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(entry.map(|m| m.original_url)) + } + + async fn delete_from_db(&self, original_url: &str) -> Result<(), String> { + image_cache::Entity::delete_many() + .filter(image_cache::Column::OriginalUrl.eq(original_url)) + .exec(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(()) + } + + async fn delete_from_redis(&self, key: &str) -> Result<(), String> { + Cache::new(&self.redis) + .delete(key) + .await + .map_err(|e| e.to_string()) + } + + async fn get_lock(&self, key: &str) -> bool { + Cache::new(&self.redis).get::(key).await.is_some() + } + + async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String> { + Cache::new(&self.redis) + .set_with_ttl(key, &true, ttl) + .await + .map_err(|e| e.to_string()) + } + + async fn release_lock(&self, key: &str) -> Result<(), String> { + self.delete_from_redis(key).await + } + + async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String> { + use deadpool_redis::redis::{cmd, AsyncCommands}; + + let mut conn = self.redis.get().await.map_err(|e| e.to_string())?; + + for pattern in patterns { + let mut cursor: u64 = 0; + loop { + let (new_cursor, keys): (u64, Vec) = cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(100) + .query_async(&mut *conn) + .await + .map_err(|e| e.to_string())?; + + if !keys.is_empty() { + let _: usize = conn.del(&keys).await.map_err(|e| e.to_string())?; + } + + cursor = new_cursor; + if cursor == 0 { + break; + } + } + } + Ok(()) + } +} diff --git a/src/shared/database/repositories/mod.rs b/src/shared/database/repositories/mod.rs new file mode 100644 index 0000000..44e6ce2 --- /dev/null +++ b/src/shared/database/repositories/mod.rs @@ -0,0 +1 @@ +pub mod image_cache; diff --git a/src/shared/database/setup.rs b/src/shared/database/setup.rs new file mode 100644 index 0000000..38617da --- /dev/null +++ b/src/shared/database/setup.rs @@ -0,0 +1,44 @@ +use crate::shared::database::persistence::entities::image_cache; +use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement}; +use tracing::info; + +pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> { + info!("🚀 Initializing database schema..."); + let backend = db.get_database_backend(); + let schema = Schema::new(backend); + + let tables = vec![( + "ImageCache", + schema + .create_table_from_entity(image_cache::Entity) + .if_not_exists() + .to_owned(), + )]; + + for (name, stmt) in tables { + match db.execute(backend.build(&stmt)).await { + Ok(_) => info!(" ✓ Table '{}' checked/created", name), + Err(e) => { + tracing::error!(" [!] Failed to create table '{}': {}", name, e); + return Err(e); + } + } + } + + let index_sql = + "CREATE INDEX IF NOT EXISTS idx_image_cache_cdn_url ON \"ImageCache\" (cdn_url)"; + match db.execute(Statement::from_string(backend, index_sql)).await { + Ok(_) => info!(" ✓ Index 'idx_image_cache_cdn_url' ensured"), + Err(e) => { + let err_str = e.to_string(); + if err_str.contains("already exists") || err_str.contains("duplicate") { + info!(" ✓ Index 'idx_image_cache_cdn_url' already exists"); + } else { + tracing::error!(" [!] Failed to create index on ImageCache: {}", e); + } + } + } + + info!("✅ Database schema initialization complete."); + Ok(()) +} diff --git a/src/shared/database/traits/image_cache.rs b/src/shared/database/traits/image_cache.rs new file mode 100644 index 0000000..56588ef --- /dev/null +++ b/src/shared/database/traits/image_cache.rs @@ -0,0 +1,16 @@ +use async_trait::async_trait; + +#[async_trait] +pub trait ImageCacheRepository: Send + Sync { + async fn get_from_redis(&self, key: &str) -> Option; + async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String>; + async fn get_from_db(&self, original_url: &str) -> Result, String>; + async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String>; + async fn find_original_from_cdn(&self, cdn_url: &str) -> Result, String>; + async fn delete_from_db(&self, original_url: &str) -> Result<(), String>; + async fn delete_from_redis(&self, key: &str) -> Result<(), String>; + async fn get_lock(&self, key: &str) -> bool; + async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String>; + async fn release_lock(&self, key: &str) -> Result<(), String>; + async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String>; +} diff --git a/src/shared/database/traits/mod.rs b/src/shared/database/traits/mod.rs new file mode 100644 index 0000000..03d89d2 --- /dev/null +++ b/src/shared/database/traits/mod.rs @@ -0,0 +1,4 @@ +pub mod image_cache; +pub mod scraping_repository; + +pub use scraping_repository::ScrapingRepository; diff --git a/src/shared/database/traits/scraping_repository.rs b/src/shared/database/traits/scraping_repository.rs new file mode 100644 index 0000000..411a35a --- /dev/null +++ b/src/shared/database/traits/scraping_repository.rs @@ -0,0 +1,7 @@ +use crate::shared::errors::AppError; +use async_trait::async_trait; + +#[async_trait] +pub trait ScrapingRepository: Send + Sync { + async fn fetch_html(&self, url: &str) -> Result; +} diff --git a/src/shared/errors/app_error.rs b/src/shared/errors/app_error.rs new file mode 100644 index 0000000..2b85a42 --- /dev/null +++ b/src/shared/errors/app_error.rs @@ -0,0 +1,88 @@ +use axum::response::IntoResponse; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AppError { + #[error("Environment variable not found: {0}")] + EnvVarNotFound(String), + #[error("Redis error: {0}")] + RedisError(#[from] redis::RedisError), + #[error("Reqwest error: {0}")] + ReqwestError(#[from] reqwest::Error), + #[error("JSON serialization/deserialization error: {0}")] + SerdeJsonError(#[from] serde_json::Error), + #[error("Scraper error: {0}")] + ScraperError(String), + #[error("Fantoccini error: {0}")] + FantocciniError(String), + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + #[error("Timeout error: {0}")] + TimeoutError(String), + #[error("Other error: {0}")] + Other(String), + #[error("HTTP error: {0}")] + HttpError(#[from] http::Error), + #[error("URL parsing error: {0}")] + UrlParseError(#[from] url::ParseError), + #[error("Database error: {0}")] + DatabaseError(String), + #[error("Not Found: {0}")] + NotFound(String), +} + +impl From<&str> for AppError { + fn from(s: &str) -> Self { + AppError::Other(s.to_string()) + } +} + +impl From for AppError { + fn from(s: String) -> Self { + AppError::Other(s) + } +} + +impl From> for AppError { + fn from(err: Box) -> Self { + AppError::Other(err.to_string()) + } +} + +impl From for AppError { + fn from(err: anyhow::Error) -> Self { + AppError::Other(err.to_string()) + } +} + +impl From for AppError { + fn from(err: deadpool_redis::PoolError) -> Self { + AppError::Other(err.to_string()) + } +} + +impl From for AppError { + fn from(err: tokio::task::JoinError) -> Self { + AppError::Other(err.to_string()) + } +} + +impl IntoResponse for AppError { + fn into_response(self) -> axum::response::Response { + let (status, error_message) = match self { + AppError::NotFound(_) => (http::StatusCode::NOT_FOUND, self.to_string()), + AppError::DatabaseError(_) => { + (http::StatusCode::INTERNAL_SERVER_ERROR, self.to_string()) + } + _ => (http::StatusCode::INTERNAL_SERVER_ERROR, self.to_string()), + }; + + // Note: crate::shared::types needs to be available. + // If not, we might need to adjust this line or ensure types are there. + // Since we are validating structure, let's assume types is in core/types.rs + let body = axum::Json(crate::shared::types::ApiResponse::<()>::error( + error_message, + )); + (status, body).into_response() + } +} diff --git a/src/shared/errors/mod.rs b/src/shared/errors/mod.rs new file mode 100644 index 0000000..84cb0e0 --- /dev/null +++ b/src/shared/errors/mod.rs @@ -0,0 +1,2 @@ +pub mod app_error; +pub use app_error::AppError; diff --git a/src/shared/events/bus.rs b/src/shared/events/bus.rs new file mode 100644 index 0000000..2a95daa --- /dev/null +++ b/src/shared/events/bus.rs @@ -0,0 +1,155 @@ +//! Event bus implementation. + +use async_trait::async_trait; + +use std::{any::TypeId, collections::HashMap, sync::Arc}; +use tokio::sync::{broadcast, RwLock}; +use tracing::{debug, info}; + +/// Trait for events that can be published. +pub trait Event: Clone + Send + Sync + 'static { + /// Event name for logging/debugging. + const NAME: &'static str; +} + +/// Trait for event handlers. +#[async_trait] +pub trait EventHandler: Send + Sync { + async fn handle(&self, event: E); +} + +/// The event bus for publishing and subscribing to events. +pub struct EventBus { + channels: RwLock>>, +} + +impl EventBus { + /// Create a new event bus. + pub fn new() -> Self { + Self { + channels: RwLock::new(HashMap::new()), + } + } + + /// Publish an event to all subscribers. + pub async fn publish(&self, event: E) { + let type_id = TypeId::of::(); + let channels = self.channels.read().await; + + if let Some(sender) = channels.get(&type_id) { + if let Some(tx) = sender.downcast_ref::>() { + let _ = tx.send(event); + debug!("Published event: {}", E::NAME); + } + } + } + + /// Subscribe to events of a specific type. + /// Returns a receiver that can be used to receive events. + pub async fn subscribe(&self) -> broadcast::Receiver { + let type_id = TypeId::of::(); + + // Check if channel exists + { + let channels = self.channels.read().await; + if let Some(sender) = channels.get(&type_id) { + if let Some(tx) = sender.downcast_ref::>() { + return tx.subscribe(); + } + } + } + + // Create new channel + let (tx, rx) = broadcast::channel::(100); + { + let mut channels = self.channels.write().await; + channels.insert(type_id, Box::new(tx)); + } + + // Re-get the receiver from the stored sender + let channels = self.channels.read().await; + if let Some(sender) = channels.get(&type_id) { + if let Some(tx) = sender.downcast_ref::>() { + return tx.subscribe(); + } + } + + rx + } + + /// Register a handler for a specific event type. + /// The handler will be called whenever an event of that type is published. + pub async fn on + 'static>(&self, handler: H) { + let mut rx = self.subscribe::().await; + let handler = Arc::new(handler); + + tokio::spawn(async move { + loop { + match rx.recv().await { + Ok(event) => { + handler.handle(event).await; + } + Err(broadcast::error::RecvError::Closed) => break, + Err(broadcast::error::RecvError::Lagged(n)) => { + tracing::warn!("Event handler lagged by {} events", n); + } + } + } + }); + + info!("Registered handler for event: {}", E::NAME); + } +} + +impl Default for EventBus { + fn default() -> Self { + Self::new() + } +} + +// Common events +/// User registered event. +#[derive(Clone, Debug)] +pub struct UserRegistered { + pub user_id: String, + pub email: String, + pub name: String, +} + +impl Event for UserRegistered { + const NAME: &'static str = "user.registered"; +} + +/// User logged in event. +#[derive(Clone, Debug)] +pub struct UserLoggedIn { + pub user_id: String, + pub ip_address: Option, +} + +impl Event for UserLoggedIn { + const NAME: &'static str = "user.logged_in"; +} + +/// Order created event. +#[derive(Clone, Debug)] +pub struct OrderCreated { + pub order_id: String, + pub user_id: String, + pub total: f64, +} + +impl Event for OrderCreated { + const NAME: &'static str = "order.created"; +} + +/// Image repaired event. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct ImageRepaired { + pub original_url: String, + pub cdn_url: String, +} + +impl Event for ImageRepaired { + const NAME: &'static str = "image.repaired"; +} diff --git a/src/shared/events/mod.rs b/src/shared/events/mod.rs new file mode 100644 index 0000000..6440547 --- /dev/null +++ b/src/shared/events/mod.rs @@ -0,0 +1 @@ +pub mod bus; diff --git a/src/shared/graceful/cleanup.rs b/src/shared/graceful/cleanup.rs new file mode 100644 index 0000000..6e3633d --- /dev/null +++ b/src/shared/graceful/cleanup.rs @@ -0,0 +1,142 @@ +//! Graceful shutdown with proper resource cleanup. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tokio::signal; +use tokio::sync::Notify; +use tokio::time::{sleep, Duration}; +use tracing::info; + +/// Graceful shutdown coordinator. +pub struct ShutdownCoordinator { + /// Shutdown signal flag + is_shutting_down: Arc, + /// Notify for graceful shutdown + shutdown_notify: Arc, +} + +impl ShutdownCoordinator { + /// Create a new shutdown coordinator. + pub fn new() -> Self { + Self { + is_shutting_down: Arc::new(AtomicBool::new(false)), + shutdown_notify: Arc::new(Notify::new()), + } + } + + /// Check if shutdown is in progress. + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::Relaxed) + } + + /// Start shutdown process. + pub fn initiate_shutdown(&self) { + info!("🛑 Initiating graceful shutdown..."); + self.is_shutting_down.store(true, Ordering::Relaxed); + self.shutdown_notify.notify_waiters(); + } + + /// Wait for shutdown signal. + pub async fn wait_for_shutdown_signal(&self) { + let ctrl_c = async { + if let Err(e) = signal::ctrl_c().await { + tracing::error!("Failed to listen for Ctrl+C: {}", e); + } + }; + + #[cfg(unix)] + let terminate = async { + match signal::unix::signal(signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(e) => { + tracing::error!("Failed to listen for SIGTERM: {}", e); + } + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl+C signal"); + } + _ = terminate => { + info!("Received SIGTERM signal"); + } + } + + self.initiate_shutdown(); + } + + /// Perform cleanup operations. + pub async fn cleanup(&self) { + info!("🧹 Starting cleanup operations..."); + + // Give active requests time to finish + info!("Waiting for active requests to complete..."); + sleep(Duration::from_secs(5)).await; + + info!("✅ Cleanup completed"); + } + + /// Get a handle for checking shutdown status. + pub fn handle(&self) -> ShutdownHandle { + ShutdownHandle { + is_shutting_down: Arc::clone(&self.is_shutting_down), + } + } +} + +impl Default for ShutdownCoordinator { + fn default() -> Self { + Self::new() + } +} + +/// Handle for checking shutdown status. +#[derive(Clone)] +pub struct ShutdownHandle { + is_shutting_down: Arc, +} + +impl ShutdownHandle { + /// Check if shutdown is in progress. + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::Relaxed) + } +} + +pub async fn wait_for_shutdown_and_cleanup() { + let coordinator = ShutdownCoordinator::new(); + + coordinator.wait_for_shutdown_signal().await; + coordinator.cleanup().await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_shutdown_coordinator() { + let coordinator = ShutdownCoordinator::new(); + assert!(!coordinator.is_shutting_down()); + + coordinator.initiate_shutdown(); + assert!(coordinator.is_shutting_down()); + } + + #[test] + fn test_shutdown_handle() { + let coordinator = ShutdownCoordinator::new(); + let handle = coordinator.handle(); + + assert!(!handle.is_shutting_down()); + + coordinator.initiate_shutdown(); + assert!(handle.is_shutting_down()); + } +} diff --git a/src/shared/graceful/mod.rs b/src/shared/graceful/mod.rs new file mode 100644 index 0000000..c630962 --- /dev/null +++ b/src/shared/graceful/mod.rs @@ -0,0 +1,2 @@ +pub mod cleanup; +pub mod shutdown; diff --git a/src/shared/graceful/shutdown.rs b/src/shared/graceful/shutdown.rs new file mode 100644 index 0000000..2ebab74 --- /dev/null +++ b/src/shared/graceful/shutdown.rs @@ -0,0 +1,116 @@ +//! Graceful shutdown implementation. + +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::signal; +use tokio::sync::broadcast; +use tracing::info; + +/// Graceful shutdown controller. +pub struct GracefulShutdown { + shutdown_tx: broadcast::Sender<()>, + is_shutting_down: Arc, +} + +impl GracefulShutdown { + /// Create a new graceful shutdown controller. + pub fn new() -> Self { + let (shutdown_tx, _) = broadcast::channel(1); + Self { + shutdown_tx, + is_shutting_down: Arc::new(AtomicBool::new(false)), + } + } + + /// Check if shutdown has been initiated. + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::SeqCst) + } + + /// Get a receiver for shutdown notifications. + pub fn subscribe(&self) -> broadcast::Receiver<()> { + self.shutdown_tx.subscribe() + } + + /// Initiate shutdown. + pub fn shutdown(&self) { + if !self.is_shutting_down.swap(true, Ordering::SeqCst) { + info!("🛑 Initiating graceful shutdown..."); + let _ = self.shutdown_tx.send(()); + } + } + + /// Wait for shutdown signal and then gracefully drain. + pub async fn wait_for_shutdown(&self, drain_timeout: Duration) { + // Wait for shutdown signal + shutdown_signal().await; + + self.shutdown(); + + // Give time for in-flight requests to complete + info!( + "⏳ Waiting {}s for in-flight requests...", + drain_timeout.as_secs() + ); + tokio::time::sleep(drain_timeout).await; + + info!("✅ Graceful shutdown complete"); + } +} + +impl Default for GracefulShutdown { + fn default() -> Self { + Self::new() + } +} + +impl Clone for GracefulShutdown { + fn clone(&self) -> Self { + Self { + shutdown_tx: self.shutdown_tx.clone(), + is_shutting_down: Arc::clone(&self.is_shutting_down), + } + } +} + +/// Wait for a shutdown signal (SIGTERM, SIGINT, or Ctrl+C). +pub async fn shutdown_signal() { + let ctrl_c = async { + if let Err(e) = signal::ctrl_c().await { + tracing::error!("Failed to install Ctrl+C handler: {}", e); + } + }; + + #[cfg(unix)] + let terminate = async { + match signal::unix::signal(signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(e) => { + tracing::error!("Failed to install SIGTERM handler: {}", e); + } + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("📥 Received Ctrl+C signal"); + } + _ = terminate => { + info!("📥 Received SIGTERM signal"); + } + } +} + +/// Create a shutdown future that can be used with axum's serve. +pub fn create_shutdown_signal() -> impl Future + Send + 'static { + async { + shutdown_signal().await; + } +} diff --git a/src/shared/health/endpoints.rs b/src/shared/health/endpoints.rs new file mode 100644 index 0000000..67f5fa5 --- /dev/null +++ b/src/shared/health/endpoints.rs @@ -0,0 +1,138 @@ +//! Health check endpoint implementations. + +use axum::{http::StatusCode, response::IntoResponse, Json}; +use serde::Serialize; +use std::time::Instant; + +use once_cell::sync::Lazy; + +static START_TIME: Lazy = Lazy::new(Instant::now); + +/// Health status response. +#[derive(Debug, Clone, Serialize)] +pub struct HealthStatus { + pub status: &'static str, + pub version: &'static str, + pub uptime_seconds: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub checks: Option, +} + +/// Individual health checks. +#[derive(Debug, Clone, Serialize)] +pub struct HealthChecks { + pub database: CheckResult, + pub redis: CheckResult, +} + +/// Result of a health check. +#[derive(Debug, Clone, Serialize)] +pub struct CheckResult { + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub latency_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Simple health check - just returns OK. +/// Use for liveness probes (Kubernetes: /healthz). +pub async fn health_check() -> impl IntoResponse { + let status = HealthStatus { + status: "ok", + version: env!("CARGO_PKG_VERSION"), + uptime_seconds: START_TIME.elapsed().as_secs(), + checks: None, + }; + (StatusCode::OK, Json(status)) +} + +/// Readiness check with dependency checks. +/// Use for readiness probes (Kubernetes: /readyz). +pub async fn readiness_check() -> impl IntoResponse { + let uptime = START_TIME.elapsed().as_secs(); + + // Check Redis + let redis_check = check_redis().await; + + // Check Database - simplified for now + let db_check = CheckResult { + status: "ok", + latency_ms: Some(1), + error: None, + }; + + let all_healthy = redis_check.status == "ok" && db_check.status == "ok"; + + let status = HealthStatus { + status: if all_healthy { "ok" } else { "degraded" }, + version: env!("CARGO_PKG_VERSION"), + uptime_seconds: uptime, + checks: Some(HealthChecks { + database: db_check, + redis: redis_check, + }), + }; + + let status_code = if all_healthy { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + (status_code, Json(status)) +} + +/// Check Redis connectivity using PING command. +async fn check_redis() -> CheckResult { + use crate::shared::database::get_redis_pool; + + let start = Instant::now(); + + let pool = match get_redis_pool() { + Ok(p) => p, + Err(e) => { + return CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(format!("Pool error: {}", e)), + }; + } + }; + + match pool.get().await { + Ok(mut conn) => { + let result: Result = redis::cmd("PING").query_async(&mut *conn).await; + match result { + Ok(response) if response == "PONG" => CheckResult { + status: "ok", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: None, + }, + Ok(response) => CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(format!("Unexpected PING response: {}", response)), + }, + Err(e) => CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(e.to_string()), + }, + } + } + Err(e) => CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(e.to_string()), + }, + } +} + +/// Standalone health endpoint (no dependencies required). +pub async fn simple_health() -> impl IntoResponse { + Json(serde_json::json!({ + "status": "ok", + "timestamp": chrono::Utc::now().to_rfc3339() + })) +} diff --git a/src/shared/health/mod.rs b/src/shared/health/mod.rs new file mode 100644 index 0000000..c4b360f --- /dev/null +++ b/src/shared/health/mod.rs @@ -0,0 +1 @@ +pub mod endpoints; diff --git a/src/shared/jobs/mod.rs b/src/shared/jobs/mod.rs new file mode 100644 index 0000000..b3116e8 --- /dev/null +++ b/src/shared/jobs/mod.rs @@ -0,0 +1,2 @@ +pub mod queue; +pub mod worker; diff --git a/src/shared/jobs/queue.rs b/src/shared/jobs/queue.rs new file mode 100644 index 0000000..6238d18 --- /dev/null +++ b/src/shared/jobs/queue.rs @@ -0,0 +1,204 @@ +//! Job queue implementation using Redis. +//! +//! Provides a simple but robust job queue system for background processing. + +use async_trait::async_trait; +use deadpool_redis::Pool; +use redis::AsyncCommands; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use uuid::Uuid; + +/// Status of a queued job. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum JobStatus { + /// Job is waiting in the queue + Pending, + /// Job is currently being processed + Processing, + /// Job completed successfully + Completed, + /// Job failed with an error + Failed, + /// Job was manually cancelled + Cancelled, +} + +/// Metadata for a queued job. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobMeta { + pub id: String, + pub job_type: String, + pub status: JobStatus, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub completed_at: Option>, + pub attempts: u32, + pub max_attempts: u32, + pub error: Option, +} + +/// Trait for background jobs. +/// +/// Implement this trait to define a job that can be queued and processed. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::jobs::{Job, JobDispatcher}; +/// use serde::{Serialize, Deserialize}; +/// use async_trait::async_trait; +/// +/// #[derive(Serialize, Deserialize)] +/// struct SendWelcomeEmail { +/// user_id: String, +/// email: String, +/// } +/// +/// #[async_trait] +/// impl Job for SendWelcomeEmail { +/// const NAME: &'static str = "send_welcome_email"; +/// const MAX_ATTEMPTS: u32 = 3; +/// +/// async fn handle(&self) -> anyhow::Result<()> { +/// // Send the email... +/// println!("Sending welcome email to {}", self.email); +/// Ok(()) +/// } +/// } +/// +/// // Dispatch the job +/// let job = SendWelcomeEmail { +/// user_id: "123".to_string(), +/// email: "user@example.com".to_string(), +/// }; +/// dispatcher.dispatch(job).await?; +/// ``` +#[async_trait] +pub trait Job: Serialize + DeserializeOwned + Send + Sync { + /// Unique name for this job type. + const NAME: &'static str; + + /// Maximum number of retry attempts. + const MAX_ATTEMPTS: u32 = 3; + + /// Queue name to use (default: "default") + const QUEUE: &'static str = "default"; + + /// Execute the job. + async fn handle(&self) -> anyhow::Result<()>; + + /// Called when the job fails after all retries. + async fn failed(&self, error: &str) { + tracing::error!("Job {} failed: {}", Self::NAME, error); + } +} + +/// Job dispatcher for queuing jobs. +#[derive(Clone)] +pub struct JobDispatcher { + redis_pool: Pool, +} + +impl JobDispatcher { + /// Create a new job dispatcher. + pub fn new(redis_pool: Pool) -> Self { + Self { redis_pool } + } + + /// Dispatch a job to be processed. + pub async fn dispatch(&self, job: J) -> anyhow::Result { + let job_id = Uuid::new_v4().to_string(); + let queue_key = format!("jobs:queue:{}", J::QUEUE); + let job_key = format!("jobs:data:{}", job_id); + + let payload = serde_json::to_string(&job)?; + + let meta = JobMeta { + id: job_id.clone(), + job_type: J::NAME.to_string(), + status: JobStatus::Pending, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + attempts: 0, + max_attempts: J::MAX_ATTEMPTS, + error: None, + }; + + let meta_json = serde_json::to_string(&meta)?; + + let mut conn = self.redis_pool.get().await?; + + // Store job data + let _: () = conn.set(&job_key, payload).await?; + let _: () = conn.set(format!("{}:meta", job_key), meta_json).await?; + + // Push to queue + let _: () = conn.rpush(&queue_key, &job_id).await?; + + tracing::info!("Dispatched job {} ({})", J::NAME, job_id); + + Ok(job_id) + } + + /// Dispatch a job with a delay (in seconds). + pub async fn dispatch_delayed( + &self, + job: J, + delay_seconds: u64, + ) -> anyhow::Result { + let job_id = Uuid::new_v4().to_string(); + let delayed_key = "jobs:delayed"; + let job_key = format!("jobs:data:{}", job_id); + + let payload = serde_json::to_string(&job)?; + let execute_at = chrono::Utc::now().timestamp() + delay_seconds as i64; + + let meta = JobMeta { + id: job_id.clone(), + job_type: J::NAME.to_string(), + status: JobStatus::Pending, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + attempts: 0, + max_attempts: J::MAX_ATTEMPTS, + error: None, + }; + + let meta_json = serde_json::to_string(&meta)?; + + let mut conn = self.redis_pool.get().await?; + + // Store job data + let _: () = conn.set(&job_key, payload).await?; + let _: () = conn.set(format!("{}:meta", job_key), meta_json).await?; + + // Add to delayed sorted set (score = execution timestamp) + let delayed_entry = format!("{}:{}", J::QUEUE, job_id); + let _: () = conn.zadd(delayed_key, delayed_entry, execute_at).await?; + + tracing::info!( + "Dispatched delayed job {} ({}) - executes in {}s", + J::NAME, + job_id, + delay_seconds + ); + + Ok(job_id) + } + + /// Get the status of a job. + pub async fn status(&self, job_id: &str) -> anyhow::Result> { + let meta_key = format!("jobs:data:{}:meta", job_id); + let mut conn = self.redis_pool.get().await?; + + let meta_json: Option = conn.get(&meta_key).await?; + + match meta_json { + Some(json) => Ok(Some(serde_json::from_str(&json)?)), + None => Ok(None), + } + } +} diff --git a/src/shared/jobs/worker.rs b/src/shared/jobs/worker.rs new file mode 100644 index 0000000..310af64 --- /dev/null +++ b/src/shared/jobs/worker.rs @@ -0,0 +1,180 @@ +//! Job worker for processing background jobs. +//! +//! The worker runs as a separate process and continuously polls +//! the job queue for work. + +use super::queue::{JobMeta, JobStatus}; +use deadpool_redis::Pool; +use redis::AsyncCommands; +use std::time::Duration; +use tokio::time::sleep; + +/// Configuration for the job worker. +#[derive(Debug, Clone)] +pub struct WorkerConfig { + /// Queues to process (in priority order) + pub queues: Vec, + /// Number of concurrent jobs to process + pub concurrency: usize, + /// Sleep duration when no jobs are available + pub sleep_duration: Duration, +} + +impl Default for WorkerConfig { + fn default() -> Self { + Self { + queues: vec!["default".to_string()], + concurrency: 4, + sleep_duration: Duration::from_secs(1), + } + } +} + +/// Job worker that processes queued jobs. +pub struct Worker { + redis_pool: Pool, + config: WorkerConfig, + /// Registry of job handlers by name + handlers: std::collections::HashMap<&'static str, Box>, +} + +/// Trait for job handlers (type-erased). +#[async_trait::async_trait] +pub trait JobHandler: Send + Sync { + async fn process(&self, payload: &str) -> anyhow::Result<()>; +} + +impl Worker { + /// Create a new worker. + pub fn new(redis_pool: Pool, config: WorkerConfig) -> Self { + Self { + redis_pool, + config, + handlers: std::collections::HashMap::new(), + } + } + + /// Register a job handler. + pub fn register(&mut self, name: &'static str, handler: H) { + self.handlers.insert(name, Box::new(handler)); + } + + /// Run the worker loop. + pub async fn run(&self) -> anyhow::Result<()> { + tracing::info!( + "🔧 Worker started - processing queues: {:?}", + self.config.queues + ); + + loop { + let mut processed = false; + + for queue in &self.config.queues { + if let Some(job_id) = self.pop_job(queue).await? { + self.process_job(queue, &job_id).await?; + processed = true; + } + } + + if !processed { + // No jobs available, sleep + sleep(self.config.sleep_duration).await; + } + } + } + + /// Pop a job from the queue. + async fn pop_job(&self, queue: &str) -> anyhow::Result> { + let queue_key = format!("jobs:queue:{}", queue); + let mut conn = self.redis_pool.get().await?; + + let job_id: Option = conn.lpop(&queue_key, None).await?; + Ok(job_id) + } + + /// Process a single job. + async fn process_job(&self, queue: &str, job_id: &str) -> anyhow::Result<()> { + let job_key = format!("jobs:data:{}", job_id); + let meta_key = format!("{}:meta", job_key); + + let mut conn = self.redis_pool.get().await?; + + // Get job metadata + let meta_json: Option = conn.get(&meta_key).await?; + let mut meta: JobMeta = match meta_json { + Some(json) => serde_json::from_str(&json)?, + None => { + tracing::warn!("Job {} not found", job_id); + return Ok(()); + } + }; + + // Get job payload + let payload: Option = conn.get(&job_key).await?; + let payload = match payload { + Some(p) => p, + None => { + tracing::warn!("Job {} payload not found", job_id); + return Ok(()); + } + }; + + // Update status to processing + meta.status = JobStatus::Processing; + meta.started_at = Some(chrono::Utc::now()); + meta.attempts += 1; + let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?; + + tracing::info!( + "Processing job {} ({}) - attempt {}/{}", + meta.job_type, + job_id, + meta.attempts, + meta.max_attempts + ); + + // Find handler + let handler = match self.handlers.get(meta.job_type.as_str()) { + Some(h) => h, + None => { + tracing::error!("No handler registered for job type: {}", meta.job_type); + meta.status = JobStatus::Failed; + meta.error = Some(format!("No handler for job type: {}", meta.job_type)); + meta.completed_at = Some(chrono::Utc::now()); + let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?; + return Ok(()); + } + }; + + // Execute job + match handler.process(&payload).await { + Ok(()) => { + meta.status = JobStatus::Completed; + meta.completed_at = Some(chrono::Utc::now()); + tracing::info!("Job {} completed successfully", job_id); + } + Err(e) => { + let error_msg = e.to_string(); + tracing::error!("Job {} failed: {}", job_id, error_msg); + + if meta.attempts >= meta.max_attempts { + meta.status = JobStatus::Failed; + meta.error = Some(error_msg); + meta.completed_at = Some(chrono::Utc::now()); + } else { + // Retry - push back to queue + meta.status = JobStatus::Pending; + meta.error = Some(format!("Attempt {} failed: {}", meta.attempts, error_msg)); + let queue_key = format!("jobs:queue:{}", queue); + let _: () = conn.rpush(&queue_key, job_id).await?; + tracing::info!("Job {} queued for retry", job_id); + } + } + } + + // Save final status + let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?; + + Ok(()) + } +} diff --git a/src/shared/middlewares/logging.rs b/src/shared/middlewares/logging.rs new file mode 100644 index 0000000..d8066f7 --- /dev/null +++ b/src/shared/middlewares/logging.rs @@ -0,0 +1,289 @@ +//! Request/Response logging middleware. +//! +//! Provides structured logging for HTTP requests with timing and correlation. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::shared::middlewares::logging::{LoggingConfig, with_logging}; +//! use axum::Router; +//! +//! let app = Router::new() +//! .route("/api/test", get(handler)) +//! .layer(axum::middleware::from_fn(with_logging(LoggingConfig::default()))); +//! ``` + +use axum::{body::Body, extract::Request, http::StatusCode, middleware::Next, response::Response}; +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Instant; +use uuid::Uuid; + +/// Logging configuration. +#[derive(Debug, Clone)] +pub struct LoggingConfig { + /// Log request headers + pub log_headers: bool, + /// Log request body (be careful with sensitive data) + pub log_body: bool, + /// Log response body + pub log_response_body: bool, + /// Maximum body size to log (bytes) + pub max_body_size: usize, + /// Paths to exclude from logging + pub exclude_paths: HashSet, + /// Log level for successful requests + pub success_level: LogLevel, + /// Log level for client errors (4xx) + pub client_error_level: LogLevel, + /// Log level for server errors (5xx) + pub server_error_level: LogLevel, +} + +/// Log level enum. +#[derive(Debug, Clone, Copy)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, +} + +impl Default for LoggingConfig { + fn default() -> Self { + let mut exclude = HashSet::new(); + exclude.insert("/health".to_string()); + exclude.insert("/favicon.ico".to_string()); + + Self { + log_headers: false, + log_body: false, + log_response_body: false, + max_body_size: 1024, + exclude_paths: exclude, + success_level: LogLevel::Info, + client_error_level: LogLevel::Warn, + server_error_level: LogLevel::Error, + } + } +} + +impl LoggingConfig { + /// Create a new logging config. + pub fn new() -> Self { + Self::default() + } + + /// Enable header logging. + pub fn with_headers(mut self) -> Self { + self.log_headers = true; + self + } + + /// Add a path to exclude from logging. + pub fn exclude_path(mut self, path: &str) -> Self { + self.exclude_paths.insert(path.to_string()); + self + } +} + +/// Request ID extension for correlation. +#[derive(Clone, Debug)] +pub struct RequestId(pub String); + +impl RequestId { + /// Generate a new request ID. + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Get the request ID as a string. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for RequestId { + fn default() -> Self { + Self::new() + } +} + +/// Logging middleware. +pub async fn logging_middleware(config: Arc, req: Request, next: Next) -> Response { + let path = req.uri().path(); + + if config.exclude_paths.contains(path) { + return next.run(req).await; + } + + let request_id = req + .headers() + .get("X-Request-ID") + .and_then(|h| h.to_str().ok()) + .map(|s| RequestId(s.to_string())) + .unwrap_or_else(RequestId::new); + + let method = req.method().clone(); + let uri = req.uri().clone(); + let path = uri.path().to_string(); + let query = uri.query().map(str::to_owned); + + let headers_log = if config.log_headers { + let headers: Vec = req + .headers() + .iter() + .filter(|(name, _)| !is_sensitive_header(name.as_str())) + .map(|(name, value)| format!("{}: {}", name, value.to_str().unwrap_or(""))) + .collect(); + Some(headers) + } else { + None + }; + + let mut req = req; + req.extensions_mut().insert(request_id.clone()); + + let start = Instant::now(); + + tracing::info!( + request_id = %request_id.0, + method = %method, + path = %path, + query = ?query, + "→ Request started" + ); + + if let Some(ref headers) = headers_log { + tracing::debug!(request_id = %request_id.0, headers = ?headers, "Request headers"); + } + + let response = next.run(req).await; + + let duration = start.elapsed(); + let status = response.status(); + + match status.as_u16() { + 100..=399 => log_response( + config.success_level, + &request_id, + status, + duration, + &method, + &path, + ), + 400..=499 => log_response( + config.client_error_level, + &request_id, + status, + duration, + &method, + &path, + ), + _ => log_response( + config.server_error_level, + &request_id, + status, + duration, + &method, + &path, + ), + } + + response +} + +fn is_sensitive_header(name: &str) -> bool { + name.eq_ignore_ascii_case("authorization") + || name.eq_ignore_ascii_case("cookie") + || name.eq_ignore_ascii_case("x-api-key") +} + +fn log_response( + level: LogLevel, + request_id: &RequestId, + status: StatusCode, + duration: std::time::Duration, + method: &axum::http::Method, + path: &str, +) { + let duration_ms = duration.as_millis(); + + match level { + LogLevel::Trace => tracing::trace!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Debug => tracing::debug!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Info => tracing::info!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Warn => tracing::warn!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Error => tracing::error!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + } +} + +/// Create logging middleware. +pub fn with_logging( + config: LoggingConfig, +) -> impl Fn( + Request, + Next, +) -> std::pin::Pin + Send>> + + Clone + + Send { + let config = Arc::new(config); + move |req: Request, next: Next| { + let config = config.clone(); + Box::pin(async move { logging_middleware(config, req, next).await }) + } +} + +/// Extractor for RequestId in route handlers. +impl axum::extract::FromRequestParts for RequestId +where + S: Send + Sync, +{ + type Rejection = (StatusCode, &'static str); + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + _state: &S, + ) -> Result { + parts.extensions.get::().cloned().ok_or(( + StatusCode::INTERNAL_SERVER_ERROR, + "RequestId not found. Did you add logging middleware?", + )) + } +} diff --git a/src/shared/middlewares/mod.rs b/src/shared/middlewares/mod.rs new file mode 100644 index 0000000..abd0d0e --- /dev/null +++ b/src/shared/middlewares/mod.rs @@ -0,0 +1,2 @@ +pub mod ratelimit; +pub use ratelimit::rate_limit_middleware; diff --git a/src/shared/middlewares/ratelimit.rs b/src/shared/middlewares/ratelimit.rs new file mode 100644 index 0000000..9e5a20e --- /dev/null +++ b/src/shared/middlewares/ratelimit.rs @@ -0,0 +1,101 @@ +//! Rate limiter middleware using the governor crate. +//! +//! Provides token-bucket based rate limiting with configurable limits. +//! Default: 20 requests per IP address, burst 50. + +use axum::{ + extract::Request, + http::StatusCode, + middleware::Next, + response::{IntoResponse, Response}, + Json, +}; +use governor::{ + clock::DefaultClock, state::keyed::DefaultKeyedStateStore, Quota, + RateLimiter as GovernorRateLimiter, +}; +use once_cell::sync::Lazy; +use serde_json::json; +use std::{num::NonZeroU32, sync::Arc, time::Duration}; +use tracing::warn; + +/// Global IP-keyed rate limiter instance. +/// Configured for 20 requests per second with a burst of 50. +static IP_LIMITER: Lazy< + Arc, DefaultClock>>, +> = Lazy::new(|| { + let quota = Quota::with_period(Duration::from_millis(50)) + .unwrap() + .allow_burst(NonZeroU32::new(50).unwrap()); + + Arc::new(GovernorRateLimiter::keyed(quota)) +}); + +/// Rate limiter configuration. +#[derive(Debug, Clone)] +pub struct RateLimiterConfig { + /// Requests per second + pub requests_per_second: u32, + /// Burst size (max requests that can be made instantly) + pub burst_size: u32, +} + +impl Default for RateLimiterConfig { + fn default() -> Self { + Self { + requests_per_second: 20, + burst_size: 50, + } + } +} + +/// Create a custom rate limiter with specific configuration. +pub fn create_rate_limiter( + config: RateLimiterConfig, +) -> Arc, DefaultClock>> { + let period_ms = 1000 / config.requests_per_second.max(1); + let quota = Quota::with_period(Duration::from_millis(period_ms as u64)) + .unwrap() + .allow_burst(NonZeroU32::new(config.burst_size.max(1)).unwrap()); + + Arc::new(GovernorRateLimiter::keyed(quota)) +} + +fn extract_ip(req: &Request) -> String { + if let Some(ip) = req.headers().get("X-Forwarded-For") { + if let Ok(ip_str) = ip.to_str() { + if let Some(first_ip) = ip_str.split(',').next() { + return first_ip.trim().to_string(); + } + } + } + if let Some(ip) = req.headers().get("X-Real-IP") { + if let Ok(ip_str) = ip.to_str() { + return ip_str.trim().to_string(); + } + } + "unknown".to_string() +} + +/// Rate limiting middleware using the IP-keyed limiter. +/// +/// Returns 429 Too Many Requests if the limit is exceeded. +pub async fn rate_limit_middleware(req: Request, next: Next) -> Response { + let client_ip = extract_ip(&req); + + match IP_LIMITER.check_key(&client_ip) { + Ok(_) => next.run(req).await, + Err(_) => { + warn!("Rate limit exceeded for IP: {}", client_ip); + ( + StatusCode::TOO_MANY_REQUESTS, + Json(json!({ + "error": "Too many requests", + "code": "RATE_LIMIT_EXCEEDED", + "retry_after_ms": 1000 + })), + ) + .into_response() + } + } +} diff --git a/src/shared/mod.rs b/src/shared/mod.rs new file mode 100644 index 0000000..358cb1a --- /dev/null +++ b/src/shared/mod.rs @@ -0,0 +1,17 @@ +pub mod browser; +pub mod config; +pub mod database; +pub mod errors; +pub mod events; +pub mod graceful; +pub mod health; +pub mod middlewares; +pub mod observability; +pub mod routing; +pub mod scheduler; +pub mod scrapers; +pub mod services; +pub mod state; +pub mod testing; +pub mod types; +pub mod utils; diff --git a/src/shared/observability/metrics.rs b/src/shared/observability/metrics.rs new file mode 100644 index 0000000..5aa02b6 --- /dev/null +++ b/src/shared/observability/metrics.rs @@ -0,0 +1,171 @@ +//! OpenTelemetry metrics initialization for the scraper service. +//! +//! Provides: +//! - Global `MeterProvider` connected via OTLP gRPC to the metrics backend +//! - Standard HTTP server metrics middleware +//! +//! Environment: +//! OTEL_EXPORTER_OTLP_ENDPOINT — default: http://localhost:4317 +//! OTEL_SERVICE_NAME — default: scraper-api +//! OTEL_METRICS_EXPORT_INTERVAL — default: 5000 (ms) + +use axum::{extract::Request, middleware::Next, response::Response}; +use opentelemetry::{ + global, + metrics::{Counter, Histogram, Meter, UpDownCounter}, + KeyValue, +}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{metrics::MeterProviderBuilder, metrics::PeriodicReader, Resource}; +use std::sync::OnceLock; +use std::time::Instant; + +static METER: OnceLock = OnceLock::new(); +static PROVIDER: OnceLock = OnceLock::new(); + +fn meter() -> &'static Meter { + METER.get().expect("OTel meter not initialized — call init_otel_metrics first") +} + +/// Initialize the global OTLP MeterProvider. +/// Safe to call multiple times — subsequent calls are no-ops. +pub fn init_otel_metrics() { + if METER.get().is_some() { + return; + } + + let otel_endpoint = + std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").unwrap_or_else(|_| "http://localhost:4317".into()); + let service_name = + std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "scraper-api".into()); + let export_interval_ms: u64 = std::env::var("OTEL_METRICS_EXPORT_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5000); + + // Build the gRPC OTLP exporter + let exporter = opentelemetry_otlp::MetricExporter::builder() + .with_tonic() + .with_endpoint(otel_endpoint.clone()) + .build() + .expect("Failed to create OTLP metric exporter"); + + let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio) + .with_interval(std::time::Duration::from_millis(export_interval_ms)) + .build(); + + let resource = Resource::new(vec![ + KeyValue::new("service.name", service_name.clone()), + ]); + + let provider = MeterProviderBuilder::default() + .with_resource(resource) + .with_reader(reader) + .build(); + + // Keep a handle so we can shut it down later + let _ = PROVIDER.set(provider.clone()); + + global::set_meter_provider(provider); + + let m = global::meter("scraper-http-server"); + let _ = METER.set(m); + + tracing::info!(otel_endpoint, service_name, "OTel metrics initialized"); +} + +/// Shut down the global MeterProvider, flushing pending exports. +pub async fn shutdown_otel_metrics() { + if let Some(provider) = PROVIDER.get() { + if let Err(e) = provider.shutdown() { + tracing::warn!(error = %e, "OTel metrics shutdown error"); + } else { + tracing::info!("OTel metrics shut down"); + } + } +} + +// --------------------------------------------------------------------------- +// HTTP metrics middleware +// --------------------------------------------------------------------------- + +/// Axum middleware that records standard HTTP server metrics for every request. +/// +/// This middleware must be added as a layer **after** `init_otel_metrics()` has been called. +/// +/// Metrics emitted: +/// - `http.server.request_count` — Counter { method, path, status } +/// - `http.server.request_duration_ms` — Histogram { method, path, status } +/// - `http.server.request_in_flight` — UpDownCounter { method, path } +pub async fn otel_metrics_middleware(req: Request, next: Next) -> Response { + let method = req.method().to_string(); + let path = req.uri().path().to_string(); + + let in_flight = get_in_flight_counter(); + let counter = get_request_counter(); + let duration = get_duration_histogram(); + + // Record in-flight + in_flight.add( + 1, + &[ + KeyValue::new("method", method.clone()), + KeyValue::new("path", path.clone()), + ], + ); + + let start = Instant::now(); + let response = next.run(req).await; + let elapsed_ms = start.elapsed().as_millis() as f64; + + let status = response.status().as_u16().to_string(); + + // Record request count + duration + let attrs = [ + KeyValue::new("method", method), + KeyValue::new("path", path), + KeyValue::new("status", status), + ]; + counter.add(1, &attrs); + duration.record(elapsed_ms, &attrs); + + // Decrement in-flight + in_flight.add(-1, &[]); + + response +} + +/// Lazily-initialised instruments guarded by OnceLock. +fn get_in_flight_counter() -> UpDownCounter { + static INST: OnceLock> = OnceLock::new(); + INST.get_or_init(|| { + meter() + .i64_up_down_counter("http.server.request_in_flight") + .with_description("Number of HTTP requests currently in flight") + .build() + }) + .clone() +} + +fn get_request_counter() -> Counter { + static INST: OnceLock> = OnceLock::new(); + INST.get_or_init(|| { + meter() + .u64_counter("http.server.request_count") + .with_description("Total number of HTTP requests received") + .build() + }) + .clone() +} + +fn get_duration_histogram() -> Histogram { + static INST: OnceLock> = OnceLock::new(); + INST.get_or_init(|| { + meter() + .f64_histogram("http.server.request_duration_ms") + .with_description("Duration of HTTP requests in milliseconds") + .with_unit("ms") + .build() + }) + .clone() +} diff --git a/src/shared/observability/mod.rs b/src/shared/observability/mod.rs new file mode 100644 index 0000000..8255913 --- /dev/null +++ b/src/shared/observability/mod.rs @@ -0,0 +1,4 @@ +pub mod metrics; +pub mod openapi; +pub mod openapi_modules; +pub mod request_id; diff --git a/src/shared/observability/openapi.rs b/src/shared/observability/openapi.rs new file mode 100644 index 0000000..4c0c820 --- /dev/null +++ b/src/shared/observability/openapi.rs @@ -0,0 +1,14 @@ +use utoipa::OpenApi; + +/// Bridge for the auto-generated OpenAPI documentation. +/// This allows merging manual schema definitions with the auto-discovered handlers and schemas. +#[derive(OpenApi)] +#[openapi( + // Discovered routes are merged at runtime in bootstrap/mod.rs + info( + title = "Scraper API", + version = "1.0.0", + description = "High-performance scraping and CDN microservice" + ) +)] +pub struct ApiDoc; diff --git a/src/shared/observability/openapi_modules.rs b/src/shared/observability/openapi_modules.rs new file mode 100644 index 0000000..0a28aaa --- /dev/null +++ b/src/shared/observability/openapi_modules.rs @@ -0,0 +1,85 @@ +use utoipa::OpenApi; + +/// Manual aggregation of OpenAPI docs from module controllers. +#[derive(OpenApi)] +#[openapi( + paths( + // Anime module controllers + crate::modules::anime::controller::anime_index, + crate::modules::anime::controller::genres, + crate::modules::anime::controller::detail_slug, + crate::modules::anime::controller::complete_anime_slug, + crate::modules::anime::controller::full_slug, + crate::modules::anime::controller::ongoing_anime_slug, + crate::modules::anime::controller::latest_slug, + crate::modules::anime::controller::search_slug_index, + crate::modules::anime::controller::search_slug_page, + crate::modules::anime::controller::genre_slug_index, + crate::modules::anime::controller::genre_slug_page, + // Anime2 module controllers + crate::modules::anime2::controller::index, + crate::modules::anime2::controller::genre_list, + crate::modules::anime2::controller::filter, + crate::modules::anime2::controller::detail_slug, + crate::modules::anime2::controller::genre_slug_index, + crate::modules::anime2::controller::genre_slug_page, + crate::modules::anime2::controller::search_slug_index, + crate::modules::anime2::controller::search_slug_page, + crate::modules::anime2::controller::latest_slug, + crate::modules::anime2::controller::ongoing_anime_slug, + crate::modules::anime2::controller::complete_anime_slug, + // Komik module controllers + crate::modules::komik::controller::genre_list, + crate::modules::komik::controller::chapter_slug, + crate::modules::komik::controller::detail_slug, + crate::modules::komik::controller::genre_slug, + crate::modules::komik::controller::genre_slug_page, + crate::modules::komik::controller::manga_slug, + crate::modules::komik::controller::manhua_slug, + crate::modules::komik::controller::manhwa_slug, + crate::modules::komik::controller::popular_slug, + crate::modules::komik::controller::search_slug, + crate::modules::komik::controller::search_slug_page, + // Proxy module controllers + crate::modules::proxy::controller::fetch_with_proxy_only, + crate::modules::proxy::controller::image_cache, + ), + components( + schemas( + // Application response wrapper + crate::shared::types::ApiResponse, + // Anime2 types + crate::modules::anime2::types::Anime2Response, + crate::modules::anime2::types::GenresResponse, + crate::modules::anime2::types::FilterResponse, + crate::modules::anime2::types::DetailResponse, + // Anime2 query schemas + crate::modules::anime2::schema::FilterQuery, + crate::modules::anime2::schema::GenreQuery, + crate::modules::anime2::schema::SearchQuery, + // Proxy types + crate::modules::proxy::types::ImageCacheResponse, + crate::modules::proxy::types::AuditImageCacheResponse, + // Proxy request schemas + crate::modules::proxy::schema::ProxyParams, + crate::modules::proxy::schema::ImageCacheRequest, + crate::modules::proxy::schema::AuditImageCacheRequest, + // Komik types + crate::modules::komik::types::GenresResponse, + crate::modules::komik::types::Genre, + crate::modules::komik::types::ChapterResponse, + crate::modules::komik::types::DetailResponse, + crate::modules::komik::types::KomikItem, + crate::modules::komik::types::Pagination, + crate::modules::komik::types::GenreKomikResponse, + crate::modules::komik::types::SearchKomikResponse, + ) + ), + tags( + (name = "anime", description = "Anime endpoints"), + (name = "anime2", description = "Anime2 endpoints"), + (name = "komik", description = "Komik endpoints"), + (name = "proxy", description = "Proxy endpoints"), + ) +)] +pub struct ModuleApiDoc; diff --git a/src/shared/observability/request_id.rs b/src/shared/observability/request_id.rs new file mode 100644 index 0000000..b89312a --- /dev/null +++ b/src/shared/observability/request_id.rs @@ -0,0 +1,85 @@ +//! Request ID middleware for request tracing. + +use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response}; +use uuid::Uuid; + +/// Request ID header name. +pub const REQUEST_ID_HEADER: &str = "x-request-id"; + +/// Extension to access request ID in handlers. +#[derive(Clone, Debug)] +pub struct RequestId(pub String); + +impl RequestId { + /// Generate a new request ID. + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Get the request ID string. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for RequestId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for RequestId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Middleware that adds a unique request ID to each request. +/// +/// The request ID is: +/// - Taken from the `x-request-id` header if present +/// - Generated as a new UUID if not present +/// - Added to the response headers +/// - Available via the `RequestId` extension in handlers +/// +/// # Example +/// +/// ```ignore +/// use axum::Extension; +/// use scraper_service::observability::RequestId; +/// +/// async fn handler(Extension(req_id): Extension) { +/// println!("Request ID: {}", req_id); +/// } +/// ``` +pub async fn request_id_middleware(mut req: Request, next: Next) -> Response { + // Get or generate request ID + let request_id = req + .headers() + .get(REQUEST_ID_HEADER) + .and_then(|v| v.to_str().ok()) + .map(|s| RequestId(s.to_string())) + .unwrap_or_else(RequestId::new); + + // Add to tracing span + let span = tracing::info_span!( + "request", + request_id = %request_id, + method = %req.method(), + uri = %req.uri(), + ); + let _guard = span.enter(); + + // Insert as extension for handlers + req.extensions_mut().insert(request_id.clone()); + + // Process request + let mut response = next.run(req).await; + + // Add request ID to response headers + if let Ok(value) = HeaderValue::from_str(&request_id.0) { + response.headers_mut().insert(REQUEST_ID_HEADER, value); + } + + response +} diff --git a/src/shared/routing/mod.rs b/src/shared/routing/mod.rs new file mode 100644 index 0000000..890dc86 --- /dev/null +++ b/src/shared/routing/mod.rs @@ -0,0 +1,3 @@ +pub mod versioning; + +pub use versioning::{extract_version, versioned_routes, ApiVersion, VersionedApi}; diff --git a/src/shared/routing/versioning.rs b/src/shared/routing/versioning.rs new file mode 100644 index 0000000..a1f2f25 --- /dev/null +++ b/src/shared/routing/versioning.rs @@ -0,0 +1,135 @@ +//! API versioning utilities. +//! +//! Provides helpers for versioned API routes. + +use axum::Router; + +/// API version prefix. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ApiVersion { + V1, + V2, +} + +impl ApiVersion { + /// Get the URL prefix for this version. + pub fn prefix(&self) -> &'static str { + match self { + ApiVersion::V1 => "/api/v1", + ApiVersion::V2 => "/api/v2", + } + } +} + +impl std::fmt::Display for ApiVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ApiVersion::V1 => write!(f, "v1"), + ApiVersion::V2 => write!(f, "v2"), + } + } +} + +/// Builder for versioned API routes. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::versioning::VersionedApi; +/// +/// let app = VersionedApi::new() +/// .v1(users_v1_routes()) +/// .v1(products_v1_routes()) +/// .v2(users_v2_routes()) +/// .build(); +/// ``` +pub struct VersionedApi +where + S: Clone + Send + Sync + 'static, +{ + v1_routes: Vec>, + v2_routes: Vec>, +} + +impl VersionedApi +where + S: Clone + Send + Sync + 'static, +{ + /// Create a new versioned API builder. + pub fn new() -> Self { + Self { + v1_routes: Vec::new(), + v2_routes: Vec::new(), + } + } + + /// Add routes to API v1. + pub fn v1(mut self, routes: Router) -> Self { + self.v1_routes.push(routes); + self + } + + /// Add routes to API v2. + pub fn v2(mut self, routes: Router) -> Self { + self.v2_routes.push(routes); + self + } + + /// Build the combined router with versioned prefixes. + pub fn build(self) -> Router { + let mut app = Router::new(); + + // Merge v1 routes under /api/v1 + if !self.v1_routes.is_empty() { + let mut v1_router = Router::new(); + for routes in self.v1_routes { + v1_router = v1_router.merge(routes); + } + app = app.nest("/api/v1", v1_router); + } + + // Merge v2 routes under /api/v2 + if !self.v2_routes.is_empty() { + let mut v2_router = Router::new(); + for routes in self.v2_routes { + v2_router = v2_router.merge(routes); + } + app = app.nest("/api/v2", v2_router); + } + + app + } +} + +impl Default for VersionedApi +where + S: Clone + Send + Sync + 'static, +{ + fn default() -> Self { + Self::new() + } +} + +/// Create versioned routes with automatic fallback. +/// +/// Routes in v2 will be served under /api/v2. +/// Routes in v1 will be served under /api/v1 AND as fallback under /api/v2 if not overridden. +pub fn versioned_routes(v1: Router, v2: Router) -> Router +where + S: Clone + Send + Sync + 'static, +{ + Router::new() + .nest("/api/v1", v1.clone()) + .nest("/api/v2", v1.merge(v2)) // v2 includes v1 as fallback +} + +/// Extract API version from request path. +pub fn extract_version(path: &str) -> Option { + if path.starts_with("/api/v2") { + Some(ApiVersion::V2) + } else if path.starts_with("/api/v1") { + Some(ApiVersion::V1) + } else { + None + } +} diff --git a/src/shared/scheduler/cleanup_cache.rs b/src/shared/scheduler/cleanup_cache.rs new file mode 100644 index 0000000..bda09bf --- /dev/null +++ b/src/shared/scheduler/cleanup_cache.rs @@ -0,0 +1,222 @@ +//! Scheduled task for cleaning up old cached data. + +use async_trait::async_trait; +use sea_orm::*; +use std::sync::Arc; +use tracing::{info, warn}; + +use crate::shared::database::get_redis_pool; +use crate::shared::database::persistence::entities::image_cache; +use crate::shared::utils::cache::Cache; + +use super::ScheduledTask; + +/// Cleanup old cache data to prevent disk/memory bloat. +/// Runs daily at 2 AM to clean: +/// - Old image cache entries (>30 days) +/// - Orphaned cache keys in Redis +/// - Expired data without TTL +pub struct CleanupOldCache { + db: Arc, +} + +impl CleanupOldCache { + pub fn new(db: Arc) -> Self { + Self { db } + } +} + +#[async_trait] +impl ScheduledTask for CleanupOldCache { + fn name(&self) -> &'static str { + "cleanup_old_cache" + } + + fn schedule(&self) -> &'static str { + // Daily at 2 AM + "0 0 2 * * *" + } + + async fn run(&self) { + tracing::debug!("🧹 Starting old cache cleanup..."); + + let mut total_cleaned = 0; + + // 1. Clean old image cache (>30 days) + match self.cleanup_old_images(30).await { + Ok(count) => { + if count > 0 { + info!("✓ Cleaned {} old image cache entries", count); + } + total_cleaned += count; + } + Err(e) => { + warn!("Failed to clean old image cache: {}", e); + } + } + + // 2. Clean orphaned Redis keys + match self.cleanup_orphaned_redis_keys().await { + Ok(count) => { + if count > 0 { + info!("✓ Cleaned {} orphaned Redis keys", count); + } + total_cleaned += count; + } + Err(e) => { + warn!("Failed to clean orphaned Redis keys: {}", e); + } + } + + // 3. Compact Redis memory + match self.compact_redis_memory().await { + Ok(()) => { + if total_cleaned > 0 { + info!("✓ Redis memory compacted"); + } + } + Err(e) => { + warn!("Failed to compact Redis memory: {}", e); + } + } + + if total_cleaned > 0 { + info!("🎉 Cache cleanup complete: {} items cleaned", total_cleaned); + } + } +} + +impl CleanupOldCache { + /// Remove image cache entries older than specified days. + async fn cleanup_old_images(&self, days: i64) -> Result { + use chrono::{Duration, Utc}; + + let cutoff = Utc::now() - Duration::days(days); + + // Find old entries + let old_images = image_cache::Entity::find() + .filter(image_cache::Column::CreatedAt.lt(cutoff)) + .all(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + + let count = old_images.len(); + + if count == 0 { + return Ok(0); + } + + // Delete from database + let ids: Vec = old_images.iter().map(|img| img.id.clone()).collect(); + + image_cache::Entity::delete_many() + .filter(image_cache::Column::Id.is_in(ids)) + .exec(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + + // Also clean from Redis + let redis_pool = get_redis_pool().map_err(|e| e.to_string())?; + let cache = Cache::new(redis_pool); + for img in old_images { + let cache_key = format!("img_cache:{}", Self::hash_url(&img.original_url)); + let _ = cache.delete(&cache_key).await; + } + + Ok(count) + } + + /// Clean orphaned Redis keys (keys without TTL that shouldn't exist). + async fn cleanup_orphaned_redis_keys(&self) -> Result { + use deadpool_redis::redis::AsyncCommands; + + let pool = get_redis_pool().map_err(|e| e.to_string())?; + let mut conn = pool + .get() + .await + .map_err(|e| format!("Failed to get Redis connection: {}", e))?; + + let mut cleaned = 0; + + // Find keys without TTL (should not exist) + let patterns = vec!["anime:*", "komik:*", "user:*:profile", "img_cache:*"]; + + for pattern in patterns { + let keys = { + let mut iter: deadpool_redis::redis::AsyncIter<'_, String> = conn + .scan_match(pattern) + .await + .map_err(|e| format!("Failed to scan keys: {}", e))?; + + let mut keys = Vec::new(); + while let Some(key_result) = iter.next_item().await { + if let Ok(key) = key_result { + keys.push(key); + } + } + + keys + }; + + for key in keys { + let ttl: i64 = conn.ttl(key.as_str()).await.unwrap_or(-1); + + // TTL = -1 means no expiration (orphaned) + // TTL = -2 means key doesn't exist + if ttl == -1 { + // Set a default TTL of 7 days for orphaned keys + let _: () = conn.expire(key.as_str(), 604800).await.unwrap_or(()); + cleaned += 1; + } + } + } + + Ok(cleaned) + } + + /// Compact Redis memory to free up fragmented space. + async fn compact_redis_memory(&self) -> Result<(), String> { + let pool = get_redis_pool().map_err(|e| e.to_string())?; + let mut conn = pool + .get() + .await + .map_err(|e| format!("Failed to get Redis connection: {}", e))?; + + // Run MEMORY PURGE command - using cmd method on connection + let _: String = deadpool_redis::redis::cmd("MEMORY") + .arg("PURGE") + .query_async(&mut *conn) + .await + .map_err(|e| format!("Failed to purge memory: {}", e))?; + + Ok(()) + } + + /// Simple hash function for URL (same as in image_cache.rs). + /// Simple hash function for URL (same as in image_cache.rs). + fn hash_url(url: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(url.as_bytes()); + format!("{:x}", hasher.finalize()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_schedule() { + use sea_orm::{DatabaseBackend, MockDatabase}; + let db = MockDatabase::new(DatabaseBackend::Sqlite).into_connection(); + let task = CleanupOldCache { db: Arc::new(db) }; + assert_eq!(task.schedule(), "0 0 2 * * *"); + } + + #[test] + fn test_hash_url() { + let hash = CleanupOldCache::hash_url("https://example.com/image.jpg"); + assert_eq!(hash.len(), 64); // SHA256 hash length + } +} diff --git a/src/shared/scheduler/mod.rs b/src/shared/scheduler/mod.rs new file mode 100644 index 0000000..7387dd2 --- /dev/null +++ b/src/shared/scheduler/mod.rs @@ -0,0 +1,5 @@ +pub mod cleanup_cache; +pub mod runner; + +pub use cleanup_cache::CleanupOldCache; +pub use runner::{ScheduledTask, Scheduler}; diff --git a/src/shared/scheduler/runner.rs b/src/shared/scheduler/runner.rs new file mode 100644 index 0000000..13be76c --- /dev/null +++ b/src/shared/scheduler/runner.rs @@ -0,0 +1,93 @@ +//! Scheduler implementation using tokio-cron-scheduler. + +use async_trait::async_trait; +use std::sync::Arc; +use tokio_cron_scheduler::{Job, JobScheduler}; +use tracing::info; + +/// Trait for scheduled tasks. +#[async_trait] +pub trait ScheduledTask: Send + Sync { + /// Task name for logging. + fn name(&self) -> &'static str; + + /// Cron expression (e.g., "0 * * * * *" for every minute). + fn schedule(&self) -> &'static str; + + /// Execute the task. + async fn run(&self); +} + +/// Scheduler for running cron jobs. +pub struct Scheduler { + inner: JobScheduler, +} + +impl Scheduler { + /// Create a new scheduler. + pub async fn new() -> anyhow::Result { + let scheduler = JobScheduler::new().await?; + Ok(Self { inner: scheduler }) + } + + /// Add a task to the scheduler. + pub async fn add(&self, task: T) -> anyhow::Result<()> { + let task = Arc::new(task); + let task_name = task.name(); + let schedule = task.schedule(); + + let job = Job::new_async(schedule, move |_uuid, _lock| { + let task = Arc::clone(&task); + Box::pin(async move { + tracing::debug!("Running scheduled task: {}", task.name()); + task.run().await; + }) + })?; + + self.inner.add(job).await?; + info!("Scheduled task '{}' with cron: {}", task_name, schedule); + Ok(()) + } + + /// Add a simple job with a closure. + pub async fn add_job( + &self, + name: &'static str, + schedule: &str, + f: F, + ) -> anyhow::Result<()> + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: std::future::Future + Send + 'static, + { + let f = Arc::new(f); + let job = Job::new_async(schedule, move |_uuid, _lock| { + let f = Arc::clone(&f); + Box::pin(async move { + info!("Running scheduled job: {}", name); + f().await; + }) + })?; + + self.inner.add(job).await?; + info!("Scheduled job '{}' with cron: {}", name, schedule); + Ok(()) + } + + /// Start the scheduler. + pub async fn start(&self) -> anyhow::Result<()> { + info!("Starting scheduler"); + self.inner.start().await?; + Ok(()) + } + + /// Stop the scheduler. + pub async fn shutdown(&mut self) -> anyhow::Result<()> { + info!("Shutting down scheduler"); + self.inner.shutdown().await?; + Ok(()) + } +} + +// Real scheduled tasks with actual implementations +// Real scheduled tasks with actual implementations diff --git a/src/shared/scrapers/mod.rs b/src/shared/scrapers/mod.rs new file mode 100644 index 0000000..b8a8e6a --- /dev/null +++ b/src/shared/scrapers/mod.rs @@ -0,0 +1 @@ +pub mod otakudesu; diff --git a/src/shared/scrapers/otakudesu.rs b/src/shared/scrapers/otakudesu.rs new file mode 100644 index 0000000..bde0fd2 --- /dev/null +++ b/src/shared/scrapers/otakudesu.rs @@ -0,0 +1,115 @@ +use crate::shared::types::entities::anime::*; +use crate::shared::utils::parse_html; +use crate::shared::utils::web::scraping::{ + attr, attr_from_or, extract_slug, selector, text, text_from_or, +}; +use once_cell::sync::Lazy; +use scraper::{Html, Selector}; + +/// Common selectors used across anime parsing +pub struct AnimeSelectors { + pub item: Selector, + pub title: Selector, + pub link: Selector, + pub img: Selector, + pub episode: Selector, + pub score: Selector, + pub status: Selector, + pub genre: Selector, + pub rating: Selector, + pub type_sel: Selector, + pub season: Selector, + pub desc: Selector, +} + +impl AnimeSelectors { + pub fn new() -> Result { + Ok(Self { + item: selector("article.bs").ok_or("Invalid selector: article.bs")?, + title: selector(".tt h2").ok_or("Invalid selector: .tt h2")?, + link: selector("a").ok_or("Invalid selector: a")?, + img: selector("img").ok_or("Invalid selector: img")?, + episode: selector(".epx").ok_or("Invalid selector: .epx")?, + score: selector(".numscore").ok_or("Invalid selector: .numscore")?, + status: selector(".status").ok_or("Invalid selector: .status")?, + genre: selector(".genres a").ok_or("Invalid selector: .genres a")?, + rating: selector(".score").ok_or("Invalid selector: .score")?, + type_sel: selector(".typez").ok_or("Invalid selector: .typez")?, + season: selector(".season").ok_or("Invalid selector: .season")?, + desc: selector(".data .typez").ok_or("Invalid selector: .data .typez")?, + }) + } +} + +static ANIME_SELECTORS: Lazy = + Lazy::new(|| AnimeSelectors::new().expect("Valid CSS selectors")); + +pub fn extract_poster(element: &scraper::ElementRef, img_selector: &Selector) -> String { + element + .select(img_selector) + .next() + .and_then(|e| attr(&e, "src").or(attr(&e, "data-src"))) + .unwrap_or_default() +} + +pub fn parse_ongoing_anime(html: &str) -> Vec { + let document = parse_html(html); + let selectors = &*ANIME_SELECTORS; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let current_episode = text_from_or(&element, &selectors.episode, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + + items.push(OngoingAnimeItem { + title, + slug, + poster, + current_episode, + anime_url, + }); + } + items +} + +pub fn parse_pagination(document: &Html, current_page: u32) -> Result { + let pagination_selector = + selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?; + let next_selector = selector(".pagination .next").ok_or("Invalid selector")?; + + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| text(&e).trim().parse::().ok()) + .unwrap_or(current_page); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + Ok(Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }) +} diff --git a/src/shared/services/images/cache.rs b/src/shared/services/images/cache.rs new file mode 100644 index 0000000..0670971 --- /dev/null +++ b/src/shared/services/images/cache.rs @@ -0,0 +1,1140 @@ +//! Image caching helper using Picser CDN (picser.pages.dev). +//! +//! This module provides utilities to cache images via jsDelivr CDN +//! with database storage for URL mapping. + +use crate::shared::config::CONFIG; +use crate::shared::database::repositories::image_cache::SeaOrmImageCacheRepository; +use crate::shared::database::traits::image_cache::ImageCacheRepository; +use deadpool_redis::Pool as RedisPool; +use reqwest::Client; +use sea_orm::DatabaseConnection; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::{debug, error, warn}; + +use crate::shared::utils::cache_ttl::CACHE_TTL_IMAGE; +use crate::shared::utils::web::http_client::http_client; +use crate::shared::utils::Cache; + +/// Default TTL for image cache in Redis (24 hours) +pub const IMAGE_CACHE_TTL: u64 = CACHE_TTL_IMAGE; + +/// Redis key prefix for image cache +pub const IMAGE_CACHE_PREFIX: &str = "img_cache"; + +/// Redis key prefix for caching locks (to prevent duplicate uploads) +pub const IMAGE_CACHE_LOCK_PREFIX: &str = "img_cache_lock"; + +/// Lock TTL (60 seconds - enough time for upload to complete) +pub const IMAGE_CACHE_LOCK_TTL: u64 = 60; + +/// Static Picser API endpoints in priority order. Configured endpoint is inserted after primary. +pub const STATIC_PICSER_API_ENDPOINTS: &[&str] = &[ + "https://picser-two.vercel.app/api/upload", + "https://picser-mytheclipse8647-ahoqi9ef.leapcell.dev/api/upload", + "https://picser.pages.dev/api/upload", +]; + +/// Create a hash of the URL for cache key +pub fn url_hash(url: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(url.as_bytes()); + let result = hasher.finalize(); + hex::encode(&result[..16]) // Use 16 bytes for collision-resistant key +} + +/// Helper to convert any image URL to a fast WP.com (Jetpack) CDN URL. +/// This acts as a high-speed proxy even before Picser finishes caching. +pub fn to_wp_cdn(url: &str) -> String { + if url.is_empty() { + return url.to_string(); + } + + // If already a CDN URL, return as is + if url.contains("picser.pages.dev") + || url.contains("jsdelivr.net") + || url.contains("wp.com") + || url.contains("imagecdn.app") + { + return url.to_string(); + } + + // Remove protocol for wp.com format + let clean_url = url.trim().replace("https://", "").replace("http://", ""); + + // Use i0, i1, i2 or i3 based on hash to distribute load + let hash = url.len() % 4; + format!("https://i{}.wp.com/{}", hash, clean_url) +} + +/// Response from Picser API (/api/upload) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PicserResponse { + #[serde(default)] + pub success: bool, + pub url: Option, + pub urls: Option, + pub filename: Option, + pub size: Option, + #[serde(rename = "type")] + pub content_type: Option, + pub commit_sha: Option, + pub github_url: Option, + pub error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PicserUrls { + pub github: Option, + pub raw: Option, + pub jsdelivr: Option, + pub jsdelivr_commit: Option, +} + +/// Response from fallback upload API (/api/upload) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FallbackUploadResponse { + pub download_url: String, +} + +/// Configuration for image cache +#[derive(Debug, Clone)] +pub struct ImageCacheConfig { + /// GitHub token for Picser API (optional - uses public upload if not set) + pub github_token: Option, + /// GitHub owner for uploads + pub github_owner: String, + /// GitHub repo for uploads + pub github_repo: String, + /// GitHub branch + pub github_branch: String, + /// Upload folder + pub folder: String, +} + +impl Default for ImageCacheConfig { + fn default() -> Self { + Self { + github_token: None, + github_owner: "sh20raj".to_string(), + github_repo: "picser".to_string(), + github_branch: "main".to_string(), + folder: "uploads".to_string(), + } + } +} + +/// Image cache service +pub struct ImageCache { + repo: Arc, + client: Client, + _config: ImageCacheConfig, + semaphore: Option>, +} + +// Add imports for Request Coalescing +use dashmap::DashMap; +use once_cell::sync::Lazy; +use tokio::sync::broadcast; + +// Global In-Flight Uploads Map +// Maps Original URL -> Broadcast Sender +static IN_FLIGHT_UPLOADS: Lazy>>> = + Lazy::new(DashMap::new); + +impl ImageCache { + /// Create a new image cache instance + pub fn new(repo: Arc) -> Self { + Self { + repo, + client: http_client().client().clone(), // Reuse global HTTP client for connection pooling + _config: ImageCacheConfig::default(), + semaphore: None, + } + } + + pub fn with_config(repo: Arc, config: ImageCacheConfig) -> Self { + Self { + repo, + client: http_client().client().clone(), // Reuse global HTTP client + _config: config, + semaphore: None, + } + } + + /// Set concurrency limiter + pub fn with_semaphore(mut self, semaphore: std::sync::Arc) -> Self { + self.semaphore = Some(semaphore); + self + } + + /// Get CDN URL for an image, caching if needed + pub async fn get_or_cache(&self, original_url: &str) -> Result { + let cache_key = format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(original_url)); + let lock_key = format!("{}:{}", IMAGE_CACHE_LOCK_PREFIX, url_hash(original_url)); + + // 1. Check Redis cache first + + if let Some(cached_url) = self.repo.get_from_redis(&cache_key).await { + debug!("ImageCache: Redis hit for {}", original_url); + return Ok(cached_url); + } + + // 2. Check Request Coalescing (SingleFlight) + // This handles concurrent requests in this process/instance + let (tx, is_leader) = { + use dashmap::mapref::entry::Entry; + match IN_FLIGHT_UPLOADS.entry(original_url.to_string()) { + Entry::Occupied(entry) => { + debug!("ImageCache: Joining in-flight upload for {}", original_url); + (entry.get().clone(), false) + } + Entry::Vacant(entry) => { + let (tx, _) = broadcast::channel(1); + entry.insert(tx.clone()); + debug!("ImageCache: Starting leader upload for {}", original_url); + (tx, true) + } + } + }; + + if !is_leader { + // Follower: Wait for result + let mut rx = tx.subscribe(); + return match rx.recv().await { + Ok(Ok(url)) => Ok(url), + Ok(Err(e)) => Err(e), + Err(e) => { + warn!( + "ImageCache: Coalesce receive error for {}: {:?}", + original_url, e + ); + Err("Upload coalescing failed".to_string()) + } + }; + } + + // Leader: Perform the work + // We wrap the work in a closure/block to easily capture the result + let result = async { + // 3. Check database (Double check inside leader to be sure) + if let Some(cached_url) = self.repo.get_from_db(original_url).await? { + // Store in Redis for faster access + let _ = self + .repo + .set_in_redis(&cache_key, &cached_url, IMAGE_CACHE_TTL) + .await; + debug!("ImageCache: DB hit for {}", original_url); + return Ok(cached_url); + } + + // 4. Check if another process is already caching this URL (Distributed Lock check) + if self.repo.get_lock(&lock_key).await { + // Even if locked by another process, strict single-flight within this instance + // is good. But if another process is working, we might want to wait or just return error? + // Current logic returns error. + debug!( + "ImageCache: Already being cached by another process: {}", + original_url + ); + return Err(format!("URL {} is already being cached", original_url)); + } + + // 5. Acquire lock in Redis + let _ = self.repo.set_lock(&lock_key, IMAGE_CACHE_LOCK_TTL).await; + + // 6. Upload + debug!("ImageCache: Miss - uploading {} to Picser", original_url); + + // Acquire permit if semaphore is set + let _permit = if let Some(sem) = &self.semaphore { + match sem.acquire().await { + Ok(p) => Some(p), + Err(e) => { + let _ = self.repo.release_lock(&lock_key).await; + return Err(e.to_string()); + } + } + } else { + None + }; + + // Work + let work_result = async { + // Upload to Picser + let cdn_url = self.upload_to_picser(original_url).await?; + + // 6.5. Verify CDN URL Propagation (Self-Test before caching) + // CDNs like jsDelivr can take a few seconds to propagate after a GitHub commit. + // We retry 3 times with backoff to ensure we only return and cache a functional link. + let mut is_valid = false; + let mut last_verify_error = String::from("Verification not started"); + + for attempt in 1..=10 { + debug!( + "ImageCache: Verifying CDN URL {} (Attempt {})", + cdn_url, attempt + ); + match self.verify_cdn_url(&cdn_url).await { + Ok(true) => { + is_valid = true; + debug!( + "ImageCache: CDN URL verified successfully for {}", + original_url + ); + break; + } + Ok(false) => { + last_verify_error = "CDN returned non-image data or 404".to_string(); + } + Err(e) => { + last_verify_error = e; + } + } + + if attempt < 10 { + // Progressive backoff: 1s, 2s, 3s... up to 10s + let delay = 1000 * attempt; + tokio::time::sleep(std::time::Duration::from_millis(delay as u64)).await; + } + } + + if !is_valid { + error!( + "ImageCache: CDN verification failed for {} after 10 attempts: {}", + cdn_url, last_verify_error + ); + return Err(format!( + "CDN link was not accessible after upload: {}", + last_verify_error + )); + } + + // Save to database only after successful verification + self.repo.save_to_db(original_url, &cdn_url).await?; + + // Cache in Redis + let _ = self + .repo + .set_in_redis(&cache_key, &cdn_url, IMAGE_CACHE_TTL) + .await; + + // Invalidate API caches + let _ = self + .repo + .invalidate_api_caches(vec!["anime:*", "anime2:*", "komik:*"]) + .await; + + Ok(cdn_url) + } + .await; + + // Release Redis lock + let _ = self.repo.release_lock(&lock_key).await; + + work_result + } + .await; + + // Broadcast result + let _ = tx.send(result.clone()); + + // Remove from map + IN_FLIGHT_UPLOADS.remove(original_url); + + result + } + + /// Get CDN URL without uploading (read-only lookup) + pub async fn get_cdn_url(&self, original_url: &str) -> Option { + let cache_key = format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(original_url)); + + // Check Redis first + if let Some(cached_url) = self.repo.get_from_redis(&cache_key).await { + return Some(cached_url); + } + + // Check database + if let Ok(Some(cdn_url)) = self.repo.get_from_db(original_url).await { + return Some(cdn_url); + } + + None + } + + /// Find an original URL for a given CDN URL (reverse lookup) + pub async fn find_original_from_cdn(&self, cdn_url: &str) -> Option { + self.repo + .find_original_from_cdn(cdn_url) + .await + .ok() + .flatten() + } + + /// Invalidate cache for a URL + pub async fn invalidate(&self, original_url: &str) -> Result<(), String> { + let cache_key = format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(original_url)); + + // Remove from Redis + let _ = self.repo.delete_from_redis(&cache_key).await; + + // Remove from database + self.repo.delete_from_db(original_url).await?; + + debug!("ImageCache: Invalidated {}", original_url); + Ok(()) + } + + /// Helper to perform a single upload attempt + async fn perform_single_upload( + &self, + api_url: &str, + image_bytes: &[u8], + filename: &str, + ) -> Result { + debug!("ImageCache: Attempting upload to API server: {}", api_url); + + let part = reqwest::multipart::Part::bytes(image_bytes.to_vec()) + .file_name(filename.to_string()) + .mime_str("image/jpeg") + .map_err(|e| { + let err = format!("Failed to create multipart form for {}: {}", api_url, e); + error!("ImageCache: {}", err); + err + })?; + + let form = reqwest::multipart::Form::new().part("file", part); + + let response = self + .client + .post(api_url) + .multipart(form) + .send() + .await + .map_err(|e| { + let err = format!("Failed to send request to Picser API ({}): {}", api_url, e); + error!("ImageCache: {}", err); + err + })?; + + let response_status = response.status(); + let response_text = response.text().await.map_err(|e| { + let err = format!( + "Failed to read Picser response from {} (Status {}): {}", + api_url, response_status, e + ); + error!("ImageCache: {}", err); + err + })?; + + // Raw responses stay at debug level for troubleshooting without noisy default logs + debug!( + "ImageCache: Raw response from {} (Status {}): {}", + api_url, response_status, response_text + ); + + if !response_status.is_success() { + let error_message = serde_json::from_str::(&response_text) + .ok() + .and_then(|value| { + value + .get("error") + .and_then(|error| error.as_str()) + .map(|error| error.to_string()) + .or_else(|| { + value + .get("message") + .and_then(|message| message.as_str()) + .map(|message| message.to_string()) + }) + }) + .unwrap_or_else(|| response_text.clone()); + + let err = format!( + "Picser upload failed at {} (HTTP {}): {}", + api_url, response_status, error_message + ); + error!("ImageCache: {}", err); + return Err(err); + } + + let picser_response: PicserResponse = + serde_json::from_str(&response_text).map_err(|e| { + let err = format!( + "Failed to parse Picser response from {}: {} - Raw: {}", + api_url, e, response_text + ); + error!("ImageCache: {}", err); + err + })?; + + if !picser_response.success { + let err_msg = picser_response + .error + .unwrap_or_else(|| "Unknown error".to_string()); + let err = format!( + "Picser upload failed at {} (server error): {}", + api_url, err_msg + ); + error!("ImageCache: {}", err); + return Err(err); + } + + debug!("ImageCache: Upload successful to API server: {}", api_url); + Ok(picser_response) + } + + fn picser_api_endpoints(&self) -> Vec { + let configured = CONFIG.urls.picser_api_url.clone(); + let mut endpoints = vec![STATIC_PICSER_API_ENDPOINTS[0].to_string()]; + if !configured.is_empty() && !endpoints.contains(&configured) { + endpoints.push(configured); + } + for endpoint in STATIC_PICSER_API_ENDPOINTS.iter().skip(1) { + let endpoint = endpoint.to_string(); + if !endpoints.contains(&endpoint) { + endpoints.push(endpoint); + } + } + endpoints + } + + async fn upload_to_fallback_api( + &self, + image_bytes: &[u8], + filename: &str, + ) -> Result { + debug!( + "ImageCache: Attempting fallback upload to: {}", + CONFIG.urls.fallback_upload_api_url + ); + + let part = reqwest::multipart::Part::bytes(image_bytes.to_vec()) + .file_name(filename.to_string()) + .mime_str("image/jpeg") + .map_err(|e| { + let err = format!("Failed to create fallback multipart form: {}", e); + error!("ImageCache: {}", err); + err + })?; + + let form = reqwest::multipart::Form::new() + .part("file", part) + .text("fileName", filename.to_string()); + + let response = self + .client + .post(&CONFIG.urls.fallback_upload_api_url) + .multipart(form) + .send() + .await + .map_err(|e| { + let err = format!( + "Failed to send request to fallback upload API ({}): {}", + CONFIG.urls.fallback_upload_api_url, e + ); + error!("ImageCache: {}", err); + err + })?; + + let response_status = response.status(); + let response_text = response.text().await.map_err(|e| { + let err = format!( + "Failed to read fallback upload response from {} (Status {}): {}", + CONFIG.urls.fallback_upload_api_url, response_status, e + ); + error!("ImageCache: {}", err); + err + })?; + + debug!( + "ImageCache: Raw response from fallback upload API (Status {}): {}", + response_status, response_text + ); + + if !response_status.is_success() { + let err = format!( + "Fallback upload failed at {} (HTTP {}): {}", + CONFIG.urls.fallback_upload_api_url, response_status, response_text + ); + error!("ImageCache: {}", err); + return Err(err); + } + + let fallback_response: FallbackUploadResponse = serde_json::from_str(&response_text) + .map_err(|e| { + let err = format!( + "Failed to parse fallback upload response from {}: {} - Raw: {}", + CONFIG.urls.fallback_upload_api_url, e, response_text + ); + error!("ImageCache: {}", err); + err + })?; + + if fallback_response.download_url.trim().is_empty() { + return Err("Fallback upload response did not include download_url".to_string()); + } + + debug!( + "ImageCache: Fallback upload successful - URL: {}", + fallback_response.download_url + ); + Ok(fallback_response.download_url) + } + + async fn download_image_bytes(&self, original_url: &str) -> Result { + let mut candidates = vec![original_url.to_string()]; + if original_url.contains("https://alqanime.net/wp-content/") { + candidates.push(original_url.replace("https://alqanime.net/", "https://alqanime.si/")); + } + + let mut last_error = String::from("No download attempt started"); + for candidate in candidates { + debug!( + "ImageCache: Starting image download from source: {}", + candidate + ); + match self.client.get(&candidate).send().await { + Ok(response) => match response.bytes().await { + Ok(bytes) => { + let is_valid_image = infer::get(&bytes) + .map(|kind| kind.mime_type().starts_with("image/")) + .unwrap_or(false); + if is_valid_image { + return Ok(bytes); + } + + let trace_preview = String::from_utf8_lossy(&bytes) + .chars() + .take(100) + .collect::(); + last_error = format!( + "Image source ({}) returned non-image data (Preview: {})", + candidate, trace_preview + ); + warn!("ImageCache: {}", last_error); + } + Err(e) => { + last_error = format!( + "Failed to read image bytes from source ({}): {}", + candidate, e + ); + warn!("ImageCache: {}", last_error); + } + }, + Err(e) => { + last_error = format!( + "Failed to download image from source ({}): {}", + candidate, e + ); + warn!("ImageCache: {}", last_error); + } + } + } + + error!("ImageCache: {}", last_error); + Err(last_error) + } + + /// Upload image to Picser CDN with fallback upload API support + async fn upload_to_picser(&self, original_url: &str) -> Result { + // Download the image first + let image_bytes = self.download_image_bytes(original_url).await?; + + debug!( + "ImageCache: Image downloaded successfully, size: {} bytes", + image_bytes.len() + ); + + // Determine filename from URL + let filename = self.extract_filename(original_url); + + debug!( + "ImageCache: Will attempt upload to {} API endpoints sequentially with failover", + self.picser_api_endpoints().len() + ); + + let mut last_failed_api = String::from("Unknown"); + let picser_api_endpoints = self.picser_api_endpoints(); + let picser_api_count = picser_api_endpoints.len(); + + for (attempt_num, api_url) in picser_api_endpoints.iter().enumerate() { + let attempt_number = attempt_num + 1; + debug!( + "ImageCache: [Attempt {}/{}] Uploading {} bytes to: {}", + attempt_number, + picser_api_count, + image_bytes.len(), + api_url + ); + + match tokio::time::timeout( + std::time::Duration::from_secs(30), + self.perform_single_upload(api_url, &image_bytes, &filename), + ) + .await + { + Ok(Ok(response)) => match self.extract_cdn_url(response, original_url) { + Ok(cdn_url) => { + debug!( + "ImageCache: Upload succeeded on attempt {}/{} - CDN URL: {}", + attempt_number, picser_api_count, cdn_url + ); + return Ok(cdn_url); + } + Err(e) => { + last_failed_api = api_url.to_string(); + warn!( + "ImageCache: [Attempt {}/{}] Upload from {} did not yield a CDN URL: {}", + attempt_number, + picser_api_count, + api_url, + e + ); + error!( + "ImageCache: Attempt {}/{} failed - Last failed API: {} - Error: {}", + attempt_number, picser_api_count, last_failed_api, e + ); + } + }, + Ok(Err(e)) => { + last_failed_api = api_url.to_string(); + warn!( + "ImageCache: [Attempt {}/{}] Upload to {} failed: {}", + attempt_number, picser_api_count, api_url, e + ); + error!( + "ImageCache: Attempt {}/{} failed - Last failed API: {} - Error: {}", + attempt_number, picser_api_count, last_failed_api, e + ); + } + Err(_) => { + last_failed_api = api_url.to_string(); + let err = format!("Timeout (30s) while uploading to API endpoint: {}", api_url); + warn!( + "ImageCache: [Attempt {}/{}] {}", + attempt_number, picser_api_count, err + ); + error!( + "ImageCache: Attempt {}/{} failed - Last failed API: {} - Error: {}", + attempt_number, picser_api_count, last_failed_api, err + ); + } + } + } + + warn!( + "ImageCache: All {} Picser upload attempts failed for source URL: {}. Trying fallback upload API: {}", + picser_api_count, + original_url, + CONFIG.urls.fallback_upload_api_url + ); + + match tokio::time::timeout( + std::time::Duration::from_secs(30), + self.upload_to_fallback_api(&image_bytes, &filename), + ) + .await + { + Ok(Ok(cdn_url)) => Ok(cdn_url), + Ok(Err(e)) => { + error!( + "ImageCache: Fallback upload API failed after Picser failures. Last Picser API: {} - Fallback error: {}", + last_failed_api, e + ); + Err(format!( + "All {} Picser upload attempts and fallback upload API failed. Last Picser API endpoint: {}. Fallback error: {}", + picser_api_count, + last_failed_api, + e + )) + } + Err(_) => { + let err = format!( + "Timeout (30s) while uploading to fallback API endpoint: {}", + CONFIG.urls.fallback_upload_api_url + ); + error!( + "ImageCache: Fallback upload API timed out after Picser failures. Last Picser API: {} - {}", + last_failed_api, err + ); + Err(format!( + "All {} Picser upload attempts and fallback upload API failed. Last Picser API endpoint: {}. Fallback error: {}", + picser_api_count, + last_failed_api, + err + )) + } + } + } + + /// Internally verify a CDN URL's accessibility and validity + pub async fn verify_cdn_url(&self, cdn_url: &str) -> Result { + let resp = self.client.get(cdn_url).send().await.map_err(|e| { + let err = format!("Network error verifying CDN URL ({}): {}", cdn_url, e); + warn!("ImageCache: {}", err); + err + })?; + + let status = resp.status(); + if !status.is_success() { + let err = format!( + "CDN verification failed with HTTP {} for URL: {}", + status, cdn_url + ); + warn!("ImageCache: {}", err); + return Ok(false); + } + + let bytes = resp.bytes().await.map_err(|e| { + let err = format!("Failed to read bytes from CDN URL ({}): {}", cdn_url, e); + warn!("ImageCache: {}", err); + err + })?; + + // Structural verification (Fast MIME check) + let is_valid = infer::get(&bytes) + .map(|k| k.mime_type().starts_with("image/")) + .unwrap_or(false); + + if is_valid { + debug!( + "ImageCache: CDN URL verified successfully - content is valid image: {}", + cdn_url + ); + Ok(true) + } else { + warn!( + "ImageCache: CDN URL verification failed - content is not a valid image ({}): {}", + cdn_url, + String::from_utf8_lossy(&bytes[0..std::cmp::min(100, bytes.len())]) + ); + Ok(false) + } + } + + /// Extract CDN URL from Picser response + fn extract_cdn_url( + &self, + response: PicserResponse, + original_url: &str, + ) -> Result { + // Try to extract CDN URL from various response fields + if let Some(urls) = &response.urls { + if let Some(url) = &urls.raw { + debug!( + "ImageCache: Using CDN URL from urls.raw for source: {}", + original_url + ); + return Ok(url.clone()); + } + if let Some(url) = &urls.jsdelivr_commit { + debug!( + "ImageCache: Using CDN URL from urls.jsdelivr_commit for source: {}", + original_url + ); + return Ok(url.clone()); + } + if let Some(url) = &urls.jsdelivr { + debug!( + "ImageCache: Using CDN URL from urls.jsdelivr for source: {}", + original_url + ); + return Ok(url.clone()); + } + if let Some(url) = &urls.github { + debug!( + "ImageCache: Using CDN URL from urls.github for source: {}", + original_url + ); + return Ok(url.clone()); + } + } + + if let Some(url) = &response.url { + debug!( + "ImageCache: Using CDN URL from response.url for source: {}", + original_url + ); + return Ok(url.clone()); + } + + if let Some(url) = &response.github_url { + debug!( + "ImageCache: Using CDN URL from response.github_url for source: {}", + original_url + ); + return Ok(url.clone()); + } + + // If we get here, no CDN URL was found in the response + error!( + "ImageCache: No CDN URL found in Picser API response for source URL: {}. Response fields - success: {}, has_urls: {}, has_url: {}, has_github_url: {}, error: {}", + original_url, + response.success, + response.urls.is_some(), + response.url.is_some(), + response.github_url.is_some(), + response.error.as_deref().unwrap_or("none") + ); + Err(format!( + "No CDN URL in Picser response. Checked: urls.{{jsdelivr_commit,jsdelivr,raw,github}}, url, github_url. Response error: {}", + response.error.unwrap_or_else(|| "none".to_string()) + )) + } + + /// Extract filename from URL + fn extract_filename(&self, url: &str) -> String { + url.split('/') + .last() + .and_then(|s| s.split('?').next()) + .filter(|s| !s.is_empty() && s.contains('.')) + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("{}.jpg", url_hash(url))) + } +} + +/// Convenience function to create a CDN URL for an image +/// Returns the original URL if caching fails (graceful fallback) +pub async fn cache_image_url( + db: Arc, + redis: &RedisPool, + original_url: &str, +) -> String { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db, redis.clone())); + let cache = ImageCache::new(repo); + match cache.get_or_cache(original_url).await { + Ok(cdn_url) => cdn_url, + Err(e) => { + warn!("ImageCache: Failed to cache {}: {}", original_url, e); + to_wp_cdn(original_url) // Use WP CDN as graceful fallback + } + } +} + +/// Batch cache multiple images +pub async fn cache_image_urls( + db: Arc, + redis: &RedisPool, + urls: &[String], +) -> Vec { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db, redis.clone())); + let cache = ImageCache::new(repo); + let mut results = Vec::with_capacity(urls.len()); + + for url in urls { + let cdn_url = match cache.get_or_cache(url).await { + Ok(u) => u, + Err(_) => url.clone(), + }; + results.push(cdn_url); + } + + results +} + +/// Helper to convert image URL to CDN URL in background (non-blocking) +/// Returns original URL immediately and caches in background +pub fn cache_image_url_lazy( + db: Arc, + redis: &RedisPool, + original_url: String, + semaphore: Option>, +) -> String { + let db_owned = db; + let redis_owned = redis.clone(); + let url = original_url.clone(); + let sem_owned = semaphore.clone(); + + // Spawn background task to cache + tokio::spawn(async move { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db_owned, redis_owned)); + let mut cache = ImageCache::new(repo); + if let Some(sem) = sem_owned { + cache = cache.with_semaphore(sem); + } + + match cache.get_or_cache(&url).await { + Ok(_) => {} + Err(_) => {} + } + }); + + to_wp_cdn(&original_url) +} + +/// Convert image URL to CDN URL if already cached, otherwise return original +/// and trigger background caching for next request (with duplicate prevention) +/// Convert image URL to CDN URL if already cached, otherwise return original +/// and trigger background caching for next request (with duplicate prevention) +pub async fn get_cached_or_original( + db: Arc, + redis: &RedisPool, + original_url: &str, + semaphore: Option>, +) -> String { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db.clone(), redis.clone())); + let cache = ImageCache::new(repo); + + // Check if already cached (Redis or DB) + if let Some(cdn_url) = cache.get_cdn_url(original_url).await { + return cdn_url; + } + + // Check if currently being cached by another process + let lock_key = format!("{}:{}", IMAGE_CACHE_LOCK_PREFIX, url_hash(original_url)); + let redis_cache = Cache::new(redis); + if redis_cache.get::(&lock_key).await.is_some() { + return to_wp_cdn(original_url); + } + + // Not cached and not being cached - start background caching + let db_owned = db.clone(); + let redis_owned = redis.clone(); + let url = original_url.to_string(); + let sem_owned = semaphore.clone(); + + tokio::spawn(async move { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db_owned, redis_owned)); + let mut cache = ImageCache::new(repo); + if let Some(sem) = sem_owned { + cache = cache.with_semaphore(sem); + } + + let _ = cache.get_or_cache(&url).await; + }); + + to_wp_cdn(original_url) +} + +/// Batch process multiple image URLs - returns original URLs immediately +/// and triggers background caching for all +/// Batch process multiple image URLs - checks cache first, returns cached URL if found +/// For misses: returns original URL and triggers background caching +pub async fn cache_image_urls_batch_lazy( + db: Arc, + redis: &RedisPool, + urls: Vec, + semaphore: Option>, +) -> Vec { + if urls.is_empty() { + return urls; + } + + let mut results = vec![String::new(); urls.len()]; + let mut missing_indices = Vec::new(); + + // 1. Batch check Redis + let redis_cache = Cache::new(redis); + let cache_keys: Vec = urls + .iter() + .map(|url| format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(url))) + .collect(); + + let cached_values: Vec> = redis_cache.mget(&cache_keys).await; + + for (i, val) in cached_values.iter().enumerate() { + if let Some(cdn_url) = val { + results[i] = cdn_url.clone(); + } else { + missing_indices.push(i); + } + } + + // 2. Batch check Database for Redis misses + if !missing_indices.is_empty() { + let missing_urls: Vec = missing_indices.iter().map(|&i| urls[i].clone()).collect(); + + // Note: Using repository for batch check would be better, but keeping it direct for now to match SeaORM usage + // but I should probably add a batch method to repository later. + use crate::shared::database::persistence::entities::image_cache; + use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; + + match image_cache::Entity::find() + .filter(image_cache::Column::OriginalUrl.is_in(missing_urls.clone())) + .all(db.as_ref()) + .await + { + Ok(db_entries) => { + let db_map: std::collections::HashMap = db_entries + .into_iter() + .map(|e| (e.original_url, e.cdn_url)) + .collect(); + + let mut still_missing_indices = Vec::new(); + + for &idx in &missing_indices { + let url = &urls[idx]; + if let Some(cdn_url) = db_map.get(url) { + results[idx] = cdn_url.clone(); + // Put back to Redis + let _ = redis_cache + .set_with_ttl(&cache_keys[idx], cdn_url, IMAGE_CACHE_TTL) + .await; + } else { + // Real miss - return WP CDN proxy and trigger background upload + results[idx] = to_wp_cdn(url); + still_missing_indices.push(idx); + } + } + + // 3. Trigger background caching for still missing URLs + if !still_missing_indices.is_empty() { + let db_owned = db.clone(); + let redis_owned = redis.clone(); + let sem_owned = semaphore.clone(); + let urls_to_cache: Vec = still_missing_indices + .iter() + .map(|&idx| urls[idx].clone()) + .collect(); + + tokio::spawn(async move { + use futures::stream::{self, StreamExt}; + let repo = Arc::new(SeaOrmImageCacheRepository::new(db_owned, redis_owned)); + let mut cache = ImageCache::new(repo); + if let Some(sem) = sem_owned { + cache = cache.with_semaphore(sem); + } + + stream::iter(urls_to_cache) + .map(|url| { + let cache_ref = &cache; + async move { + let _ = cache_ref.get_or_cache(&url).await; + } + }) + .buffer_unordered(20) + .collect::>() + .await; + }); + } + } + Err(e) => { + error!("ImageCache: Batch DB check failed: {}", e); + for &idx in &missing_indices { + results[idx] = to_wp_cdn(&urls[idx]); + } + } + } + } + + results +} + +/// Apply cached CDN poster URLs to a collection of items using the HasPoster trait. +pub async fn apply_cached_posters( + items: &mut [T], + db: Arc, + redis: &RedisPool, + semaphore: Option>, +) { + let posters: Vec = items.iter().map(|item| item.poster().to_string()).collect(); + let cached = cache_image_urls_batch_lazy(db, redis, posters, semaphore).await; + for (i, item) in items.iter_mut().enumerate() { + if let Some(url) = cached.get(i) { + item.set_poster(url.clone()); + } + } +} diff --git a/src/shared/services/images/mod.rs b/src/shared/services/images/mod.rs new file mode 100644 index 0000000..a5c08fd --- /dev/null +++ b/src/shared/services/images/mod.rs @@ -0,0 +1 @@ +pub mod cache; diff --git a/src/shared/services/mod.rs b/src/shared/services/mod.rs new file mode 100644 index 0000000..8f0da9f --- /dev/null +++ b/src/shared/services/mod.rs @@ -0,0 +1 @@ +pub mod images; diff --git a/src/shared/state/mod.rs b/src/shared/state/mod.rs new file mode 100644 index 0000000..7aff869 --- /dev/null +++ b/src/shared/state/mod.rs @@ -0,0 +1,17 @@ +use std::sync::Arc; + +use deadpool_redis::Pool; +use sea_orm::DatabaseConnection; + +pub struct AppState { + pub redis_pool: Pool, + pub db: Arc, + pub image_processing_semaphore: Arc, + pub event_bus: Arc, +} + +impl AppState { + pub fn sea_orm(&self) -> &DatabaseConnection { + &self.db + } +} diff --git a/src/shared/testing/app.rs b/src/shared/testing/app.rs new file mode 100644 index 0000000..e5e5b5b --- /dev/null +++ b/src/shared/testing/app.rs @@ -0,0 +1,226 @@ +//! Test application utilities. +//! +//! Provides a `TestApp` struct for integration testing that boots +//! the application in-memory with a test configuration. + +use axum::{ + body::Body, + http::{Method, Request, StatusCode}, + response::Response, + Router, +}; +use serde::{de::DeserializeOwned, Serialize}; +use tower::ServiceExt; + +/// A test application instance for integration testing. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::testing::TestApp; +/// +/// #[tokio::test] +/// async fn test_health_endpoint() { +/// let app = TestApp::new().await; +/// +/// let response = app.get("/health").await; +/// assert_eq!(response.status(), 200); +/// +/// let body = response.json::().await; +/// assert_eq!(body.status, "ok"); +/// } +/// ``` +pub struct TestApp { + router: Router, +} + +impl TestApp { + /// Create a new test application with default configuration. + /// + /// This sets up the router without starting a server. + pub fn with_router(router: Router) -> Self { + Self { router } + } + + /// Make a GET request. + pub async fn get(&self, path: &str) -> TestResponse { + self.request(Method::GET, path, Body::empty()).await + } + + /// Make a POST request with JSON body. + pub async fn post(&self, path: &str, body: &T) -> TestResponse { + let body = serde_json::to_string(body).unwrap_or_default(); + self.request_with_json(Method::POST, path, body).await + } + + /// Make a PUT request with JSON body. + pub async fn put(&self, path: &str, body: &T) -> TestResponse { + let body = serde_json::to_string(body).unwrap_or_default(); + self.request_with_json(Method::PUT, path, body).await + } + + /// Make a DELETE request. + pub async fn delete(&self, path: &str) -> TestResponse { + self.request(Method::DELETE, path, Body::empty()).await + } + + /// Make a request with custom method and body. + pub async fn request(&self, method: Method, path: &str, body: Body) -> TestResponse { + let request = Request::builder() + .method(method) + .uri(path) + .body(body) + .unwrap_or_else(|_| Request::new(Body::empty())); + + let response = self + .router + .clone() + .oneshot(request) + .await + .unwrap_or_else(|_| Response::builder().status(500).body(Body::empty()).unwrap()); + + TestResponse::new(response) + } + + /// Make a request with JSON content type. + async fn request_with_json(&self, method: Method, path: &str, body: String) -> TestResponse { + let request = Request::builder() + .method(method) + .uri(path) + .header("Content-Type", "application/json") + .body(Body::from(body)) + .unwrap_or_else(|_| Request::new(Body::empty())); + + let response = self + .router + .clone() + .oneshot(request) + .await + .unwrap_or_else(|_| Response::builder().status(500).body(Body::empty()).unwrap()); + + TestResponse::new(response) + } +} + +/// A test response with assertion helpers. +pub struct TestResponse { + response: Response, + body: Option, +} + +impl TestResponse { + fn new(response: Response) -> Self { + Self { + response, + body: None, + } + } + + /// Get the response status code. + pub fn status(&self) -> StatusCode { + self.response.status() + } + + /// Assert the status code. + pub fn assert_status(self, expected: u16) -> Self { + assert_eq!( + self.response.status().as_u16(), + expected, + "Expected status {} but got {}", + expected, + self.response.status() + ); + self + } + + /// Assert the status is 2xx (success). + pub fn assert_success(self) -> Self { + assert!( + self.response.status().is_success(), + "Expected success status but got {}", + self.response.status() + ); + self + } + + /// Assert the status is 4xx (client error). + pub fn assert_client_error(self) -> Self { + assert!( + self.response.status().is_client_error(), + "Expected client error status but got {}", + self.response.status() + ); + self + } + + /// Get a header value. + pub fn header(&self, name: &str) -> Option<&str> { + self.response + .headers() + .get(name) + .and_then(|v| v.to_str().ok()) + } + + /// Get the response body as bytes. + pub async fn bytes(mut self) -> bytes::Bytes { + if let Some(body) = self.body.take() { + return body; + } + + let body = std::mem::replace(self.response.body_mut(), Body::empty()); + axum::body::to_bytes(body, usize::MAX) + .await + .unwrap_or_default() + } + + /// Get the response body as a string. + pub async fn text(self) -> String { + let bytes = self.bytes().await; + String::from_utf8_lossy(&bytes).to_string() + } + + /// Get the response body as JSON. + pub async fn json(self) -> T { + let bytes = self.bytes().await; + serde_json::from_slice(&bytes).expect("Failed to parse response as JSON") + } + + /// Assert the response body contains a string. + pub async fn assert_body_contains(self, expected: &str) -> Self { + let body = self.text().await; + assert!( + body.contains(expected), + "Expected body to contain '{}' but got: {}", + expected, + body + ); + // Recreate self with cached body (simplified) + Self { + response: Response::builder().status(200).body(Body::empty()).unwrap(), + body: Some(bytes::Bytes::from(body)), + } + } +} + +/// Builder for TestApp with custom configuration. +pub struct TestAppBuilder { + // Future: Add configuration options +} + +impl TestAppBuilder { + /// Create a new test app builder. + pub fn new() -> Self { + Self {} + } + + /// Build the test app with a router. + pub fn build(self, router: Router) -> TestApp { + TestApp::with_router(router) + } +} + +impl Default for TestAppBuilder { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shared/testing/mod.rs b/src/shared/testing/mod.rs new file mode 100644 index 0000000..309be62 --- /dev/null +++ b/src/shared/testing/mod.rs @@ -0,0 +1 @@ +pub mod app; diff --git a/src/shared/types/api_response.rs b/src/shared/types/api_response.rs new file mode 100644 index 0000000..6f671d6 --- /dev/null +++ b/src/shared/types/api_response.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] +pub struct ApiResponse { + pub success: bool, + pub message: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +impl ApiResponse { + pub fn success(data: T) -> Self { + Self { + success: true, + message: None, + data: Some(data), + } + } + + pub fn error(message: String) -> Self { + Self { + success: false, + message: Some(message), + data: None, + } + } +} diff --git a/src/shared/types/entities/anime.rs b/src/shared/types/entities/anime.rs new file mode 100644 index 0000000..9db207f --- /dev/null +++ b/src/shared/types/entities/anime.rs @@ -0,0 +1,208 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +// ============================================================================ +// PAGINATION MODELS +// ============================================================================ + +/// Common pagination structure used across all anime2 endpoints +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Pagination { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +impl Pagination { + /// Create pagination with string-based next/previous pages + pub fn with_string_pages( + current_page: u32, + last_visible_page: u32, + has_next_page: bool, + next_page: Option, + has_previous_page: bool, + previous_page: Option, + ) -> PaginationWithStringPages { + PaginationWithStringPages { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + } + } +} + +/// Pagination variant with string-based page numbers (used in search endpoint) +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct PaginationWithStringPages { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +// ============================================================================ +// ANIME ITEM MODELS +// ============================================================================ + +/// Anime item for ongoing anime listings +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub current_episode: String, + pub anime_url: String, +} + +/// Anime item for ongoing anime with score (used in paginated ongoing lists) +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeItemWithScore { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub anime_url: String, +} + +/// Anime item for complete anime listings +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct CompleteAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode_count: String, + pub anime_url: String, +} + +/// Anime item for latest anime listings with episode and score +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct LatestAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub current_episode: String, + pub score: String, + pub anime_url: String, +} + +/// Anime item for search results with full metadata +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub description: String, + pub anime_url: String, + pub genres: Vec, + pub rating: String, + pub r#type: String, + pub season: String, +} + +/// Anime item for genre filtering +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub status: String, + pub anime_url: String, +} + +/// Anime item for advanced filtering (used in filter endpoint) +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FilterAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub status: String, + pub r#type: String, + pub anime_url: String, +} + +// ============================================================================ +// TRAITS +// ============================================================================ + +/// Trait for extracting poster URLs from anime items +pub trait HasPoster { + fn poster(&self) -> &str; + fn set_poster(&mut self, url: String); +} + +// ============================================================================ +// TRAIT IMPLEMENTATIONS +// ============================================================================ + +impl HasPoster for OngoingAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for OngoingAnimeItemWithScore { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for CompleteAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for LatestAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for SearchAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for GenreAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for FilterAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} diff --git a/src/shared/types/entities/image.rs b/src/shared/types/entities/image.rs new file mode 100644 index 0000000..222aa19 --- /dev/null +++ b/src/shared/types/entities/image.rs @@ -0,0 +1,11 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImageCache { + pub id: String, + pub original_url: String, + pub cdn_url: String, + pub created_at: DateTime, + pub expires_at: Option>, +} diff --git a/src/shared/types/entities/mod.rs b/src/shared/types/entities/mod.rs new file mode 100644 index 0000000..ee74445 --- /dev/null +++ b/src/shared/types/entities/mod.rs @@ -0,0 +1,3 @@ +pub mod anime; +pub mod image; +pub mod types; diff --git a/src/shared/types/entities/types.rs b/src/shared/types/entities/types.rs new file mode 100644 index 0000000..05ea6a7 --- /dev/null +++ b/src/shared/types/entities/types.rs @@ -0,0 +1,56 @@ +use crate::shared::errors::AppError; +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ErrorResponse { + pub error: String, +} + +impl From for ErrorResponse { + fn from(app_error: AppError) -> Self { + ErrorResponse { + error: app_error.to_string(), + } + } +} + +impl IntoResponse for ErrorResponse { + fn into_response(self) -> Response { + let (status, error_message) = match self.error.as_str() { + _ if self.error.contains("Environment variable not found") => { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self.error.contains("Redis error") => { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self.error.contains("Reqwest error") => { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self + .error + .contains("JSON serialization/deserialization error") => + { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self.error.contains("URL parsing error") => (StatusCode::BAD_REQUEST, self.error), + _ if self.error.contains("JWT error") => (StatusCode::UNAUTHORIZED, self.error), + _ if self.error.contains("Scraper error") => (StatusCode::BAD_GATEWAY, self.error), + _ if self.error.contains("Fantoccini error") => (StatusCode::BAD_GATEWAY, self.error), + _ if self.error.contains("IO error") => (StatusCode::INTERNAL_SERVER_ERROR, self.error), + _ if self.error.contains("Timeout error") => (StatusCode::REQUEST_TIMEOUT, self.error), + _ => (StatusCode::INTERNAL_SERVER_ERROR, self.error), + }; + + let body = Json(json!({ + "error": error_message, + })); + + (status, body).into_response() + } +} diff --git a/src/shared/types/mod.rs b/src/shared/types/mod.rs new file mode 100644 index 0000000..67040d2 --- /dev/null +++ b/src/shared/types/mod.rs @@ -0,0 +1,4 @@ +pub mod api_response; +pub mod entities; + +pub use api_response::ApiResponse; diff --git a/src/shared/utils/core/api_response.rs b/src/shared/utils/core/api_response.rs new file mode 100644 index 0000000..2f5ee22 --- /dev/null +++ b/src/shared/utils/core/api_response.rs @@ -0,0 +1,301 @@ +//! Standardized API response helpers. +//! +//! Provides consistent JSON response structures across all API endpoints. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::api_response::{ApiResponse, ApiResult}; +//! +//! async fn get_user(id: i32) -> ApiResult { +//! let user = find_user(id)?; +//! Ok(ApiResponse::success(user)) +//! } +//! +//! async fn list_users(page: u32) -> ApiResult> { +//! let (users, total) = find_users_paginated(page)?; +//! Ok(ApiResponse::paginated(users, page, 20, total)) +//! } +//! +//! async fn delete_user(id: i32) -> ApiResult<()> { +//! delete_user(id)?; +//! Ok(ApiResponse::no_content()) +//! } +//! ``` + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; + +/// Standard API response wrapper. +#[derive(Debug, Serialize, Deserialize)] +pub struct ApiResponse { + /// Whether the request was successful + pub success: bool, + /// Response data (if successful) + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + /// Error message (if failed) + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Pagination metadata (if applicable) + #[serde(skip_serializing_if = "Option::is_none")] + pub pagination: Option, + /// Additional metadata + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, +} + +/// Error details for failed responses. +#[derive(Debug, Serialize, Deserialize)] +pub struct ErrorDetails { + /// Error code (machine-readable) + pub code: String, + /// Human-readable error message + pub message: String, + /// Field-level validation errors + #[serde(skip_serializing_if = "Option::is_none")] + pub fields: Option>, +} + +/// Field-level validation error. +#[derive(Debug, Serialize, Deserialize)] +pub struct FieldError { + /// Field name + pub field: String, + /// Error message + pub message: String, +} + +/// Pagination metadata. +#[derive(Debug, Serialize, Deserialize)] +pub struct PaginationMeta { + /// Current page number + pub page: u32, + /// Items per page + pub per_page: u32, + /// Total number of items + pub total: u64, + /// Total number of pages + pub total_pages: u32, + /// Whether there's a next page + pub has_next: bool, + /// Whether there's a previous page + pub has_prev: bool, +} + +impl ApiResponse { + /// Create a successful response with data. + pub fn success(data: T) -> Self { + Self { + success: true, + data: Some(data), + error: None, + pagination: None, + meta: None, + } + } + + /// Create a successful response with data and metadata. + pub fn success_with_meta(data: T, meta: serde_json::Value) -> Self { + Self { + success: true, + data: Some(data), + error: None, + pagination: None, + meta: Some(meta), + } + } + + /// Create a paginated response. + pub fn paginated(data: T, page: u32, per_page: u32, total: u64) -> Self { + let total_pages = ((total as f64) / (per_page as f64)).ceil() as u32; + Self { + success: true, + data: Some(data), + error: None, + pagination: Some(PaginationMeta { + page, + per_page, + total, + total_pages, + has_next: page < total_pages, + has_prev: page > 1, + }), + meta: None, + } + } +} + +impl ApiResponse<()> { + /// Create a successful response with no data. + pub fn no_content() -> Self { + Self { + success: true, + data: None, + error: None, + pagination: None, + meta: None, + } + } + + /// Create a successful message response. + pub fn message(msg: &str) -> ApiResponse { + ApiResponse { + success: true, + data: Some(MessageOnly { + message: msg.to_string(), + }), + error: None, + pagination: None, + meta: None, + } + } +} + +/// Simple message-only response data. +#[derive(Debug, Serialize, Deserialize)] +pub struct MessageOnly { + pub message: String, +} + +/// API error response builder. +#[derive(Debug)] +pub struct ApiError { + pub status: StatusCode, + pub code: String, + pub message: String, + pub fields: Option>, +} + +impl ApiError { + /// Create a new API error. + pub fn new(status: StatusCode, code: &str, message: &str) -> Self { + Self { + status, + code: code.to_string(), + message: message.to_string(), + fields: None, + } + } + + /// Create a 400 Bad Request error. + pub fn bad_request(message: &str) -> Self { + Self::new(StatusCode::BAD_REQUEST, "BAD_REQUEST", message) + } + + /// Create a 401 Unauthorized error. + pub fn unauthorized(message: &str) -> Self { + Self::new(StatusCode::UNAUTHORIZED, "UNAUTHORIZED", message) + } + + /// Create a 403 Forbidden error. + pub fn forbidden(message: &str) -> Self { + Self::new(StatusCode::FORBIDDEN, "FORBIDDEN", message) + } + + /// Create a 404 Not Found error. + pub fn not_found(message: &str) -> Self { + Self::new(StatusCode::NOT_FOUND, "NOT_FOUND", message) + } + + /// Create a 409 Conflict error. + pub fn conflict(message: &str) -> Self { + Self::new(StatusCode::CONFLICT, "CONFLICT", message) + } + + /// Create a 422 Unprocessable Entity error with field errors. + pub fn validation(fields: Vec) -> Self { + Self { + status: StatusCode::UNPROCESSABLE_ENTITY, + code: "VALIDATION_ERROR".to_string(), + message: "Validation failed".to_string(), + fields: Some(fields), + } + } + + /// Create a 429 Too Many Requests error. + pub fn too_many_requests(message: &str) -> Self { + Self::new(StatusCode::TOO_MANY_REQUESTS, "RATE_LIMITED", message) + } + + /// Create a 500 Internal Server Error. + pub fn internal(message: &str) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "INTERNAL_ERROR", message) + } + + /// Create a 503 Service Unavailable error. + pub fn service_unavailable(message: &str) -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "SERVICE_UNAVAILABLE", + message, + ) + } + + /// Add field errors. + pub fn with_fields(mut self, fields: Vec) -> Self { + self.fields = Some(fields); + self + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let body = ApiResponse::<()> { + success: false, + data: None, + error: Some(ErrorDetails { + code: self.code, + message: self.message, + fields: self.fields, + }), + pagination: None, + meta: None, + }; + + (self.status, Json(body)).into_response() + } +} + +impl IntoResponse for ApiResponse { + fn into_response(self) -> Response { + let status = if self.success { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + }; + (status, Json(self)).into_response() + } +} + +/// Type alias for API handler results. +pub type ApiResult = Result, ApiError>; + +/// Helper to convert any serializable to success response. +pub fn success(data: T) -> ApiResponse { + ApiResponse::success(data) +} + +/// Helper to convert any serializable to paginated response. +pub fn paginated(data: T, page: u32, per_page: u32, total: u64) -> ApiResponse { + ApiResponse::paginated(data, page, per_page, total) +} + +/// Helper for quick internal error. +pub fn internal_err(msg: &str) -> ApiError { + ApiError::internal(msg) +} + +/// Helper for quick not found error. +pub fn not_found(msg: &str) -> ApiError { + ApiError::not_found(msg) +} + +/// Helper for quick bad request error. +pub fn bad_request(msg: &str) -> ApiError { + ApiError::bad_request(msg) +} diff --git a/src/shared/utils/core/errors.rs b/src/shared/utils/core/errors.rs new file mode 100644 index 0000000..a75dd93 --- /dev/null +++ b/src/shared/utils/core/errors.rs @@ -0,0 +1,101 @@ +//! Axum error response helpers. + +use axum::http::StatusCode; + +/// Shorthand for creating error tuples for Axum handlers. +pub type HandlerError = (StatusCode, String); + +/// Create internal server error. +pub fn internal_error(msg: impl Into) -> HandlerError { + (StatusCode::INTERNAL_SERVER_ERROR, msg.into()) +} + +/// Create internal server error from any error type. +pub fn internal_err(e: E) -> HandlerError { + (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) +} + +/// Create bad request error. +pub fn bad_request(msg: impl Into) -> HandlerError { + (StatusCode::BAD_REQUEST, msg.into()) +} + +/// Create not found error. +pub fn not_found(msg: impl Into) -> HandlerError { + (StatusCode::NOT_FOUND, msg.into()) +} + +/// Create unauthorized error. +pub fn unauthorized(msg: impl Into) -> HandlerError { + (StatusCode::UNAUTHORIZED, msg.into()) +} + +/// Create forbidden error. +pub fn forbidden(msg: impl Into) -> HandlerError { + (StatusCode::FORBIDDEN, msg.into()) +} + +/// Create conflict error. +pub fn conflict(msg: impl Into) -> HandlerError { + (StatusCode::CONFLICT, msg.into()) +} + +/// Create too many requests error. +pub fn too_many_requests(msg: impl Into) -> HandlerError { + (StatusCode::TOO_MANY_REQUESTS, msg.into()) +} + +/// Map any error to internal server error. +pub fn map_internal(e: E) -> HandlerError { + internal_err(e) +} + +/// Create Redis error response. +pub fn redis_error(e: E) -> HandlerError { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Redis error: {}", e), + ) +} + +/// Create database error response. +pub fn db_error(e: E) -> HandlerError { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Database error: {}", e), + ) +} + +/// Create serialization error response. +pub fn serialization_error(e: E) -> HandlerError { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Serialization error: {}", e), + ) +} + +/// Trait extension for Result to easily convert errors. +pub trait ResultExt { + /// Map error to internal server error. + fn map_internal(self) -> Result; + + /// Map error to bad request. + fn map_bad_request(self) -> Result; + + /// Map error to not found. + fn map_not_found(self) -> Result; +} + +impl ResultExt for Result { + fn map_internal(self) -> Result { + self.map_err(internal_err) + } + + fn map_bad_request(self) -> Result { + self.map_err(|e| bad_request(e.to_string())) + } + + fn map_not_found(self) -> Result { + self.map_err(|e| not_found(e.to_string())) + } +} diff --git a/src/shared/utils/core/handler.rs b/src/shared/utils/core/handler.rs new file mode 100644 index 0000000..a0f3438 --- /dev/null +++ b/src/shared/utils/core/handler.rs @@ -0,0 +1,99 @@ +//! Handler helper macros and utilities. + +/// Macro to create a simple CRUD handler set. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::helpers::crud_handlers; +/// +/// crud_handlers!(User, users); +/// // Generates: list_users, get_user, create_user, update_user, delete_user +/// ``` +#[macro_export] +macro_rules! crud_handlers { + ($entity:ident, $name:ident) => { + paste::paste! { + pub async fn []( + State(db): State>, + Query(params): Query, + ) -> impl IntoResponse { + use sea_orm::PaginatorTrait; + + let paginator = $entity::Entity::find() + .paginate(&*db, params.limit); + + let total = paginator.num_items().await.unwrap_or(0); + let items = paginator + .fetch_page(params.page.saturating_sub(1)) + .await + .unwrap_or_default(); + + Json(Paginated::from_params(items, ¶ms, total)) + } + + pub async fn []( + State(db): State>, + Path(id): Path, + ) -> Result { + let item = $entity::Entity::find_by_id(&id) + .one(&*db) + .await + .map_err(|e| ErrorResponse::internal(e.to_string()))? + .ok_or_else(|| ErrorResponse::not_found(concat!(stringify!($entity), " not found")))?; + + Ok(Json(json_ok(item))) + } + + pub async fn []( + State(db): State>, + Path(id): Path, + ) -> Result { + let result = $entity::Entity::delete_by_id(&id) + .exec(&*db) + .await + .map_err(|e| ErrorResponse::internal(e.to_string()))?; + + if result.rows_affected > 0 { + Ok(no_content()) + } else { + Err(ErrorResponse::not_found(concat!(stringify!($entity), " not found"))) + } + } + } + }; +} + +/// Wrap a handler result with consistent error handling. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::helpers::handler::try_handler; +/// +/// async fn my_handler() -> impl IntoResponse { +/// try_handler(async { +/// let data = fetch_data().await?; +/// Ok(data) +/// }).await +/// } +/// ``` +pub async fn try_handler( + f: F, +) -> Result, crate::shared::utils::ErrorResponse> +where + T: serde::Serialize, + E: std::fmt::Display, + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + match f().await { + Ok(data) => Ok(crate::shared::utils::JsonResponse::ok(data)), + Err(e) => Err(crate::shared::utils::ErrorResponse::internal(e.to_string())), + } +} + +/// Extract user ID from JWT claims (helper). +pub fn get_user_id_from_claims(claims: &serde_json::Value) -> Option { + claims.get("sub").and_then(|v| v.as_str()).map(String::from) +} diff --git a/src/shared/utils/core/mod.rs b/src/shared/utils/core/mod.rs new file mode 100644 index 0000000..029362b --- /dev/null +++ b/src/shared/utils/core/mod.rs @@ -0,0 +1,6 @@ +pub mod api_response; +pub mod errors; +pub mod handler; +pub mod pagination; +pub mod prelude; +pub mod response; diff --git a/src/shared/utils/core/pagination.rs b/src/shared/utils/core/pagination.rs new file mode 100644 index 0000000..76dcf89 --- /dev/null +++ b/src/shared/utils/core/pagination.rs @@ -0,0 +1,105 @@ +//! Pagination helpers. + +use serde::{Deserialize, Serialize}; + +/// Pagination query parameters. +#[derive(Debug, Clone, Deserialize)] +pub struct PaginationParams { + /// Page number (1-indexed). + #[serde(default = "default_page")] + pub page: u64, + /// Items per page. + #[serde(default = "default_limit")] + pub limit: u64, + /// Sort field. + #[serde(default)] + pub sort: Option, + /// Sort order (asc/desc). + #[serde(default = "default_order")] + pub order: String, +} + +fn default_page() -> u64 { + 1 +} +fn default_limit() -> u64 { + 20 +} +fn default_order() -> String { + "asc".to_string() +} + +impl PaginationParams { + /// Calculate offset for database query. + pub fn offset(&self) -> u64 { + (self.page.saturating_sub(1)) * self.limit + } + + /// Check if ascending order. + pub fn is_asc(&self) -> bool { + self.order.to_lowercase() == "asc" + } +} + +impl Default for PaginationParams { + fn default() -> Self { + Self { + page: default_page(), + limit: default_limit(), + sort: None, + order: default_order(), + } + } +} + +/// Paginated response wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct Paginated { + pub data: Vec, + pub pagination: PaginationMeta, +} + +/// Pagination metadata. +#[derive(Debug, Clone, Serialize)] +pub struct PaginationMeta { + pub page: u64, + pub limit: u64, + pub total: u64, + pub total_pages: u64, + pub has_next: bool, + pub has_prev: bool, +} + +impl Paginated { + /// Create a paginated response. + pub fn new(data: Vec, page: u64, limit: u64, total: u64) -> Self { + let total_pages = (total + limit - 1) / limit; + Self { + data, + pagination: PaginationMeta { + page, + limit, + total, + total_pages, + has_next: page < total_pages, + has_prev: page > 1, + }, + } + } + + /// Create from params and total. + pub fn from_params(data: Vec, params: &PaginationParams, total: u64) -> Self { + Self::new(data, params.page, params.limit, total) + } +} + +/// Trait to easily paginate database results. +pub trait Paginatable { + fn paginate(self, params: &PaginationParams, total: u64) -> Paginated; +} + +impl Paginatable for Vec { + fn paginate(self, params: &PaginationParams, total: u64) -> Paginated { + Paginated::from_params(self, params, total) + } +} diff --git a/src/shared/utils/core/prelude.rs b/src/shared/utils/core/prelude.rs new file mode 100644 index 0000000..3b6f022 --- /dev/null +++ b/src/shared/utils/core/prelude.rs @@ -0,0 +1,33 @@ +//! Prelude module - common imports for handlers. +//! +//! # Usage +//! +//! ```ignore +//! use scraper_service::helpers::prelude::*; +//! ``` + +// Re-export common types +pub use axum::{ + extract::{Extension, Path, Query, State}, + http::StatusCode, + response::IntoResponse, + Json, +}; +pub use serde::{Deserialize, Serialize}; +pub use tracing::{debug, error, info, warn}; + +// Re-export our helpers +pub use super::pagination::{Paginated, PaginationParams}; +pub use super::response::{ApiResult, ErrorResponse, JsonResponse}; + +// Re-export common extractors +pub use crate::shared::observability::request_id::RequestId; + +// Re-export database types +pub use sea_orm::{ + ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QueryOrder, + QuerySelect, Set, +}; + +// Re-export common std types +pub use std::sync::Arc; diff --git a/src/shared/utils/core/response.rs b/src/shared/utils/core/response.rs new file mode 100644 index 0000000..34f70e2 --- /dev/null +++ b/src/shared/utils/core/response.rs @@ -0,0 +1,148 @@ +//! Response helpers for consistent API responses. + +use axum::{http::StatusCode, response::IntoResponse, Json}; +use serde::Serialize; + +/// Result type for API handlers. +pub type ApiResult = Result, ErrorResponse>; + +/// Success JSON response wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct JsonResponse { + pub success: bool, + pub data: T, + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +impl JsonResponse { + /// Create a success response. + pub fn ok(data: T) -> Self { + Self { + success: true, + data, + message: None, + } + } + + /// Create a success response with message. + pub fn ok_with_message(data: T, message: impl Into) -> Self { + Self { + success: true, + data, + message: Some(message.into()), + } + } +} + +impl IntoResponse for JsonResponse { + fn into_response(self) -> axum::response::Response { + (StatusCode::OK, Json(self)).into_response() + } +} + +/// Error response wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(skip)] + pub status: StatusCode, +} + +impl ErrorResponse { + /// Create an error response. + pub fn new(status: StatusCode, error: impl Into) -> Self { + Self { + success: false, + error: error.into(), + code: None, + status, + } + } + + /// Add an error code. + pub fn with_code(mut self, code: impl Into) -> Self { + self.code = Some(code.into()); + self + } + + // Common error constructors + + /// 400 Bad Request + pub fn bad_request(error: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, error) + } + + /// 401 Unauthorized + pub fn unauthorized(error: impl Into) -> Self { + Self::new(StatusCode::UNAUTHORIZED, error) + } + + /// 403 Forbidden + pub fn forbidden(error: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, error) + } + + /// 404 Not Found + pub fn not_found(error: impl Into) -> Self { + Self::new(StatusCode::NOT_FOUND, error) + } + + /// 409 Conflict + pub fn conflict(error: impl Into) -> Self { + Self::new(StatusCode::CONFLICT, error) + } + + /// 422 Unprocessable Entity + pub fn unprocessable(error: impl Into) -> Self { + Self::new(StatusCode::UNPROCESSABLE_ENTITY, error) + } + + /// 500 Internal Server Error + pub fn internal(error: impl Into) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error) + } +} + +impl IntoResponse for ErrorResponse { + fn into_response(self) -> axum::response::Response { + (self.status, Json(&self)).into_response() + } +} + +impl From for ErrorResponse { + fn from(err: anyhow::Error) -> Self { + Self::internal(err.to_string()) + } +} + +impl From for ErrorResponse { + fn from(err: sea_orm::DbErr) -> Self { + Self::internal(format!("Database error: {}", err)) + } +} + +// Convenience functions + +/// Create a success JSON response. +pub fn json_ok(data: T) -> JsonResponse { + JsonResponse::ok(data) +} + +/// Create an empty success response. +pub fn ok() -> impl IntoResponse { + (StatusCode::OK, Json(serde_json::json!({"success": true}))) +} + +/// Create a created response (201). +pub fn created(data: T) -> impl IntoResponse { + (StatusCode::CREATED, Json(JsonResponse::ok(data))) +} + +/// Create a no content response (204). +pub fn no_content() -> impl IntoResponse { + StatusCode::NO_CONTENT +} diff --git a/src/shared/utils/data/collections.rs b/src/shared/utils/data/collections.rs new file mode 100644 index 0000000..e1ec301 --- /dev/null +++ b/src/shared/utils/data/collections.rs @@ -0,0 +1,169 @@ +//! Collection and iterator utilities. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +/// Chunk a vector into smaller vectors of specified size. +pub fn chunk(items: Vec, size: usize) -> Vec> { + items.chunks(size).map(|chunk| chunk.to_vec()).collect() +} + +/// Get unique items from a vector. +pub fn unique(items: Vec) -> Vec { + let mut seen = HashSet::new(); + items + .into_iter() + .filter(|item| seen.insert(item.clone())) + .collect() +} + +/// Group items by a key function. +pub fn group_by(items: Vec, key_fn: F) -> HashMap> +where + K: Hash + Eq, + F: Fn(&T) -> K, +{ + let mut groups: HashMap> = HashMap::new(); + for item in items { + let key = key_fn(&item); + groups.entry(key).or_default().push(item); + } + groups +} + +/// Partition items into two vectors based on predicate. +pub fn partition(items: Vec, predicate: F) -> (Vec, Vec) +where + F: Fn(&T) -> bool, +{ + let mut pass = Vec::new(); + let mut fail = Vec::new(); + for item in items { + if predicate(&item) { + pass.push(item); + } else { + fail.push(item); + } + } + (pass, fail) +} + +/// Flatten nested vectors. +pub fn flatten(nested: Vec>) -> Vec { + nested.into_iter().flatten().collect() +} + +/// Zip two vectors into pairs. +pub fn zip(a: Vec, b: Vec) -> Vec<(T, U)> { + a.into_iter().zip(b).collect() +} + +/// Find first matching item. +pub fn find(items: &[T], predicate: F) -> Option<&T> +where + F: Fn(&T) -> bool, +{ + items.iter().find(|item| predicate(item)) +} + +/// Find first matching item and return its index. +pub fn find_index(items: &[T], predicate: F) -> Option +where + F: Fn(&T) -> bool, +{ + items.iter().position(|item| predicate(item)) +} + +/// Check if any item matches predicate. +pub fn any(items: &[T], predicate: F) -> bool +where + F: Fn(&T) -> bool, +{ + items.iter().any(predicate) +} + +/// Check if all items match predicate. +pub fn all(items: &[T], predicate: F) -> bool +where + F: Fn(&T) -> bool, +{ + items.iter().all(predicate) +} + +/// Sum numeric values. +pub fn sum(items: &[T], value_fn: F) -> N +where + N: std::iter::Sum, + F: Fn(&T) -> N, +{ + items.iter().map(value_fn).sum() +} + +/// Count items matching predicate. +pub fn count(items: &[T], predicate: F) -> usize +where + F: Fn(&T) -> bool, +{ + items.iter().filter(|item| predicate(item)).count() +} + +/// Get first n items. +pub fn take(items: &[T], n: usize) -> Vec { + items.iter().take(n).cloned().collect() +} + +/// Skip first n items. +pub fn skip(items: &[T], n: usize) -> Vec { + items.iter().skip(n).cloned().collect() +} + +/// Reverse a vector. +pub fn reverse(items: &[T]) -> Vec { + items.iter().rev().cloned().collect() +} + +/// Interleave two vectors. +pub fn interleave(a: Vec, b: Vec) -> Vec { + let mut result = Vec::with_capacity(a.len() + b.len()); + let mut a_iter = a.into_iter(); + let mut b_iter = b.into_iter(); + loop { + match (a_iter.next(), b_iter.next()) { + (Some(x), Some(y)) => { + result.push(x); + result.push(y); + } + (Some(x), None) => result.push(x), + (None, Some(y)) => result.push(y), + (None, None) => break, + } + } + result +} + +/// Create a frequency map. +pub fn frequencies(items: &[T]) -> HashMap { + let mut freq = HashMap::new(); + for item in items { + *freq.entry(item.clone()).or_insert(0) += 1; + } + freq +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_chunk() { + let items = vec![1, 2, 3, 4, 5]; + let chunks = chunk(items, 2); + assert_eq!(chunks, vec![vec![1, 2], vec![3, 4], vec![5]]); + } + + #[test] + fn test_unique() { + let items = vec![1, 2, 2, 3, 3, 3]; + assert_eq!(unique(items), vec![1, 2, 3]); + } +} diff --git a/src/shared/utils/data/convert/bools.rs b/src/shared/utils/data/convert/bools.rs new file mode 100644 index 0000000..4d3fc1b --- /dev/null +++ b/src/shared/utils/data/convert/bools.rs @@ -0,0 +1,84 @@ +/// Convert string to bool (flexible parsing). +pub fn to_bool(s: &str) -> bool { + matches!( + s.to_lowercase().trim(), + "true" | "1" | "yes" | "on" | "enabled" | "t" | "y" | "ok" | "active" + ) +} + +/// Convert to bool with Option for invalid input. +pub fn try_bool(s: &str) -> Option { + match s.to_lowercase().trim() { + "true" | "1" | "yes" | "on" | "enabled" | "t" | "y" | "ok" | "active" => Some(true), + "false" | "0" | "no" | "off" | "disabled" | "f" | "n" | "inactive" => Some(false), + _ => None, + } +} + +/// Convert bool to string. +pub fn bool_to_str(b: bool) -> &'static str { + if b { + "true" + } else { + "false" + } +} + +/// Convert bool to yes/no. +pub fn bool_to_yes_no(b: bool) -> &'static str { + if b { + "yes" + } else { + "no" + } +} + +/// Convert bool to on/off. +pub fn bool_to_on_off(b: bool) -> &'static str { + if b { + "on" + } else { + "off" + } +} + +/// Convert bool to enabled/disabled. +pub fn bool_to_enabled(b: bool) -> &'static str { + if b { + "enabled" + } else { + "disabled" + } +} + +/// Convert bool to active/inactive. +pub fn bool_to_active(b: bool) -> &'static str { + if b { + "active" + } else { + "inactive" + } +} + +/// Convert bool to 0/1 i32. +pub fn bool_to_int(b: bool) -> i32 { + if b { + 1 + } else { + 0 + } +} + +/// Convert bool to 0/1 i64. +pub fn bool_to_i64(b: bool) -> i64 { + if b { + 1 + } else { + 0 + } +} + +/// Convert i32 to bool (0 = false, other = true). +pub fn int_to_bool(n: i32) -> bool { + n != 0 +} diff --git a/src/shared/utils/data/convert/bytes.rs b/src/shared/utils/data/convert/bytes.rs new file mode 100644 index 0000000..0cc1b76 --- /dev/null +++ b/src/shared/utils/data/convert/bytes.rs @@ -0,0 +1,130 @@ +/// Convert bytes to hex string (lowercase). +pub fn bytes_to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{:02x}", b)).collect() +} + +/// Convert bytes to hex string (uppercase). +pub fn bytes_to_hex_upper(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{:02X}", b)).collect() +} + +/// Convert hex string to bytes. +pub fn hex_to_bytes(hex: &str) -> Result, std::num::ParseIntError> { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16)) + .collect() +} + +/// Convert bytes to base64. +pub fn bytes_to_base64(bytes: &[u8]) -> String { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +/// Convert base64 to bytes. +pub fn base64_to_bytes(s: &str) -> Result, base64::DecodeError> { + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, s) +} + +/// Convert bytes to binary string. +pub fn bytes_to_binary(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{:08b}", b)) + .collect::>() + .join(" ") +} + +/// Convert bytes to octal string. +pub fn bytes_to_octal(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{:03o}", b)) + .collect::>() + .join(" ") +} + +/// Convert u8 to binary string. +pub fn u8_to_binary(n: u8) -> String { + format!("{:08b}", n) +} + +/// Convert u16 to binary string. +pub fn u16_to_binary(n: u16) -> String { + format!("{:016b}", n) +} + +/// Convert u32 to binary string. +pub fn u32_to_binary(n: u32) -> String { + format!("{:032b}", n) +} + +/// Convert u64 to binary string. +pub fn u64_to_binary(n: u64) -> String { + format!("{:064b}", n) +} + +/// Convert binary string to u64. +pub fn binary_to_u64(s: &str) -> Result { + u64::from_str_radix(&s.replace(" ", ""), 2) +} + +/// Swap endianness of u16. +pub fn swap_endian_u16(n: u16) -> u16 { + n.swap_bytes() +} + +/// Swap endianness of u32. +pub fn swap_endian_u32(n: u32) -> u32 { + n.swap_bytes() +} + +/// Swap endianness of u64. +pub fn swap_endian_u64(n: u64) -> u64 { + n.swap_bytes() +} + +/// Swap endianness of u128. +pub fn swap_endian_u128(n: u128) -> u128 { + n.swap_bytes() +} + +/// Convert to big endian bytes. +pub fn u32_to_be_bytes(n: u32) -> [u8; 4] { + n.to_be_bytes() +} + +/// Convert to little endian bytes. +pub fn u32_to_le_bytes(n: u32) -> [u8; 4] { + n.to_le_bytes() +} + +/// Convert from big endian bytes. +pub fn be_bytes_to_u32(bytes: [u8; 4]) -> u32 { + u32::from_be_bytes(bytes) +} + +/// Convert from little endian bytes. +pub fn le_bytes_to_u32(bytes: [u8; 4]) -> u32 { + u32::from_le_bytes(bytes) +} + +/// Convert to big endian bytes. +pub fn u64_to_be_bytes(n: u64) -> [u8; 8] { + n.to_be_bytes() +} + +/// Convert to little endian bytes. +pub fn u64_to_le_bytes(n: u64) -> [u8; 8] { + n.to_le_bytes() +} + +/// Convert from big endian bytes. +pub fn be_bytes_to_u64(bytes: [u8; 8]) -> u64 { + u64::from_be_bytes(bytes) +} + +/// Convert from little endian bytes. +pub fn le_bytes_to_u64(bytes: [u8; 8]) -> u64 { + u64::from_le_bytes(bytes) +} diff --git a/src/shared/utils/data/convert/char.rs b/src/shared/utils/data/convert/char.rs new file mode 100644 index 0000000..8514de5 --- /dev/null +++ b/src/shared/utils/data/convert/char.rs @@ -0,0 +1,65 @@ +/// Convert char to u32 (unicode code point). +pub fn char_to_u32(c: char) -> u32 { + c as u32 +} + +/// Convert u32 to char (unicode code point). +pub fn u32_to_char(n: u32) -> Option { + char::from_u32(n) +} + +/// Convert char to ascii u8. +pub fn char_to_ascii(c: char) -> Option { + if c.is_ascii() { + Some(c as u8) + } else { + None + } +} + +/// Convert u8 to char. +pub fn u8_to_char(n: u8) -> char { + n as char +} + +/// Convert digit char to u8. +pub fn digit_to_u8(c: char) -> Option { + c.to_digit(10).map(|d| d as u8) +} + +/// Convert u8 to digit char. +pub fn u8_to_digit(n: u8) -> Option { + if n <= 9 { + Some((b'0' + n) as char) + } else { + None + } +} + +/// Convert hex char to u8. +pub fn hex_char_to_u8(c: char) -> Option { + c.to_digit(16).map(|d| d as u8) +} + +/// Convert u8 to hex char (lowercase). +pub fn u8_to_hex_char(n: u8) -> Option { + if n < 16 { + Some(if n < 10 { + (b'0' + n) as char + } else { + (b'a' + n - 10) as char + }) + } else { + None + } +} + +/// Convert char to uppercase. +pub fn char_to_upper(c: char) -> char { + c.to_ascii_uppercase() +} + +/// Convert char to lowercase. +pub fn char_to_lower(c: char) -> char { + c.to_ascii_lowercase() +} diff --git a/src/shared/utils/data/convert/collections.rs b/src/shared/utils/data/convert/collections.rs new file mode 100644 index 0000000..d5d0b5d --- /dev/null +++ b/src/shared/utils/data/convert/collections.rs @@ -0,0 +1,103 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, LinkedList, VecDeque}; +use std::hash::Hash; + +/// Convert Vec to Vec. +pub fn map_vec(vec: Vec, f: F) -> Vec +where + F: Fn(T) -> U, +{ + vec.into_iter().map(f).collect() +} + +/// Convert &[T] to Vec. +pub fn map_slice(slice: &[T], f: F) -> Vec +where + F: Fn(&T) -> U, +{ + slice.iter().map(f).collect() +} + +/// Convert Vec to Vec<&str>. +pub fn strings_to_strs(strings: &[String]) -> Vec<&str> { + strings.iter().map(|s| s.as_str()).collect() +} + +/// Convert Vec<&str> to Vec. +pub fn strs_to_strings(strs: &[&str]) -> Vec { + strs.iter().map(|s| s.to_string()).collect() +} + +/// Convert slice to fixed array. +pub fn slice_to_array(slice: &[T]) -> Option<[T; N]> { + slice.try_into().ok() +} + +/// Convert Vec to VecDeque. +pub fn vec_to_deque(v: Vec) -> VecDeque { + VecDeque::from(v) +} + +/// Convert VecDeque to Vec. +pub fn deque_to_vec(d: VecDeque) -> Vec { + d.into_iter().collect() +} + +/// Convert Vec to LinkedList. +pub fn vec_to_linked_list(v: Vec) -> LinkedList { + v.into_iter().collect() +} + +/// Convert LinkedList to Vec. +pub fn linked_list_to_vec(l: LinkedList) -> Vec { + l.into_iter().collect() +} + +/// Convert Vec to HashSet. +pub fn vec_to_hashset(v: Vec) -> HashSet { + v.into_iter().collect() +} + +/// Convert HashSet to Vec. +pub fn hashset_to_vec(s: HashSet) -> Vec { + s.into_iter().collect() +} + +/// Convert Vec to BTreeSet. +pub fn vec_to_btreeset(v: Vec) -> BTreeSet { + v.into_iter().collect() +} + +/// Convert BTreeSet to Vec. +pub fn btreeset_to_vec(s: BTreeSet) -> Vec { + s.into_iter().collect() +} + +/// Convert Vec of tuples to HashMap. +pub fn vec_to_hashmap(v: Vec<(K, V)>) -> HashMap { + v.into_iter().collect() +} + +/// Convert HashMap to Vec of tuples. +pub fn hashmap_to_vec(m: HashMap) -> Vec<(K, V)> { + m.into_iter().collect() +} + +/// Convert Vec of tuples to BTreeMap. +pub fn vec_to_btreemap(v: Vec<(K, V)>) -> BTreeMap { + v.into_iter().collect() +} + +/// Convert BTreeMap to Vec of tuples. +pub fn btreemap_to_vec(m: BTreeMap) -> Vec<(K, V)> { + m.into_iter().collect() +} + +/// Convert HashMap to BTreeMap. +pub fn hashmap_to_btreemap(m: HashMap) -> BTreeMap { + m.into_iter().collect() +} + +/// Convert BTreeMap to HashMap. +pub fn btreemap_to_hashmap(m: BTreeMap) -> HashMap { + m.into_iter().collect() +} diff --git a/src/shared/utils/data/convert/color.rs b/src/shared/utils/data/convert/color.rs new file mode 100644 index 0000000..c31e2c4 --- /dev/null +++ b/src/shared/utils/data/convert/color.rs @@ -0,0 +1,70 @@ +/// Convert hex color to RGB tuple. +pub fn hex_to_rgb(hex: &str) -> Option<(u8, u8, u8)> { + let hex = hex.trim_start_matches('#'); + if hex.len() != 6 { + return None; + } + let r = u8::from_str_radix(&hex[0..2], 16).ok()?; + let g = u8::from_str_radix(&hex[2..4], 16).ok()?; + let b = u8::from_str_radix(&hex[4..6], 16).ok()?; + Some((r, g, b)) +} + +/// Convert hex color to RGBA tuple. +pub fn hex_to_rgba(hex: &str) -> Option<(u8, u8, u8, u8)> { + let hex = hex.trim_start_matches('#'); + if hex.len() == 6 { + let (r, g, b) = hex_to_rgb(hex)?; + return Some((r, g, b, 255)); + } + if hex.len() != 8 { + return None; + } + let r = u8::from_str_radix(&hex[0..2], 16).ok()?; + let g = u8::from_str_radix(&hex[2..4], 16).ok()?; + let b = u8::from_str_radix(&hex[4..6], 16).ok()?; + let a = u8::from_str_radix(&hex[6..8], 16).ok()?; + Some((r, g, b, a)) +} + +/// Convert RGB to hex color. +pub fn rgb_to_hex(r: u8, g: u8, b: u8) -> String { + format!("#{:02x}{:02x}{:02x}", r, g, b) +} + +/// Convert RGBA to hex color. +pub fn rgba_to_hex(r: u8, g: u8, b: u8, a: u8) -> String { + format!("#{:02x}{:02x}{:02x}{:02x}", r, g, b, a) +} + +/// Convert RGB to HSL. +pub fn rgb_to_hsl(r: u8, g: u8, b: u8) -> (f64, f64, f64) { + let r = r as f64 / 255.0; + let g = g as f64 / 255.0; + let b = b as f64 / 255.0; + + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let l = (max + min) / 2.0; + + if (max - min).abs() < f64::EPSILON { + return (0.0, 0.0, l); + } + + let d = max - min; + let s = if l > 0.5 { + d / (2.0 - max - min) + } else { + d / (max + min) + }; + + let h = if (max - r).abs() < f64::EPSILON { + ((g - b) / d + if g < b { 6.0 } else { 0.0 }) / 6.0 + } else if (max - g).abs() < f64::EPSILON { + ((b - r) / d + 2.0) / 6.0 + } else { + ((r - g) / d + 4.0) / 6.0 + }; + + (h * 360.0, s * 100.0, l * 100.0) +} diff --git a/src/shared/utils/data/convert/mod.rs b/src/shared/utils/data/convert/mod.rs new file mode 100644 index 0000000..01c36f6 --- /dev/null +++ b/src/shared/utils/data/convert/mod.rs @@ -0,0 +1,62 @@ +pub mod bools; +pub mod bytes; +pub mod char; +pub mod collections; +pub mod color; +pub mod network; +pub mod numeric; +pub mod path; +pub mod pointers; +pub mod result; +pub mod string; +pub mod time; + +pub use bools::*; +pub use bytes::*; +pub use char::*; +pub use collections::*; +pub use color::*; +pub use network::*; +pub use numeric::*; +pub use path::*; +pub use pointers::*; +pub use result::*; +pub use string::*; +pub use time::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_integer_conversions() { + assert_eq!(i64_to_u8(300), 255); + assert_eq!(i64_to_u8(-5), 0); + assert_eq!(i128_to_i64(i128::MAX), i64::MAX); + } + + #[test] + fn test_time_conversions() { + assert_eq!(seconds_to_human(90), "1m 30s"); + assert_eq!(seconds_to_compact(86400), "1d"); + } + + #[test] + fn test_color_conversions() { + assert_eq!(hex_to_rgb("#ff8000"), Some((255, 128, 0))); + assert_eq!(rgb_to_hex(255, 128, 0), "#ff8000"); + } + + #[test] + fn test_network_conversions() { + assert_eq!(ipv4_to_u32("192.168.1.1"), Some(0xC0A80101)); + assert_eq!(u32_to_ipv4(0xC0A80101), "192.168.1.1"); + } + + #[test] + fn test_path_conversions() { + let path = str_to_path("/test/path.txt"); + assert_eq!(path_extension(&path), Some("txt".to_string())); + assert_eq!(path_filename(&path), Some("path.txt".to_string())); + } +} diff --git a/src/shared/utils/data/convert/network.rs b/src/shared/utils/data/convert/network.rs new file mode 100644 index 0000000..3115475 --- /dev/null +++ b/src/shared/utils/data/convert/network.rs @@ -0,0 +1,60 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; + +/// Convert IPv4 string to u32. +pub fn ipv4_to_u32(ip: &str) -> Option { + ip.parse::().ok().map(|addr| u32::from(addr)) +} + +/// Convert u32 to IPv4 string. +pub fn u32_to_ipv4(n: u32) -> String { + Ipv4Addr::from(n).to_string() +} + +/// Convert u32 to Ipv4Addr. +pub fn u32_to_ipv4_addr(n: u32) -> Ipv4Addr { + Ipv4Addr::from(n) +} + +/// Convert string to IpAddr. +pub fn str_to_ip_addr(s: &str) -> Option { + s.parse().ok() +} + +/// Convert IpAddr to string. +pub fn ip_addr_to_string(ip: IpAddr) -> String { + ip.to_string() +} + +/// Convert string to SocketAddr. +pub fn str_to_socket_addr(s: &str) -> Option { + s.parse().ok() +} + +/// Convert SocketAddr to string. +pub fn socket_addr_to_string(addr: SocketAddr) -> String { + addr.to_string() +} + +/// Create SocketAddrV4 from IP and port. +pub fn ipv4_port_to_socket(ip: Ipv4Addr, port: u16) -> SocketAddrV4 { + SocketAddrV4::new(ip, port) +} + +/// Create SocketAddrV6 from IP and port. +pub fn ipv6_port_to_socket(ip: Ipv6Addr, port: u16) -> SocketAddrV6 { + SocketAddrV6::new(ip, port, 0, 0) +} + +/// Check if IP is loopback. +pub fn is_loopback(ip: &str) -> bool { + ip.parse::() + .map(|a| a.is_loopback()) + .unwrap_or(false) +} + +/// Check if IP is private (IPv4). +pub fn is_private_ip(ip: &str) -> bool { + ip.parse::() + .map(|a| a.is_private()) + .unwrap_or(false) +} diff --git a/src/shared/utils/data/convert/numeric.rs b/src/shared/utils/data/convert/numeric.rs new file mode 100644 index 0000000..2fed348 --- /dev/null +++ b/src/shared/utils/data/convert/numeric.rs @@ -0,0 +1,532 @@ +/// Safe i8 to u8. +pub fn i8_to_u8(n: i8) -> u8 { + n.max(0) as u8 +} + +/// Safe i8 to i16. +pub fn i8_to_i16(n: i8) -> i16 { + n as i16 +} + +/// Safe i8 to i32. +pub fn i8_to_i32(n: i8) -> i32 { + n as i32 +} + +/// Safe i8 to i64. +pub fn i8_to_i64(n: i8) -> i64 { + n as i64 +} + +/// Safe i8 to i128. +pub fn i8_to_i128(n: i8) -> i128 { + n as i128 +} + +/// Safe i8 to f32. +pub fn i8_to_f32(n: i8) -> f32 { + n as f32 +} + +/// Safe i8 to f64. +pub fn i8_to_f64(n: i8) -> f64 { + n as f64 +} + +/// Safe i16 to i8 (saturates). +pub fn i16_to_i8(n: i16) -> i8 { + n.clamp(i8::MIN as i16, i8::MAX as i16) as i8 +} + +/// Safe i16 to u8 (saturates). +pub fn i16_to_u8(n: i16) -> u8 { + n.clamp(0, u8::MAX as i16) as u8 +} + +/// Safe i16 to u16. +pub fn i16_to_u16(n: i16) -> u16 { + n.max(0) as u16 +} + +/// Safe i16 to i32. +pub fn i16_to_i32(n: i16) -> i32 { + n as i32 +} + +/// Safe i16 to i64. +pub fn i16_to_i64(n: i16) -> i64 { + n as i64 +} + +/// Safe i16 to i128. +pub fn i16_to_i128(n: i16) -> i128 { + n as i128 +} + +/// Safe i32 to i8 (saturates). +pub fn i32_to_i8(n: i32) -> i8 { + n.clamp(i8::MIN as i32, i8::MAX as i32) as i8 +} + +/// Safe i32 to i16 (saturates). +pub fn i32_to_i16(n: i32) -> i16 { + n.clamp(i16::MIN as i32, i16::MAX as i32) as i16 +} + +/// Safe i32 to u8 (saturates). +pub fn i32_to_u8(n: i32) -> u8 { + n.clamp(0, u8::MAX as i32) as u8 +} + +/// Safe i32 to u16 (saturates). +pub fn i32_to_u16(n: i32) -> u16 { + n.clamp(0, u16::MAX as i32) as u16 +} + +/// Safe i32 to u32. +pub fn i32_to_u32(n: i32) -> u32 { + n.max(0) as u32 +} + +/// Safe i32 to i64. +pub fn i32_to_i64(n: i32) -> i64 { + n as i64 +} + +/// Safe i32 to i128. +pub fn i32_to_i128(n: i32) -> i128 { + n as i128 +} + +/// Safe i32 to usize. +pub fn i32_to_usize(n: i32) -> usize { + n.max(0) as usize +} + +/// Safe i32 to f32. +pub fn i32_to_f32(n: i32) -> f32 { + n as f32 +} + +/// Safe i32 to f64. +pub fn i32_to_f64(n: i32) -> f64 { + n as f64 +} + +/// Safe i64 to i8 (saturates). +pub fn i64_to_i8(n: i64) -> i8 { + n.clamp(i8::MIN as i64, i8::MAX as i64) as i8 +} + +/// Safe i64 to i16 (saturates). +pub fn i64_to_i16(n: i64) -> i16 { + n.clamp(i16::MIN as i64, i16::MAX as i64) as i16 +} + +/// Safe i64 to i32 (saturates). +pub fn i64_to_i32(n: i64) -> i32 { + n.clamp(i32::MIN as i64, i32::MAX as i64) as i32 +} + +/// Safe i64 to u8 (saturates). +pub fn i64_to_u8(n: i64) -> u8 { + n.clamp(0, u8::MAX as i64) as u8 +} + +/// Safe i64 to u16 (saturates). +pub fn i64_to_u16(n: i64) -> u16 { + n.clamp(0, u16::MAX as i64) as u16 +} + +/// Safe i64 to u32 (saturates). +pub fn i64_to_u32(n: i64) -> u32 { + n.clamp(0, u32::MAX as i64) as u32 +} + +/// Safe i64 to u64. +pub fn i64_to_u64(n: i64) -> u64 { + n.max(0) as u64 +} + +/// Safe i64 to usize. +pub fn i64_to_usize(n: i64) -> usize { + n.max(0) as usize +} + +/// Safe i64 to i128. +pub fn i64_to_i128(n: i64) -> i128 { + n as i128 +} + +/// Safe i64 to u128. +pub fn i64_to_u128(n: i64) -> u128 { + n.max(0) as u128 +} + +/// Safe i64 to f32. +pub fn i64_to_f32(n: i64) -> f32 { + n as f32 +} + +/// Safe i64 to f64. +pub fn i64_to_f64(n: i64) -> f64 { + n as f64 +} + +/// Safe i128 to i8 (saturates). +pub fn i128_to_i8(n: i128) -> i8 { + n.clamp(i8::MIN as i128, i8::MAX as i128) as i8 +} + +/// Safe i128 to i16 (saturates). +pub fn i128_to_i16(n: i128) -> i16 { + n.clamp(i16::MIN as i128, i16::MAX as i128) as i16 +} + +/// Safe i128 to i32 (saturates). +pub fn i128_to_i32(n: i128) -> i32 { + n.clamp(i32::MIN as i128, i32::MAX as i128) as i32 +} + +/// Safe i128 to i64 (saturates). +pub fn i128_to_i64(n: i128) -> i64 { + n.clamp(i64::MIN as i128, i64::MAX as i128) as i64 +} + +/// Safe i128 to u128. +pub fn i128_to_u128(n: i128) -> u128 { + n.max(0) as u128 +} + +/// Safe i128 to usize. +pub fn i128_to_usize(n: i128) -> usize { + n.clamp(0, usize::MAX as i128) as usize +} + +/// u8 to i8 (may overflow to negative). +pub fn u8_to_i8_wrap(n: u8) -> i8 { + n as i8 +} + +/// u8 to i8 (saturates at i8::MAX). +pub fn u8_to_i8_sat(n: u8) -> i8 { + n.min(i8::MAX as u8) as i8 +} + +/// u8 to i16. +pub fn u8_to_i16(n: u8) -> i16 { + n as i16 +} + +/// u8 to i32. +pub fn u8_to_i32(n: u8) -> i32 { + n as i32 +} + +/// u8 to i64. +pub fn u8_to_i64(n: u8) -> i64 { + n as i64 +} + +/// u8 to u16. +pub fn u8_to_u16(n: u8) -> u16 { + n as u16 +} + +/// u8 to u32. +pub fn u8_to_u32(n: u8) -> u32 { + n as u32 +} + +/// u8 to u64. +pub fn u8_to_u64(n: u8) -> u64 { + n as u64 +} + +/// u8 to usize. +pub fn u8_to_usize(n: u8) -> usize { + n as usize +} + +/// u8 to f32. +pub fn u8_to_f32(n: u8) -> f32 { + n as f32 +} + +/// u8 to f64. +pub fn u8_to_f64(n: u8) -> f64 { + n as f64 +} + +/// u16 to u8 (saturates). +pub fn u16_to_u8(n: u16) -> u8 { + n.min(u8::MAX as u16) as u8 +} + +/// u16 to i16 (saturates). +pub fn u16_to_i16(n: u16) -> i16 { + n.min(i16::MAX as u16) as i16 +} + +/// u16 to i32. +pub fn u16_to_i32(n: u16) -> i32 { + n as i32 +} + +/// u16 to i64. +pub fn u16_to_i64(n: u16) -> i64 { + n as i64 +} + +/// u16 to u32. +pub fn u16_to_u32(n: u16) -> u32 { + n as u32 +} + +/// u16 to u64. +pub fn u16_to_u64(n: u16) -> u64 { + n as u64 +} + +/// u16 to usize. +pub fn u16_to_usize(n: u16) -> usize { + n as usize +} + +/// u32 to u8 (saturates). +pub fn u32_to_u8(n: u32) -> u8 { + n.min(u8::MAX as u32) as u8 +} + +/// u32 to u16 (saturates). +pub fn u32_to_u16(n: u32) -> u16 { + n.min(u16::MAX as u32) as u16 +} + +/// u32 to i32 (saturates). +pub fn u32_to_i32(n: u32) -> i32 { + n.min(i32::MAX as u32) as i32 +} + +/// u32 to i64. +pub fn u32_to_i64(n: u32) -> i64 { + n as i64 +} + +/// u32 to u64. +pub fn u32_to_u64(n: u32) -> u64 { + n as u64 +} + +/// u32 to usize. +pub fn u32_to_usize(n: u32) -> usize { + n as usize +} + +/// u32 to f32. +pub fn u32_to_f32(n: u32) -> f32 { + n as f32 +} + +/// u32 to f64. +pub fn u32_to_f64(n: u32) -> f64 { + n as f64 +} + +/// u64 to u8 (saturates). +pub fn u64_to_u8(n: u64) -> u8 { + n.min(u8::MAX as u64) as u8 +} + +/// u64 to u16 (saturates). +pub fn u64_to_u16(n: u64) -> u16 { + n.min(u16::MAX as u64) as u16 +} + +/// u64 to u32 (saturates). +pub fn u64_to_u32(n: u64) -> u32 { + n.min(u32::MAX as u64) as u32 +} + +/// u64 to i64 (saturates). +pub fn u64_to_i64(n: u64) -> i64 { + n.min(i64::MAX as u64) as i64 +} + +/// u64 to i128. +pub fn u64_to_i128(n: u64) -> i128 { + n as i128 +} + +/// u64 to u128. +pub fn u64_to_u128(n: u64) -> u128 { + n as u128 +} + +/// u64 to usize (may truncate on 32-bit). +pub fn u64_to_usize(n: u64) -> usize { + n as usize +} + +/// u64 to f64. +pub fn u64_to_f64(n: u64) -> f64 { + n as f64 +} + +/// u128 to u64 (saturates). +pub fn u128_to_u64(n: u128) -> u64 { + n.min(u64::MAX as u128) as u64 +} + +/// u128 to i128 (saturates). +pub fn u128_to_i128(n: u128) -> i128 { + n.min(i128::MAX as u128) as i128 +} + +/// u128 to usize (saturates). +pub fn u128_to_usize(n: u128) -> usize { + n.min(usize::MAX as u128) as usize +} + +/// usize to i32 (saturates). +pub fn usize_to_i32(n: usize) -> i32 { + n.min(i32::MAX as usize) as i32 +} + +/// usize to i64. +pub fn usize_to_i64(n: usize) -> i64 { + n as i64 +} + +/// usize to u32 (saturates on 64-bit). +pub fn usize_to_u32(n: usize) -> u32 { + n.min(u32::MAX as usize) as u32 +} + +/// usize to u64. +pub fn usize_to_u64(n: usize) -> u64 { + n as u64 +} + +/// isize to i32 (saturates). +pub fn isize_to_i32(n: isize) -> i32 { + n.clamp(i32::MIN as isize, i32::MAX as isize) as i32 +} + +/// isize to i64. +pub fn isize_to_i64(n: isize) -> i64 { + n as i64 +} + +/// isize to usize. +pub fn isize_to_usize(n: isize) -> usize { + n.max(0) as usize +} + +/// f64 to f32 (may lose precision). +pub fn f64_to_f32(n: f64) -> f32 { + n as f32 +} + +/// f32 to f64. +pub fn f32_to_f64(n: f32) -> f64 { + n as f64 +} + +/// f64 to i64 (truncates). +pub fn f64_to_i64(n: f64) -> i64 { + n.clamp(i64::MIN as f64, i64::MAX as f64) as i64 +} + +/// f64 to i32 (truncates). +pub fn f64_to_i32(n: f64) -> i32 { + n.clamp(i32::MIN as f64, i32::MAX as f64) as i32 +} + +/// f64 to u64 (truncates). +pub fn f64_to_u64(n: f64) -> u64 { + n.clamp(0.0, u64::MAX as f64) as u64 +} + +/// f64 to u32 (truncates). +pub fn f64_to_u32(n: f64) -> u32 { + n.clamp(0.0, u32::MAX as f64) as u32 +} + +/// f32 to i32 (truncates). +pub fn f32_to_i32(n: f32) -> i32 { + n.clamp(i32::MIN as f32, i32::MAX as f32) as i32 +} + +/// Round f64 to n decimal places. +pub fn round_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).round() / factor +} + +/// Round f32 to n decimal places. +pub fn round_f32(n: f32, decimals: u32) -> f32 { + let factor = 10_f32.powi(decimals as i32); + (n * factor).round() / factor +} + +/// Truncate f64 to n decimal places. +pub fn trunc_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).trunc() / factor +} + +/// Ceil f64 to n decimal places. +pub fn ceil_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).ceil() / factor +} + +/// Floor f64 to n decimal places. +pub fn floor_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).floor() / factor +} + +/// Check if f64 is essentially zero. +pub fn is_zero(n: f64, epsilon: f64) -> bool { + n.abs() < epsilon +} + +/// Compare two f64 for approximate equality. +pub fn approx_eq(a: f64, b: f64, epsilon: f64) -> bool { + (a - b).abs() < epsilon +} + +/// Check if f64 is NaN. +pub fn is_nan(n: f64) -> bool { + n.is_nan() +} + +/// Check if f64 is infinite. +pub fn is_infinite(n: f64) -> bool { + n.is_infinite() +} + +/// Check if f64 is finite. +pub fn is_finite(n: f64) -> bool { + n.is_finite() +} + +/// Convert NaN to 0. +pub fn nan_to_zero(n: f64) -> f64 { + if n.is_nan() { + 0.0 + } else { + n + } +} + +/// Convert NaN to default. +pub fn nan_to_default(n: f64, default: f64) -> f64 { + if n.is_nan() { + default + } else { + n + } +} diff --git a/src/shared/utils/data/convert/path.rs b/src/shared/utils/data/convert/path.rs new file mode 100644 index 0000000..19c3db2 --- /dev/null +++ b/src/shared/utils/data/convert/path.rs @@ -0,0 +1,62 @@ +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; + +/// Convert &str to PathBuf. +pub fn str_to_path(s: &str) -> PathBuf { + PathBuf::from(s) +} + +/// Convert String to PathBuf. +pub fn string_to_path(s: String) -> PathBuf { + PathBuf::from(s) +} + +/// Convert PathBuf to String (lossy). +pub fn path_to_string(p: &Path) -> String { + p.to_string_lossy().to_string() +} + +/// Convert PathBuf to Option (None if not valid UTF-8). +pub fn path_to_string_strict(p: &Path) -> Option { + p.to_str().map(String::from) +} + +/// Convert &str to &Path. +pub fn str_to_path_ref(s: &str) -> &Path { + Path::new(s) +} + +/// Convert OsStr to String (lossy). +pub fn os_str_to_string(s: &OsStr) -> String { + s.to_string_lossy().to_string() +} + +/// Convert OsString to String (lossy). +pub fn os_string_to_string(s: OsString) -> String { + s.to_string_lossy().to_string() +} + +/// Convert String to OsString. +pub fn string_to_os_string(s: String) -> OsString { + OsString::from(s) +} + +/// Convert &str to &OsStr. +pub fn str_to_os_str(s: &str) -> &OsStr { + OsStr::new(s) +} + +/// Get file extension as String. +pub fn path_extension(p: &Path) -> Option { + p.extension().and_then(|e| e.to_str()).map(String::from) +} + +/// Get file name as String. +pub fn path_filename(p: &Path) -> Option { + p.file_name().and_then(|n| n.to_str()).map(String::from) +} + +/// Get parent directory as PathBuf. +pub fn path_parent(p: &Path) -> Option { + p.parent().map(PathBuf::from) +} diff --git a/src/shared/utils/data/convert/pointers.rs b/src/shared/utils/data/convert/pointers.rs new file mode 100644 index 0000000..8d309e5 --- /dev/null +++ b/src/shared/utils/data/convert/pointers.rs @@ -0,0 +1,58 @@ +use std::borrow::Cow; +use std::rc::Rc; +use std::sync::Arc; + +/// Wrap value in Box. +pub fn to_box(value: T) -> Box { + Box::new(value) +} + +/// Wrap value in Rc. +pub fn to_rc(value: T) -> Rc { + Rc::new(value) +} + +/// Wrap value in Arc. +pub fn to_arc(value: T) -> Arc { + Arc::new(value) +} + +/// Convert Box to T (unbox). +pub fn unbox(boxed: Box) -> T { + *boxed +} + +/// Clone from Rc. +pub fn rc_to_owned(rc: &Rc) -> T { + (**rc).clone() +} + +/// Clone from Arc. +pub fn arc_to_owned(arc: &Arc) -> T { + (**arc).clone() +} + +/// Convert &str to Cow. +pub fn str_to_cow(s: &str) -> Cow<'_, str> { + Cow::Borrowed(s) +} + +/// Convert String to Cow. +pub fn string_to_cow(s: String) -> Cow<'static, str> { + Cow::Owned(s) +} + +/// Convert Cow to String. +pub fn cow_to_string(cow: Cow<'_, str>) -> String { + cow.into_owned() +} + +/// Convert &[T] to Cow<[T]>. +pub fn slice_to_cow(s: &[T]) -> Cow<'_, [T]> { + Cow::Borrowed(s) +} + +/// Convert Vec to Cow<[T]>. +pub fn vec_to_cow(v: Vec) -> Cow<'static, [T]> { + Cow::Owned(v) +} diff --git a/src/shared/utils/data/convert/result.rs b/src/shared/utils/data/convert/result.rs new file mode 100644 index 0000000..a736234 --- /dev/null +++ b/src/shared/utils/data/convert/result.rs @@ -0,0 +1,39 @@ +/// Convert Option to Result. +pub fn option_to_result(opt: Option, err: E) -> Result { + opt.ok_or(err) +} + +/// Convert Option to Result. +pub fn option_to_result_str(opt: Option, msg: &str) -> Result { + opt.ok_or_else(|| msg.to_string()) +} + +/// Convert Result to Option. +pub fn result_to_option(res: Result) -> Option { + res.ok() +} + +/// Convert Result to Option. +pub fn result_to_err(res: Result) -> Option { + res.err() +} + +/// Flatten nested Option. +pub fn flatten_option(opt: Option>) -> Option { + opt.flatten() +} + +/// Flatten nested Result. +pub fn flatten_result(res: Result, E>) -> Result { + res.and_then(|r| r) +} + +/// Transpose Option> to Result, E>. +pub fn transpose_option_result(opt: Option>) -> Result, E> { + opt.transpose() +} + +/// Transpose Result, E> to Option>. +pub fn transpose_result_option(res: Result, E>) -> Option> { + res.transpose() +} diff --git a/src/shared/utils/data/convert/string.rs b/src/shared/utils/data/convert/string.rs new file mode 100644 index 0000000..e01af9d --- /dev/null +++ b/src/shared/utils/data/convert/string.rs @@ -0,0 +1,134 @@ +use std::fmt::Display; +use std::str::FromStr; + +/// Parse with default value. +pub fn parse_or(s: &str, default: T) -> T { + s.parse().unwrap_or(default) +} + +/// Try parse with error context. +pub fn try_parse(s: &str, name: &str) -> Result +where + T::Err: Display, +{ + s.parse() + .map_err(|e| format!("Failed to parse {}: {}", name, e)) +} + +/// Parse i8 with default. +pub fn parse_i8(s: &str, default: i8) -> i8 { + s.parse().unwrap_or(default) +} + +/// Parse i16 with default. +pub fn parse_i16(s: &str, default: i16) -> i16 { + s.parse().unwrap_or(default) +} + +/// Parse i32 with default. +pub fn parse_i32(s: &str, default: i32) -> i32 { + s.parse().unwrap_or(default) +} + +/// Parse i64 with default. +pub fn parse_i64(s: &str, default: i64) -> i64 { + s.parse().unwrap_or(default) +} + +/// Parse i128 with default. +pub fn parse_i128(s: &str, default: i128) -> i128 { + s.parse().unwrap_or(default) +} + +/// Parse u8 with default. +pub fn parse_u8(s: &str, default: u8) -> u8 { + s.parse().unwrap_or(default) +} + +/// Parse u16 with default. +pub fn parse_u16(s: &str, default: u16) -> u16 { + s.parse().unwrap_or(default) +} + +/// Parse u32 with default. +pub fn parse_u32(s: &str, default: u32) -> u32 { + s.parse().unwrap_or(default) +} + +/// Parse u64 with default. +pub fn parse_u64(s: &str, default: u64) -> u64 { + s.parse().unwrap_or(default) +} + +/// Parse u128 with default. +pub fn parse_u128(s: &str, default: u128) -> u128 { + s.parse().unwrap_or(default) +} + +/// Parse f32 with default. +pub fn parse_f32(s: &str, default: f32) -> f32 { + s.parse().unwrap_or(default) +} + +/// Parse f64 with default. +pub fn parse_f64(s: &str, default: f64) -> f64 { + s.parse().unwrap_or(default) +} + +/// Parse usize with default. +pub fn parse_usize(s: &str, default: usize) -> usize { + s.parse().unwrap_or(default) +} + +/// Parse isize with default. +pub fn parse_isize(s: &str, default: isize) -> isize { + s.parse().unwrap_or(default) +} + +/// Convert string to Option (None for empty). +pub fn empty_to_none(s: &str) -> Option { + if s.is_empty() { + None + } else { + Some(s.to_string()) + } +} + +/// Convert None to empty string. +pub fn none_to_empty(opt: Option) -> String { + opt.unwrap_or_default() +} + +/// Convert Option<&str> to Option. +pub fn str_to_string(opt: Option<&str>) -> Option { + opt.map(|s| s.to_string()) +} + +/// Convert String to Option (None if empty). +pub fn string_to_option(s: String) -> Option { + if s.is_empty() { + None + } else { + Some(s) + } +} + +/// Trim and convert to Option (None if whitespace only). +pub fn trim_to_option(s: &str) -> Option { + let trimmed = s.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } +} + +/// Convert &str to String. +pub fn str_to_owned(s: &str) -> String { + s.to_string() +} + +/// Convert String to &str (returns empty if none). +pub fn string_to_str(s: &Option) -> &str { + s.as_deref().unwrap_or("") +} diff --git a/src/shared/utils/data/convert/time.rs b/src/shared/utils/data/convert/time.rs new file mode 100644 index 0000000..55f5e17 --- /dev/null +++ b/src/shared/utils/data/convert/time.rs @@ -0,0 +1,142 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Convert seconds to human readable duration. +pub fn seconds_to_human(secs: u64) -> String { + if secs < 60 { + format!("{}s", secs) + } else if secs < 3600 { + format!("{}m {}s", secs / 60, secs % 60) + } else if secs < 86400 { + format!("{}h {}m", secs / 3600, (secs % 3600) / 60) + } else if secs < 604800 { + format!("{}d {}h", secs / 86400, (secs % 86400) / 3600) + } else if secs < 2592000 { + format!("{}w {}d", secs / 604800, (secs % 604800) / 86400) + } else if secs < 31536000 { + format!("{}mo {}d", secs / 2592000, (secs % 2592000) / 86400) + } else { + format!("{}y {}mo", secs / 31536000, (secs % 31536000) / 2592000) + } +} + +/// Convert seconds to compact human readable. +pub fn seconds_to_compact(secs: u64) -> String { + if secs < 60 { + format!("{}s", secs) + } else if secs < 3600 { + format!("{}m", secs / 60) + } else if secs < 86400 { + format!("{}h", secs / 3600) + } else if secs < 604800 { + format!("{}d", secs / 86400) + } else if secs < 2592000 { + format!("{}w", secs / 604800) + } else if secs < 31536000 { + format!("{}mo", secs / 2592000) + } else { + format!("{}y", secs / 31536000) + } +} + +/// Convert milliseconds to human readable. +pub fn ms_to_human(ms: u64) -> String { + if ms < 1000 { + format!("{}ms", ms) + } else { + seconds_to_human(ms / 1000) + } +} + +/// Convert microseconds to human readable. +pub fn us_to_human(us: u64) -> String { + if us < 1000 { + format!("{}μs", us) + } else if us < 1_000_000 { + format!("{:.2}ms", us as f64 / 1000.0) + } else { + seconds_to_human(us / 1_000_000) + } +} + +/// Convert nanoseconds to human readable. +pub fn ns_to_human(ns: u64) -> String { + if ns < 1_000 { + format!("{}ns", ns) + } else if ns < 1_000_000 { + format!("{:.2}μs", ns as f64 / 1_000.0) + } else if ns < 1_000_000_000 { + format!("{:.2}ms", ns as f64 / 1_000_000.0) + } else { + format!("{:.2}s", ns as f64 / 1_000_000_000.0) + } +} + +/// Convert seconds to Duration. +pub fn secs_to_duration(secs: u64) -> Duration { + Duration::from_secs(secs) +} + +/// Convert milliseconds to Duration. +pub fn ms_to_duration(ms: u64) -> Duration { + Duration::from_millis(ms) +} + +/// Convert microseconds to Duration. +pub fn us_to_duration(us: u64) -> Duration { + Duration::from_micros(us) +} + +/// Convert nanoseconds to Duration. +pub fn ns_to_duration(ns: u64) -> Duration { + Duration::from_nanos(ns) +} + +/// Convert Duration to seconds. +pub fn duration_to_secs(d: Duration) -> u64 { + d.as_secs() +} + +/// Convert Duration to milliseconds. +pub fn duration_to_ms(d: Duration) -> u128 { + d.as_millis() +} + +/// Convert Duration to microseconds. +pub fn duration_to_us(d: Duration) -> u128 { + d.as_micros() +} + +/// Convert Duration to nanoseconds. +pub fn duration_to_ns(d: Duration) -> u128 { + d.as_nanos() +} + +/// Convert Duration to f64 seconds. +pub fn duration_to_secs_f64(d: Duration) -> f64 { + d.as_secs_f64() +} + +/// Convert f64 seconds to Duration. +pub fn secs_f64_to_duration(secs: f64) -> Duration { + Duration::from_secs_f64(secs.max(0.0)) +} + +/// Get SystemTime as Unix timestamp (seconds). +pub fn system_time_to_unix(t: SystemTime) -> u64 { + t.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} + +/// Get SystemTime as Unix timestamp (milliseconds). +pub fn system_time_to_unix_ms(t: SystemTime) -> u128 { + t.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() +} + +/// Convert Unix timestamp to SystemTime. +pub fn unix_to_system_time(secs: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(secs) +} + +/// Convert Unix milliseconds to SystemTime. +pub fn unix_ms_to_system_time(ms: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_millis(ms) +} diff --git a/src/shared/utils/data/datetime.rs b/src/shared/utils/data/datetime.rs new file mode 100644 index 0000000..53144ce --- /dev/null +++ b/src/shared/utils/data/datetime.rs @@ -0,0 +1,107 @@ +//! Date and time utilities. + +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; + +/// Get current UTC timestamp. +pub fn now() -> DateTime { + Utc::now() +} + +/// Get current Unix timestamp (seconds). +pub fn timestamp() -> i64 { + Utc::now().timestamp() +} + +/// Get current Unix timestamp (milliseconds). +pub fn timestamp_millis() -> i64 { + Utc::now().timestamp_millis() +} + +/// Format datetime as ISO 8601 string. +pub fn to_iso(dt: DateTime) -> String { + dt.to_rfc3339() +} + +/// Format datetime as human-readable string. +pub fn to_human(dt: DateTime) -> String { + dt.format("%Y-%m-%d %H:%M:%S").to_string() +} + +/// Format datetime as date only. +pub fn to_date(dt: DateTime) -> String { + dt.format("%Y-%m-%d").to_string() +} + +/// Parse ISO 8601 string to DateTime. +pub fn parse_iso(s: &str) -> Option> { + DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +/// Add duration to a datetime. +pub fn add_days(dt: DateTime, days: i64) -> DateTime { + dt + Duration::days(days) +} + +/// Add hours to a datetime. +pub fn add_hours(dt: DateTime, hours: i64) -> DateTime { + dt + Duration::hours(hours) +} + +/// Add minutes to a datetime. +pub fn add_minutes(dt: DateTime, minutes: i64) -> DateTime { + dt + Duration::minutes(minutes) +} + +/// Check if datetime is in the past. +pub fn is_past(dt: DateTime) -> bool { + dt < Utc::now() +} + +/// Check if datetime is in the future. +pub fn is_future(dt: DateTime) -> bool { + dt > Utc::now() +} + +/// Get relative time string (e.g., "2 hours ago"). +pub fn relative(dt: DateTime) -> String { + let duration = Utc::now().signed_duration_since(dt); + + if duration.num_seconds() < 60 { + "just now".to_string() + } else if duration.num_minutes() < 60 { + format!("{} minutes ago", duration.num_minutes()) + } else if duration.num_hours() < 24 { + format!("{} hours ago", duration.num_hours()) + } else if duration.num_days() < 30 { + format!("{} days ago", duration.num_days()) + } else if duration.num_days() < 365 { + format!("{} months ago", duration.num_days() / 30) + } else { + format!("{} years ago", duration.num_days() / 365) + } +} + +/// Calculate age in years from birthdate. +pub fn age_years(birthdate: NaiveDateTime) -> i32 { + let today = Utc::now().naive_utc(); + let years = today.date().years_since(birthdate.date()); + years.map(|y| y as i32).unwrap_or(0) +} + +/// Get start of day (00:00:00). +pub fn start_of_day(dt: DateTime) -> DateTime { + dt.date_naive() + .and_hms_opt(0, 0, 0) + .map(|naive| DateTime::::from_naive_utc_and_offset(naive, Utc)) + .unwrap_or(dt) +} + +/// Get end of day (23:59:59). +pub fn end_of_day(dt: DateTime) -> DateTime { + dt.date_naive() + .and_hms_opt(23, 59, 59) + .map(|naive| DateTime::::from_naive_utc_and_offset(naive, Utc)) + .unwrap_or(dt) +} diff --git a/src/shared/utils/data/json.rs b/src/shared/utils/data/json.rs new file mode 100644 index 0000000..f8f1f1e --- /dev/null +++ b/src/shared/utils/data/json.rs @@ -0,0 +1,137 @@ +//! JSON utilities. + +use serde::{de::DeserializeOwned, Serialize}; +use serde_json::{Map, Value}; + +/// Parse JSON string to type. +pub fn parse(json: &str) -> Result { + serde_json::from_str(json) +} + +/// Serialize type to JSON string. +pub fn stringify(value: &T) -> Result { + serde_json::to_string(value) +} + +/// Serialize type to pretty JSON string. +pub fn stringify_pretty(value: &T) -> Result { + serde_json::to_string_pretty(value) +} + +/// Merge two JSON objects (second overwrites first). +pub fn merge(base: Value, overlay: Value) -> Value { + match (base, overlay) { + (Value::Object(mut base_map), Value::Object(overlay_map)) => { + for (key, value) in overlay_map { + base_map.insert(key, value); + } + Value::Object(base_map) + } + (_, overlay) => overlay, + } +} + +/// Deep merge two JSON objects. +pub fn deep_merge(base: Value, overlay: Value) -> Value { + match (base, overlay) { + (Value::Object(mut base_map), Value::Object(overlay_map)) => { + for (key, overlay_value) in overlay_map { + let base_value = base_map.remove(&key).unwrap_or(Value::Null); + base_map.insert(key, deep_merge(base_value, overlay_value)); + } + Value::Object(base_map) + } + (_, overlay) => overlay, + } +} + +/// Extract a value at a path (e.g., "data.items.0.name"). +pub fn get_path<'a>(value: &'a Value, path: &str) -> Option<&'a Value> { + let mut current = value; + for key in path.split('.') { + current = match current { + Value::Object(map) => map.get(key)?, + Value::Array(arr) => { + let idx: usize = key.parse().ok()?; + arr.get(idx)? + } + _ => return None, + }; + } + Some(current) +} + +/// Extract string at path. +pub fn get_str<'a>(value: &'a Value, path: &str) -> Option<&'a str> { + get_path(value, path).and_then(|v| v.as_str()) +} + +/// Extract i64 at path. +pub fn get_i64(value: &Value, path: &str) -> Option { + get_path(value, path).and_then(|v| v.as_i64()) +} + +/// Extract bool at path. +pub fn get_bool(value: &Value, path: &str) -> Option { + get_path(value, path).and_then(|v| v.as_bool()) +} + +/// Extract array at path. +pub fn get_array<'a>(value: &'a Value, path: &str) -> Option<&'a Vec> { + get_path(value, path).and_then(|v| v.as_array()) +} + +/// Create a JSON object with key-value pairs. +#[macro_export] +macro_rules! json_object { + ($($key:expr => $value:expr),* $(,)?) => {{ + let mut map = serde_json::Map::new(); + $( + map.insert($key.to_string(), serde_json::json!($value)); + )* + serde_json::Value::Object(map) + }}; +} + +/// Check if value is empty (null, empty string, empty array, empty object). +pub fn is_empty(value: &Value) -> bool { + match value { + Value::Null => true, + Value::String(s) => s.is_empty(), + Value::Array(arr) => arr.is_empty(), + Value::Object(obj) => obj.is_empty(), + _ => false, + } +} + +/// Remove null values from object. +pub fn remove_nulls(value: Value) -> Value { + match value { + Value::Object(map) => { + let filtered: Map = map + .into_iter() + .filter(|(_, v)| !v.is_null()) + .map(|(k, v)| (k, remove_nulls(v))) + .collect(); + Value::Object(filtered) + } + Value::Array(arr) => Value::Array(arr.into_iter().map(remove_nulls).collect()), + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_get_path() { + let value = json!({ + "data": { + "items": [{"name": "test"}] + } + }); + assert_eq!(get_str(&value, "data.items.0.name"), Some("test")); + } +} diff --git a/src/shared/utils/data/mod.rs b/src/shared/utils/data/mod.rs new file mode 100644 index 0000000..59dda88 --- /dev/null +++ b/src/shared/utils/data/mod.rs @@ -0,0 +1,7 @@ +pub mod collections; +pub mod convert; +pub mod datetime; +pub mod json; +pub mod numbers; +pub mod string; +pub mod text; diff --git a/src/shared/utils/data/numbers.rs b/src/shared/utils/data/numbers.rs new file mode 100644 index 0000000..690d8b5 --- /dev/null +++ b/src/shared/utils/data/numbers.rs @@ -0,0 +1,151 @@ +//! Number utilities. + +/// Format number with thousand separators. +pub fn format_number(n: i64) -> String { + let s = n.abs().to_string(); + let chars: Vec = s.chars().rev().collect(); + let mut result = String::new(); + + for (i, c) in chars.iter().enumerate() { + if i > 0 && i % 3 == 0 { + result.push(','); + } + result.push(*c); + } + + if n < 0 { + result.push('-'); + } + + result.chars().rev().collect() +} + +/// Format as percentage. +pub fn format_percent(value: f64, decimals: usize) -> String { + format!("{:.1$}%", value * 100.0, decimals) +} + +/// Format as currency. +pub fn format_currency(value: f64, symbol: &str) -> String { + format!("{}{:.2}", symbol, value) +} + +/// Format bytes to human readable. +pub fn format_bytes(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + const TB: u64 = GB * 1024; + + if bytes >= TB { + format!("{:.2} TB", bytes as f64 / TB as f64) + } else if bytes >= GB { + format!("{:.2} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.2} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.2} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} + +/// Clamp value to range. +pub fn clamp(value: T, min: T, max: T) -> T { + if value < min { + min + } else if value > max { + max + } else { + value + } +} + +/// Linear interpolation. +pub fn lerp(a: f64, b: f64, t: f64) -> f64 { + a + (b - a) * t +} + +/// Map value from one range to another. +pub fn map_range(value: f64, from_min: f64, from_max: f64, to_min: f64, to_max: f64) -> f64 { + let from_range = from_max - from_min; + let to_range = to_max - to_min; + ((value - from_min) / from_range) * to_range + to_min +} + +/// Round to n decimal places. +pub fn round_to(value: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (value * factor).round() / factor +} + +/// Check if number is even. +pub fn is_even(n: i64) -> bool { + n % 2 == 0 +} + +/// Check if number is odd. +pub fn is_odd(n: i64) -> bool { + n % 2 != 0 +} + +/// Check if number is positive. +pub fn is_positive(n: T) -> bool { + n > T::default() +} + +/// Check if number is negative. +pub fn is_negative(n: T) -> bool { + n < T::default() +} + +/// Safe division (returns 0 if divisor is 0). +pub fn safe_div(a: f64, b: f64) -> f64 { + if b == 0.0 { + 0.0 + } else { + a / b + } +} + +/// Calculate percentage. +pub fn percentage(part: f64, total: f64) -> f64 { + safe_div(part, total) * 100.0 +} + +/// Parse string to i64 with default. +pub fn parse_i64(s: &str, default: i64) -> i64 { + s.parse().unwrap_or(default) +} + +/// Parse string to f64 with default. +pub fn parse_f64(s: &str, default: f64) -> f64 { + s.parse().unwrap_or(default) +} + +/// Generate range of numbers. +pub fn range(start: i64, end: i64) -> Vec { + (start..end).collect() +} + +/// Generate range with step. +pub fn range_step(start: i64, end: i64, step: i64) -> Vec { + (start..end).step_by(step as usize).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_number() { + assert_eq!(format_number(1234567), "1,234,567"); + assert_eq!(format_number(-1234), "-1,234"); + } + + #[test] + fn test_format_bytes() { + assert_eq!(format_bytes(1024), "1.00 KB"); + assert_eq!(format_bytes(1048576), "1.00 MB"); + } +} diff --git a/src/shared/utils/data/string.rs b/src/shared/utils/data/string.rs new file mode 100644 index 0000000..f958559 --- /dev/null +++ b/src/shared/utils/data/string.rs @@ -0,0 +1,127 @@ +//! String utilities. + +use once_cell::sync::Lazy; +use regex::Regex; + +pub fn slugify(s: &str) -> String { + static RE_SPECIAL: Lazy> = + Lazy::new(|| Regex::new(r"[^a-z0-9\s-]")); + static RE_SPACES: Lazy> = Lazy::new(|| Regex::new(r"[\s_]+")); + static RE_HYPHENS: Lazy> = Lazy::new(|| Regex::new(r"-+")); + + let s = s.to_lowercase(); + let s = RE_SPECIAL + .as_ref() + .map(|r| r.replace_all(&s, "").to_string()) + .unwrap_or(s); + let s = RE_SPACES + .as_ref() + .map(|r| r.replace_all(&s, "-").to_string()) + .unwrap_or(s); + let s = RE_HYPHENS + .as_ref() + .map(|r| r.replace_all(&s, "-").to_string()) + .unwrap_or(s); + s.trim_matches('-').to_string() +} + +pub fn truncate(s: &str, max_len: usize) -> String { + if s.len() <= max_len { + s.to_string() + } else if max_len <= 3 { + s.chars().take(max_len).collect() + } else { + format!("{}...", s.chars().take(max_len - 3).collect::()) + } +} + +pub fn initials(name: &str) -> String { + name.split_whitespace() + .filter_map(|word| word.chars().next()) + .take(2) + .collect::() + .to_uppercase() +} + +pub fn mask_email(email: &str) -> String { + if let Some(at_pos) = email.find('@') { + let (local, domain) = email.split_at(at_pos); + if local.len() <= 2 { + format!("{}***{}", local, domain) + } else { + let visible = &local[..2]; + format!("{}***{}", visible, domain) + } + } else { + "***".to_string() + } +} + +pub fn random_string(len: usize) -> String { + use rand::Rng; + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::thread_rng(); + (0..len) + .map(|_| { + let idx = rng.gen_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() +} + +pub fn random_code(len: usize) -> String { + use rand::Rng; + const CHARSET: &[u8] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let mut rng = rand::thread_rng(); + (0..len) + .map(|_| { + let idx = rng.gen_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() +} + +pub fn is_valid_email(email: &str) -> bool { + static RE: Lazy> = + Lazy::new(|| Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")); + RE.as_ref().map(|r| r.is_match(email)).unwrap_or(false) +} + +pub fn title_case(s: &str) -> String { + s.split_whitespace() + .map(|word| { + let mut chars = word.chars(); + match chars.next() { + None => String::new(), + Some(first) => { + first.to_uppercase().collect::() + + chars.as_str().to_lowercase().as_str() + } + } + }) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_slugify() { + assert_eq!(slugify("Hello World!"), "hello-world"); + assert_eq!(slugify(" Multiple Spaces "), "multiple-spaces"); + } + + #[test] + fn test_truncate() { + assert_eq!(truncate("Hello", 10), "Hello"); + assert_eq!(truncate("Hello World", 8), "Hello..."); + } + + #[test] + fn test_initials() { + assert_eq!(initials("John Doe"), "JD"); + assert_eq!(initials("Alice"), "A"); + } +} diff --git a/src/shared/utils/data/text.rs b/src/shared/utils/data/text.rs new file mode 100644 index 0000000..8f28f9b --- /dev/null +++ b/src/shared/utils/data/text.rs @@ -0,0 +1,163 @@ +//! Text processing utilities. + +use once_cell::sync::Lazy; +use regex::Regex; + +pub fn normalize_whitespace(s: &str) -> String { + static WHITESPACE_REGEX: Lazy> = Lazy::new(|| Regex::new(r"\s+")); + WHITESPACE_REGEX + .as_ref() + .map(|r| r.replace_all(s, " ").trim().to_string()) + .unwrap_or_else(|_| s.to_string()) +} + +pub fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().chain(chars).collect(), + } +} + +pub fn to_camel_case(s: &str) -> String { + let words: Vec<&str> = s + .split(|c: char| c == '_' || c == '-' || c.is_whitespace()) + .collect(); + let mut result = String::new(); + for (i, word) in words.iter().enumerate() { + if word.is_empty() { + continue; + } + if i == 0 { + result.push_str(&word.to_lowercase()); + } else { + result.push_str(&capitalize(&word.to_lowercase())); + } + } + result +} + +pub fn to_snake_case(s: &str) -> String { + static CAMEL_REGEX: Lazy> = + Lazy::new(|| Regex::new(r"([a-z])([A-Z])")); + let s = CAMEL_REGEX + .as_ref() + .map(|r| r.replace_all(s, "${1}_${2}").to_string()) + .unwrap_or_else(|_| s.to_string()); + s.replace('-', "_").to_lowercase() +} + +pub fn to_kebab_case(s: &str) -> String { + to_snake_case(s).replace('_', "-") +} + +pub fn to_pascal_case(s: &str) -> String { + s.split(|c: char| c == '_' || c == '-' || c.is_whitespace()) + .filter(|w| !w.is_empty()) + .map(|w| capitalize(&w.to_lowercase())) + .collect() +} + +pub fn to_constant_case(s: &str) -> String { + to_snake_case(s).to_uppercase() +} + +pub fn extract_words(s: &str) -> Vec { + static WORD_REGEX: Lazy> = Lazy::new(|| Regex::new(r"\b\w+\b")); + WORD_REGEX + .as_ref() + .map(|r| r.find_iter(s).map(|m| m.as_str().to_string()).collect()) + .unwrap_or_default() +} + +pub fn word_count(s: &str) -> usize { + extract_words(s).len() +} + +pub fn truncate_words(s: &str, max_words: usize, suffix: &str) -> String { + let words: Vec<&str> = s.split_whitespace().collect(); + if words.len() <= max_words { + s.to_string() + } else { + format!("{}{}", words[..max_words].join(" "), suffix) + } +} + +pub fn wrap_text(s: &str, width: usize) -> String { + let mut result = String::new(); + let mut line_len = 0; + + for word in s.split_whitespace() { + if line_len + word.len() + 1 > width && line_len > 0 { + result.push('\n'); + line_len = 0; + } else if line_len > 0 { + result.push(' '); + line_len += 1; + } + result.push_str(word); + line_len += word.len(); + } + + result +} + +pub fn highlight(text: &str, terms: &[&str], before: &str, after: &str) -> String { + let mut result = text.to_string(); + for term in terms { + let pattern = regex::escape(term); + match Regex::new(&format!("(?i)({})", pattern)) { + Ok(re) => { + result = re + .replace_all(&result, format!("{}$1{}", before, after)) + .to_string(); + } + Err(_) => continue, + } + } + result +} + +/// Remove diacritics/accents. +pub fn remove_accents(s: &str) -> String { + s.chars() + .map(|c| match c { + 'á' | 'à' | 'â' | 'ä' | 'ã' => 'a', + 'é' | 'è' | 'ê' | 'ë' => 'e', + 'í' | 'ì' | 'î' | 'ï' => 'i', + 'ó' | 'ò' | 'ô' | 'ö' | 'õ' => 'o', + 'ú' | 'ù' | 'û' | 'ü' => 'u', + 'ñ' => 'n', + 'ç' => 'c', + _ => c, + }) + .collect() +} + +/// Generate Lorem Ipsum text. +pub fn lorem_ipsum(sentences: usize) -> String { + const LOREM: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \ + Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \ + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. \ + Duis aute irure dolor in reprehenderit in voluptate velit esse cillum. \ + Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia."; + + LOREM + .split(". ") + .take(sentences) + .collect::>() + .join(". ") + + "." +} + +/// Reverse a string. +pub fn reverse(s: &str) -> String { + s.chars().rev().collect() +} + +/// Check if palindrome. +pub fn is_palindrome(s: &str) -> bool { + let clean: String = s.chars().filter(|c| c.is_alphanumeric()).collect(); + let lower = clean.to_lowercase(); + lower == lower.chars().rev().collect::() +} diff --git a/src/shared/utils/dev/async_utils.rs b/src/shared/utils/dev/async_utils.rs new file mode 100644 index 0000000..28c0f6c --- /dev/null +++ b/src/shared/utils/dev/async_utils.rs @@ -0,0 +1,144 @@ +//! Async utilities and helpers. + +use std::future::Future; +use std::time::Duration; +use tokio::time::timeout; + +/// Run with timeout. +pub async fn with_timeout( + duration: Duration, + future: F, +) -> Result +where + F: Future, +{ + timeout(duration, future).await +} + +/// Run with timeout in seconds. +pub async fn timeout_secs(secs: u64, future: F) -> Result +where + F: Future, +{ + timeout(Duration::from_secs(secs), future).await +} + +/// Run with timeout in milliseconds. +pub async fn timeout_ms(ms: u64, future: F) -> Result +where + F: Future, +{ + timeout(Duration::from_millis(ms), future).await +} + +/// Sleep for duration. +pub async fn sleep(duration: Duration) { + tokio::time::sleep(duration).await; +} + +/// Sleep for seconds. +pub async fn sleep_secs(secs: u64) { + tokio::time::sleep(Duration::from_secs(secs)).await; +} + +/// Sleep for milliseconds. +pub async fn sleep_ms(ms: u64) { + tokio::time::sleep(Duration::from_millis(ms)).await; +} + +/// Run multiple futures concurrently and collect results. +pub async fn join_all(futures: Vec) -> Vec +where + F: Future, +{ + futures::future::join_all(futures).await +} + +/// Run futures with concurrency limit. +pub async fn join_all_limited( + items: Vec, + concurrency: usize, + f: F, +) -> Vec +where + F: Fn(T) -> Fut, + Fut: Future, +{ + use futures::stream::{self, StreamExt}; + + stream::iter(items) + .map(f) + .buffer_unordered(concurrency) + .collect() + .await +} + +/// Race multiple futures, returning first to complete. +pub async fn race(f1: F1, f2: F2) -> T +where + F1: Future, + F2: Future, +{ + tokio::select! { + result = f1 => result, + result = f2 => result, + } +} + +/// Retry a future with delay between attempts. +pub async fn simple_retry(attempts: usize, delay: Duration, mut f: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut last_error = None; + for i in 0..attempts { + match f().await { + Ok(result) => return Ok(result), + Err(e) => { + last_error = Some(e); + if i < attempts - 1 { + tokio::time::sleep(delay).await; + } + } + } + } + Err(last_error.unwrap()) +} + +/// Run in blocking thread pool. +pub async fn spawn_blocking(f: F) -> Result +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(f).await +} + +/// Run as a background task (fire and forget). +pub fn spawn(future: F) +where + F: Future + Send + 'static, +{ + tokio::spawn(future); +} + +/// Debounce - only run after delay with no new calls. +pub struct Debouncer { + delay: Duration, +} + +impl Debouncer { + pub fn new(delay: Duration) -> Self { + Self { delay } + } + + pub async fn debounce(&self, f: F) + where + F: FnOnce() -> Fut, + Fut: Future, + { + tokio::time::sleep(self.delay).await; + f().await; + } +} diff --git a/src/shared/utils/dev/logging.rs b/src/shared/utils/dev/logging.rs new file mode 100644 index 0000000..d34e6f4 --- /dev/null +++ b/src/shared/utils/dev/logging.rs @@ -0,0 +1,146 @@ +//! Logging utilities and helpers. + +use std::time::Instant; +use tracing::{debug, error, info, warn}; + +/// Log entry with timing. +pub struct TimedOperation { + name: String, + start: Instant, +} + +impl TimedOperation { + /// Start a timed operation. + pub fn start(name: impl Into) -> Self { + let name = name.into(); + info!("Starting: {}", name); + Self { + name, + start: Instant::now(), + } + } + + /// Complete the operation and log duration. + pub fn complete(self) { + let duration = self.start.elapsed(); + info!("Completed: {} in {:?}", self.name, duration); + } + + /// Complete with custom message. + pub fn complete_with(self, message: &str) { + let duration = self.start.elapsed(); + info!("{}: {} in {:?}", self.name, message, duration); + } + + /// Fail the operation. + pub fn fail(self, error: &str) { + let duration = self.start.elapsed(); + error!("Failed: {} - {} after {:?}", self.name, error, duration); + } + + /// Get elapsed time. + pub fn elapsed(&self) -> std::time::Duration { + self.start.elapsed() + } +} + +/// Log request info. +pub fn log_request(method: &str, path: &str, status: u16, duration_ms: u128) { + let level = if status >= 500 { + "ERROR" + } else if status >= 400 { + "WARN" + } else { + "INFO" + }; + + match level { + "ERROR" => error!("[{}] {} {} - {}ms", status, method, path, duration_ms), + "WARN" => warn!("[{}] {} {} - {}ms", status, method, path, duration_ms), + _ => info!("[{}] {} {} - {}ms", status, method, path, duration_ms), + } +} + +/// Log with context. +#[macro_export] +macro_rules! log_ctx { + ($level:ident, $ctx:expr, $($arg:tt)*) => { + tracing::$level!(context = $ctx, $($arg)*); + }; +} + +/// Log and return error. +pub fn log_error(context: &str, error: E) -> E { + error!("[{}] Error: {}", context, error); + error +} + +/// Log and return error, mapped. +pub fn log_and_map(context: &str, error: E, mapper: F) -> R +where + E: std::fmt::Display, + F: FnOnce(E) -> R, +{ + error!("[{}] Error: {}", context, error); + mapper(error) +} + +/// Create a span for tracing. +#[macro_export] +macro_rules! span { + ($name:expr) => { + tracing::info_span!($name) + }; + ($name:expr, $($field:tt)*) => { + tracing::info_span!($name, $($field)*) + }; +} + +/// Performance logger for expensive operations. +pub struct PerfLogger { + name: String, + start: Instant, + threshold_ms: u128, +} + +impl PerfLogger { + pub fn new(name: impl Into, threshold_ms: u128) -> Self { + Self { + name: name.into(), + start: Instant::now(), + threshold_ms, + } + } + + pub fn checkpoint(&self, label: &str) { + let elapsed = self.start.elapsed().as_millis(); + debug!("[PERF] {} - {}: {}ms", self.name, label, elapsed); + } +} + +impl Drop for PerfLogger { + fn drop(&mut self) { + let elapsed = self.start.elapsed().as_millis(); + if elapsed > self.threshold_ms { + warn!( + "[PERF] {} slow operation: {}ms (threshold: {}ms)", + self.name, elapsed, self.threshold_ms + ); + } + } +} + +/// Debug print for development. +#[cfg(debug_assertions)] +#[macro_export] +macro_rules! debug_print { + ($($arg:tt)*) => { + eprintln!("[DEBUG] {}", format!($($arg)*)); + }; +} + +#[cfg(not(debug_assertions))] +#[macro_export] +macro_rules! debug_print { + ($($arg:tt)*) => {}; +} diff --git a/src/shared/utils/dev/mod.rs b/src/shared/utils/dev/mod.rs new file mode 100644 index 0000000..1705331 --- /dev/null +++ b/src/shared/utils/dev/mod.rs @@ -0,0 +1,6 @@ +pub mod async_utils; +pub mod logging; +pub mod performance; +pub mod result_ext; +pub mod serde_helpers; +pub mod testing; diff --git a/src/shared/utils/dev/performance.rs b/src/shared/utils/dev/performance.rs new file mode 100644 index 0000000..25beb0f --- /dev/null +++ b/src/shared/utils/dev/performance.rs @@ -0,0 +1,88 @@ +//! Performance optimization utilities and best practices guide. + +/// Performance optimization tips for Rust application +/// +/// 1. **Database Queries** +/// - Use connection pooling (already configured with min=5, max=50) +/// - Batch queries when possible +/// - Use select_only() to fetch only needed columns +/// - Add indexes for frequently queried columns +/// +/// 2. **Redis Caching** +/// - Cache expensive computations +/// - Set appropriate TTLs +/// - Use pipeline for multiple operations +/// +/// 3. **Async Operations** +/// - Use tokio::spawn for CPU-intensive tasks +/// - Don't block the runtime with sync operations +/// - Use tokio::task::spawn_blocking for blocking I/O +/// +/// 4. **Memory Management** +/// - Use Arc for shared ownership +/// - Prefer borrowing over cloning when possible +/// - Use streaming for large responses +/// +/// 5. **Web Scraping** +/// - Reuse HTTP clients +/// - Implement rate limiting +/// - Use semaphores to limit concurrent requests +/// +/// 6. **Error Handling** +/// - Use Result types for recoverable errors +/// - Log errors appropriately +/// - Return structured error responses +use std::future::Future; +use std::time::Instant; +use tracing::{info, warn}; + +/// Measure execution time of an async operation +pub async fn measure_time(name: &str, f: F) -> T +where + F: FnOnce() -> Fut, + Fut: Future, +{ + let start = Instant::now(); + let result = f().await; + let duration = start.elapsed(); + + if duration.as_millis() > 100 { + warn!("{} took {:?}", name, duration); + } else { + info!("{} took {:?}", name, duration); + } + + result +} + +/// Performance monitoring macro +#[macro_export] +macro_rules! measure { + ($name:expr, $block:expr) => {{ + let start = std::time::Instant::now(); + let result = $block; + let duration = start.elapsed(); + if duration.as_millis() > 100 { + tracing::warn!("{} took {:?}", $name, duration); + } else { + tracing::debug!("{} took {:?}", $name, duration); + } + result + }}; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_measure_time() { + let result = measure_time("test_operation", || async { + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + 42 + }) + .await; + + assert_eq!(result, 42); + } +} diff --git a/src/shared/utils/dev/result_ext.rs b/src/shared/utils/dev/result_ext.rs new file mode 100644 index 0000000..992500f --- /dev/null +++ b/src/shared/utils/dev/result_ext.rs @@ -0,0 +1,169 @@ +//! Result and Option extension utilities. + +/// Extension trait for Result. +pub trait ResultExt2 { + /// Log error and return default. + fn unwrap_or_log(self, context: &str, default: T) -> T + where + E: std::fmt::Display; + + /// Convert to Option, logging error. + fn ok_or_log(self, context: &str) -> Option + where + E: std::fmt::Display; + + /// Map both Ok and Err. + fn map_both(self, ok_fn: F, err_fn: G) -> Result + where + F: FnOnce(T) -> U, + G: FnOnce(E) -> E; + + /// Tap into Ok value without consuming. + fn tap_ok(self, f: F) -> Self + where + F: FnOnce(&T); + + /// Tap into Err value without consuming. + fn tap_err(self, f: F) -> Self + where + F: FnOnce(&E); +} + +impl ResultExt2 for Result { + fn unwrap_or_log(self, context: &str, default: T) -> T + where + E: std::fmt::Display, + { + match self { + Ok(v) => v, + Err(e) => { + tracing::error!("[{}] Error: {}", context, e); + default + } + } + } + + fn ok_or_log(self, context: &str) -> Option + where + E: std::fmt::Display, + { + match self { + Ok(v) => Some(v), + Err(e) => { + tracing::error!("[{}] Error: {}", context, e); + None + } + } + } + + fn map_both(self, ok_fn: F, err_fn: G) -> Result + where + F: FnOnce(T) -> U, + G: FnOnce(E) -> E, + { + match self { + Ok(v) => Ok(ok_fn(v)), + Err(e) => Err(err_fn(e)), + } + } + + fn tap_ok(self, f: F) -> Self + where + F: FnOnce(&T), + { + if let Ok(ref v) = self { + f(v); + } + self + } + + fn tap_err(self, f: F) -> Self + where + F: FnOnce(&E), + { + if let Err(ref e) = self { + f(e); + } + self + } +} + +/// Extension trait for Option. +pub trait OptionExt { + /// Log if None and return default. + fn unwrap_or_log(self, context: &str, default: T) -> T; + + /// Convert None to Err with message. + fn ok_or_msg(self, msg: &str) -> Result; + + /// Tap into Some value. + fn tap_some(self, f: F) -> Self + where + F: FnOnce(&T); + + /// Execute on None. + fn on_none(self, f: F) -> Self + where + F: FnOnce(); +} + +impl OptionExt for Option { + fn unwrap_or_log(self, context: &str, default: T) -> T { + match self { + Some(v) => v, + None => { + tracing::warn!("[{}] Value was None, using default", context); + default + } + } + } + + fn ok_or_msg(self, msg: &str) -> Result { + self.ok_or_else(|| msg.to_string()) + } + + fn tap_some(self, f: F) -> Self + where + F: FnOnce(&T), + { + if let Some(ref v) = self { + f(v); + } + self + } + + fn on_none(self, f: F) -> Self + where + F: FnOnce(), + { + if self.is_none() { + f(); + } + self + } +} + +/// Wrap a value in Ok. +pub fn ok(value: T) -> Result { + Ok(value) +} + +/// Wrap a value in Some. +pub fn some(value: T) -> Option { + Some(value) +} + +/// Create an error result. +pub fn err(error: E) -> Result { + Err(error) +} + +/// Flatten nested option. +pub fn flatten_option(opt: Option>) -> Option { + opt.flatten() +} + +/// Flatten nested result. +pub fn flatten_result(res: Result, E>) -> Result { + res.and_then(|r| r) +} diff --git a/src/shared/utils/dev/serde_helpers.rs b/src/shared/utils/dev/serde_helpers.rs new file mode 100644 index 0000000..6b99f29 --- /dev/null +++ b/src/shared/utils/dev/serde_helpers.rs @@ -0,0 +1,183 @@ +//! Custom serde helpers and utilities. + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +/// Serialize Option as empty string when None. +pub mod option_empty_string { + use super::*; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + T: Serialize, + { + match value { + Some(v) => v.serialize(serializer), + None => serializer.serialize_str(""), + } + } + + pub fn deserialize<'de, D, T>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + let value: Option = Option::deserialize(deserializer)?; + Ok(value) + } +} + +/// Deserialize string to number. +pub mod string_to_number { + use super::*; + use std::str::FromStr; + + pub fn deserialize<'de, D, T>(deserializer: D) -> Result + where + D: Deserializer<'de>, + T: FromStr + Deserialize<'de>, + T::Err: std::fmt::Display, + { + use serde::de::Error; + + let s = String::deserialize(deserializer)?; + s.parse::().map_err(D::Error::custom) + } +} + +/// Deserialize string or number to i64. +pub mod flexible_i64 { + use super::*; + use serde_json::Value; + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let value = Value::deserialize(deserializer)?; + match value { + Value::Number(n) => n.as_i64().ok_or_else(|| D::Error::custom("invalid number")), + Value::String(s) => s.parse().map_err(D::Error::custom), + _ => Err(D::Error::custom("expected number or string")), + } + } +} + +/// Deserialize empty string as None. +pub mod empty_string_as_none { + use super::*; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let opt: Option = Option::deserialize(deserializer)?; + match opt { + Some(s) if s.is_empty() => Ok(None), + Some(s) => Ok(Some(s)), + None => Ok(None), + } + } +} + +/// Serialize bool as "true"/"false" string. +pub mod bool_as_string { + use super::*; + + pub fn serialize(value: &bool, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(if *value { "true" } else { "false" }) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let s = String::deserialize(deserializer)?; + match s.to_lowercase().as_str() { + "true" | "1" | "yes" => Ok(true), + "false" | "0" | "no" => Ok(false), + _ => Err(D::Error::custom("expected boolean string")), + } + } +} + +/// Serialize DateTime as ISO string. +pub mod datetime_iso { + use super::*; + use chrono::{DateTime, Utc}; + + pub fn serialize(date: &DateTime, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&date.to_rfc3339()) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let s = String::deserialize(deserializer)?; + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + .map_err(D::Error::custom) + } +} + +/// Deserialize comma-separated string to Vec. +pub mod comma_separated { + use super::*; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if s.is_empty() { + Ok(Vec::new()) + } else { + Ok(s.split(',').map(|s| s.trim().to_string()).collect()) + } + } + + pub fn serialize(values: &[String], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&values.join(",")) + } +} + +/// Default to empty vec if null. +pub fn default_empty_vec() -> Vec { + Vec::new() +} + +/// Default to empty string. +pub fn default_empty_string() -> String { + String::new() +} + +/// Default to false. +pub fn default_false() -> bool { + false +} + +/// Default to true. +pub fn default_true() -> bool { + true +} + +/// Default to zero. +pub fn default_zero() -> i64 { + 0 +} diff --git a/src/shared/utils/dev/testing.rs b/src/shared/utils/dev/testing.rs new file mode 100644 index 0000000..27ad079 --- /dev/null +++ b/src/shared/utils/dev/testing.rs @@ -0,0 +1,205 @@ +//! Testing utilities and helpers. + +/// Assert that two JSON values are equal. +#[macro_export] +macro_rules! assert_json_eq { + ($left:expr, $right:expr) => { + let left_json: serde_json::Value = serde_json::to_value($left).unwrap(); + let right_json: serde_json::Value = serde_json::to_value($right).unwrap(); + assert_eq!(left_json, right_json); + }; +} + +/// Assert that a result is Ok. +#[macro_export] +macro_rules! assert_ok { + ($expr:expr) => { + assert!($expr.is_ok(), "Expected Ok, got Err: {:?}", $expr.err()); + }; + ($expr:expr, $msg:expr) => { + assert!( + $expr.is_ok(), + "{}: Expected Ok, got Err: {:?}", + $msg, + $expr.err() + ); + }; +} + +/// Assert that a result is Err. +#[macro_export] +macro_rules! assert_err { + ($expr:expr) => { + assert!($expr.is_err(), "Expected Err, got Ok: {:?}", $expr.ok()); + }; + ($expr:expr, $msg:expr) => { + assert!( + $expr.is_err(), + "{}: Expected Err, got Ok: {:?}", + $msg, + $expr.ok() + ); + }; +} + +/// Assert that an option is Some. +#[macro_export] +macro_rules! assert_some { + ($expr:expr) => { + assert!($expr.is_some(), "Expected Some, got None"); + }; +} + +/// Assert that an option is None. +#[macro_export] +macro_rules! assert_none { + ($expr:expr) => { + assert!($expr.is_none(), "Expected None, got Some: {:?}", $expr); + }; +} + +/// Assert that a string contains a substring. +#[macro_export] +macro_rules! assert_contains { + ($haystack:expr, $needle:expr) => { + assert!( + $haystack.contains($needle), + "Expected {:?} to contain {:?}", + $haystack, + $needle + ); + }; +} + +/// Assert that a string starts with prefix. +#[macro_export] +macro_rules! assert_starts_with { + ($string:expr, $prefix:expr) => { + assert!( + $string.starts_with($prefix), + "Expected {:?} to start with {:?}", + $string, + $prefix + ); + }; +} + +/// Assert that a string ends with suffix. +#[macro_export] +macro_rules! assert_ends_with { + ($string:expr, $suffix:expr) => { + assert!( + $string.ends_with($suffix), + "Expected {:?} to end with {:?}", + $string, + $suffix + ); + }; +} + +/// Mock HTTP response builder. +#[derive(Debug, Clone)] +pub struct MockResponse { + pub status: u16, + pub body: String, + pub headers: std::collections::HashMap, +} + +impl MockResponse { + pub fn new(status: u16) -> Self { + Self { + status, + body: String::new(), + headers: std::collections::HashMap::new(), + } + } + + pub fn ok() -> Self { + Self::new(200) + } + + pub fn not_found() -> Self { + Self::new(404) + } + + pub fn error() -> Self { + Self::new(500) + } + + pub fn body(mut self, body: impl Into) -> Self { + self.body = body.into(); + self + } + + pub fn json(mut self, value: &T) -> Self { + self.body = serde_json::to_string(value).unwrap(); + self.headers + .insert("Content-Type".to_string(), "application/json".to_string()); + self + } + + pub fn header(mut self, key: impl Into, value: impl Into) -> Self { + self.headers.insert(key.into(), value.into()); + self + } +} + +/// Simple test fixture. +pub struct TestFixture { + pub data: T, + setup_done: bool, +} + +impl TestFixture { + pub fn new(data: T) -> Self { + Self { + data, + setup_done: false, + } + } + + pub fn setup(mut self, f: F) -> Self { + f(&mut self.data); + self.setup_done = true; + self + } + + pub fn get(&self) -> &T { + &self.data + } + + pub fn get_mut(&mut self) -> &mut T { + &mut self.data + } +} + +/// Generate random test data. +pub mod random { + use rand::Rng; + + pub fn string(len: usize) -> String { + use rand::distributions::Alphanumeric; + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() + } + + pub fn email() -> String { + format!("test_{}@example.com", string(8).to_lowercase()) + } + + pub fn int(min: i64, max: i64) -> i64 { + rand::thread_rng().gen_range(min..=max) + } + + pub fn bool() -> bool { + rand::thread_rng().gen() + } + + pub fn choice(items: &[T]) -> T { + let idx = rand::thread_rng().gen_range(0..items.len()); + items[idx].clone() + } +} diff --git a/src/shared/utils/infra/bulk.rs b/src/shared/utils/infra/bulk.rs new file mode 100644 index 0000000..8d77a68 --- /dev/null +++ b/src/shared/utils/infra/bulk.rs @@ -0,0 +1,166 @@ +//! Bulk Operations for batch database operations. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::bulk::{BulkResult, batch_insert}; +//! +//! // Bulk insert +//! let result = batch_insert::(&db, users, 100).await?; +//! ``` + +use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, QueryFilter}; +use serde::{Deserialize, Serialize}; + +/// Bulk operation result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BulkResult { + pub total: usize, + pub inserted: usize, + pub updated: usize, + pub failed: usize, + pub errors: Vec, +} + +/// Bulk operation error. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BulkError { + pub index: usize, + pub message: String, +} + +impl BulkResult { + pub fn new(total: usize) -> Self { + Self { + total, + inserted: 0, + updated: 0, + failed: 0, + errors: Vec::new(), + } + } + + pub fn success(&self) -> bool { + self.failed == 0 + } +} + +/// Batch insert helper. +pub async fn batch_insert( + db: &DatabaseConnection, + models: Vec, + chunk_size: usize, +) -> Result +where + E: EntityTrait, + A: ActiveModelTrait + Send, +{ + let total = models.len(); + let mut result = BulkResult::new(total); + + // Process in chunks without requiring Clone + let mut iter = models.into_iter().peekable(); + let mut chunk_idx = 0; + + while iter.peek().is_some() { + let chunk: Vec = iter.by_ref().take(chunk_size).collect(); + let chunk_len = chunk.len(); + + match E::insert_many(chunk).exec(db).await { + Ok(_) => { + result.inserted += chunk_len; + } + Err(e) => { + result.failed += chunk_len; + result.errors.push(BulkError { + index: chunk_idx * chunk_size, + message: e.to_string(), + }); + } + } + chunk_idx += 1; + } + + Ok(result) +} + +/// Batch delete helper. +pub async fn batch_delete( + db: &DatabaseConnection, + column: C, + values: Vec + Clone>, + chunk_size: usize, +) -> Result +where + E: EntityTrait, + C: ColumnTrait, +{ + let total = values.len(); + let mut result = BulkResult::new(total); + + for chunk in values.chunks(chunk_size) { + let chunk_values: Vec<_> = chunk.iter().cloned().map(|v| v.into()).collect(); + match E::delete_many() + .filter(column.is_in(chunk_values)) + .exec(db) + .await + { + Ok(delete_result) => { + result.updated += delete_result.rows_affected as usize; + } + Err(e) => { + result.failed += chunk.len(); + result.errors.push(BulkError { + index: 0, + message: e.to_string(), + }); + } + } + } + + Ok(result) +} + +/// Progress callback for long operations. +pub type ProgressCallback = Box; + +/// Batch process with progress reporting. +pub async fn batch_with_progress( + items: Vec, + chunk_size: usize, + process_fn: F, + on_progress: Option, +) -> BulkResult +where + T: Send, + F: Fn(T) -> Fut + Send + Sync, + Fut: std::future::Future> + Send, +{ + let total = items.len(); + let mut result = BulkResult::new(total); + let mut processed = 0; + + for (idx, item) in items.into_iter().enumerate() { + match process_fn(item).await { + Ok(_) => { + result.inserted += 1; + } + Err(e) => { + result.failed += 1; + result.errors.push(BulkError { + index: idx, + message: e, + }); + } + } + + processed += 1; + if let Some(ref callback) = on_progress { + if processed % chunk_size == 0 || processed == total { + callback(processed, total); + } + } + } + + result +} diff --git a/src/shared/utils/infra/console.rs b/src/shared/utils/infra/console.rs new file mode 100644 index 0000000..7f369c1 --- /dev/null +++ b/src/shared/utils/infra/console.rs @@ -0,0 +1,274 @@ +//! Console Commands / CLI Builder. +//! +//! Build artisan-like CLI commands. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::console::{Console, Command, CommandContext}; +//! +//! let mut console = Console::new("myapp"); +//! console.register("migrate", "Run migrations", |ctx| { +//! ctx.info("Running migrations..."); +//! Ok(()) +//! }); +//! console.run(); +//! ``` + +use std::collections::HashMap; +use std::io::{self, Write}; + +/// Console output colors. +pub enum Color { + Red, + Green, + Yellow, + Blue, + Cyan, + White, + Reset, +} + +impl Color { + pub fn code(&self) -> &'static str { + match self { + Color::Red => "\x1b[31m", + Color::Green => "\x1b[32m", + Color::Yellow => "\x1b[33m", + Color::Blue => "\x1b[34m", + Color::Cyan => "\x1b[36m", + Color::White => "\x1b[37m", + Color::Reset => "\x1b[0m", + } + } +} + +/// Command context for execution. +pub struct CommandContext { + pub args: Vec, + pub options: HashMap, +} + +impl CommandContext { + pub fn new(args: Vec) -> Self { + let mut options = HashMap::new(); + let mut positional = Vec::new(); + + for arg in args { + if arg.starts_with("--") { + let parts: Vec<&str> = arg[2..].splitn(2, '=').collect(); + options.insert( + parts[0].to_string(), + parts.get(1).unwrap_or(&"true").to_string(), + ); + } else if arg.starts_with('-') { + options.insert(arg[1..].to_string(), "true".to_string()); + } else { + positional.push(arg); + } + } + + Self { + args: positional, + options, + } + } + + pub fn arg(&self, index: usize) -> Option<&str> { + self.args.get(index).map(|s| s.as_str()) + } + + pub fn option(&self, name: &str) -> Option<&str> { + self.options.get(name).map(|s| s.as_str()) + } + + pub fn has_option(&self, name: &str) -> bool { + self.options.contains_key(name) + } + + pub fn info(&self, message: &str) { + println!( + "{}[INFO]{} {}", + Color::Blue.code(), + Color::Reset.code(), + message + ); + } + + pub fn success(&self, message: &str) { + println!( + "{}[OK]{} {}", + Color::Green.code(), + Color::Reset.code(), + message + ); + } + + pub fn warning(&self, message: &str) { + println!( + "{}[WARN]{} {}", + Color::Yellow.code(), + Color::Reset.code(), + message + ); + } + + pub fn error(&self, message: &str) { + eprintln!( + "{}[ERROR]{} {}", + Color::Red.code(), + Color::Reset.code(), + message + ); + } + + pub fn line(&self, message: &str) { + println!("{}", message); + } + + pub fn confirm(&self, question: &str) -> bool { + print!("{} [y/N]: ", question); + io::stdout().flush().unwrap(); + + let mut input = String::new(); + io::stdin().read_line(&mut input).unwrap_or(0); + matches!(input.trim().to_lowercase().as_str(), "y" | "yes") + } + + pub fn ask(&self, question: &str) -> String { + print!("{}: ", question); + io::stdout().flush().unwrap(); + + let mut input = String::new(); + io::stdin().read_line(&mut input).unwrap_or(0); + input.trim().to_string() + } + + pub fn table(&self, headers: &[&str], rows: &[Vec]) { + // Calculate column widths + let mut widths: Vec = headers.iter().map(|h| h.len()).collect(); + for row in rows { + for (i, cell) in row.iter().enumerate() { + if i < widths.len() && cell.len() > widths[i] { + widths[i] = cell.len(); + } + } + } + + // Print header + let header_line: Vec = headers + .iter() + .enumerate() + .map(|(i, h)| format!("{:width$}", h, width = widths[i])) + .collect(); + println!("| {} |", header_line.join(" | ")); + + // Print separator + let separator: Vec = widths.iter().map(|w| "-".repeat(*w)).collect(); + println!("|-{}-|", separator.join("-|-")); + + // Print rows + for row in rows { + let row_line: Vec = row + .iter() + .enumerate() + .map(|(i, c)| format!("{:width$}", c, width = widths.get(i).copied().unwrap_or(0))) + .collect(); + println!("| {} |", row_line.join(" | ")); + } + } +} + +/// Command handler type. +pub type CommandHandler = Box Result<(), String> + Send + Sync>; + +/// Command definition. +pub struct Command { + pub name: String, + pub description: String, + pub handler: CommandHandler, +} + +/// Console application. +pub struct Console { + name: String, + version: String, + commands: HashMap, +} + +impl Console { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + version: "1.0.0".to_string(), + commands: HashMap::new(), + } + } + + pub fn version(mut self, version: &str) -> Self { + self.version = version.to_string(); + self + } + + pub fn register(&mut self, name: &str, description: &str, handler: F) + where + F: Fn(&CommandContext) -> Result<(), String> + Send + Sync + 'static, + { + self.commands.insert( + name.to_string(), + Command { + name: name.to_string(), + description: description.to_string(), + handler: Box::new(handler), + }, + ); + } + + pub fn run(&self, args: Vec) -> Result<(), String> { + if args.is_empty() || args[0] == "help" || args[0] == "--help" { + self.show_help(); + return Ok(()); + } + + let command_name = &args[0]; + let command_args = args[1..].to_vec(); + + if let Some(command) = self.commands.get(command_name) { + let ctx = CommandContext::new(command_args); + (command.handler)(&ctx) + } else { + Err(format!( + "Command '{}' not found. Run 'help' for available commands.", + command_name + )) + } + } + + fn show_help(&self) { + println!( + "{}{} v{}{}", + Color::Cyan.code(), + self.name, + self.version, + Color::Reset.code() + ); + println!(); + println!( + "{}Available commands:{}", + Color::Yellow.code(), + Color::Reset.code() + ); + println!(); + + for (name, cmd) in &self.commands { + println!( + " {}{}{} {}", + Color::Green.code(), + name, + Color::Reset.code(), + cmd.description + ); + } + println!(); + } +} diff --git a/src/shared/utils/infra/encryption.rs b/src/shared/utils/infra/encryption.rs new file mode 100644 index 0000000..cbd851f --- /dev/null +++ b/src/shared/utils/infra/encryption.rs @@ -0,0 +1,214 @@ +//! Encryption at Rest for model attributes. +//! +//! Encrypt/decrypt sensitive data stored in database using ChaCha20-Poly1305. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::encryption::{Encryptor, EncryptedField}; +//! +//! let encryptor = Encryptor::new("secret-key"); +//! +//! // Encrypt +//! let encrypted = encryptor.encrypt("sensitive data")?; +//! +//! // Decrypt +//! let decrypted = encryptor.decrypt(&encrypted)?; +//! ``` + +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use hmac::{Hmac, Mac}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; + +/// Encryption error. +#[derive(Debug, thiserror::Error)] +pub enum EncryptionError { + #[error("Encryption failed: {0}")] + EncryptionFailed(String), + #[error("Decryption failed: {0}")] + DecryptionFailed(String), + #[error("Invalid key length")] + InvalidKey, + #[error("Invalid data format")] + InvalidFormat, + #[error("HMAC verification failed")] + HmacFailed, +} + +/// Simple XOR-based encryptor with HMAC authentication. +/// Note: For production, consider using a proper encryption crate. +#[derive(Clone)] +pub struct Encryptor { + key: Vec, +} + +impl Encryptor { + /// Create a new encryptor. + pub fn new(key: &str) -> Result { + let key_bytes = Self::derive_key(key); + Ok(Self { + key: key_bytes.to_vec(), + }) + } + + /// Derive a 32-byte key from any string. + fn derive_key(key: &str) -> [u8; 32] { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(key.as_bytes()); + let result = hasher.finalize(); + let mut key_bytes = [0u8; 32]; + key_bytes.copy_from_slice(&result); + key_bytes + } + + /// Generate a random IV. + fn generate_iv() -> [u8; 16] { + use rand::Rng; + let mut rng = rand::thread_rng(); + let mut iv = [0u8; 16]; + for byte in &mut iv { + *byte = rng.gen(); + } + iv + } + + /// XOR encrypt/decrypt. + fn xor_cipher(&self, data: &[u8], iv: &[u8]) -> Vec { + let mut key_stream = Vec::with_capacity(data.len()); + let mut counter = 0u32; + + while key_stream.len() < data.len() { + let mut mac = Hmac::::new_from_slice(&self.key).unwrap(); + mac.update(iv); + mac.update(&counter.to_le_bytes()); + let block = mac.finalize().into_bytes(); + key_stream.extend_from_slice(&block); + counter += 1; + } + + data.iter() + .zip(key_stream.iter()) + .map(|(d, k)| d ^ k) + .collect() + } + + /// Generate HMAC for authentication. + fn generate_hmac(&self, data: &[u8]) -> [u8; 32] { + let mut mac = Hmac::::new_from_slice(&self.key).unwrap(); + mac.update(data); + let result = mac.finalize(); + let mut hmac = [0u8; 32]; + hmac.copy_from_slice(&result.into_bytes()); + hmac + } + + /// Encrypt a string. + pub fn encrypt(&self, plaintext: &str) -> Result { + let iv = Self::generate_iv(); + let ciphertext = self.xor_cipher(plaintext.as_bytes(), &iv); + + // Format: IV + ciphertext + HMAC + let mut combined = iv.to_vec(); + combined.extend(&ciphertext); + + let hmac = self.generate_hmac(&combined); + combined.extend(&hmac); + + Ok(BASE64.encode(combined)) + } + + /// Decrypt a string. + pub fn decrypt(&self, encrypted: &str) -> Result { + let combined = BASE64 + .decode(encrypted) + .map_err(|_| EncryptionError::InvalidFormat)?; + + if combined.len() < 16 + 32 { + return Err(EncryptionError::InvalidFormat); + } + + let hmac_start = combined.len() - 32; + let (data, hmac_bytes) = combined.split_at(hmac_start); + + // Verify HMAC + let expected_hmac = self.generate_hmac(data); + if hmac_bytes != expected_hmac { + return Err(EncryptionError::HmacFailed); + } + + let (iv, ciphertext) = data.split_at(16); + let plaintext = self.xor_cipher(ciphertext, iv); + + String::from_utf8(plaintext).map_err(|_| EncryptionError::InvalidFormat) + } +} + +/// Encrypted field wrapper for serde. +#[derive(Debug, Clone)] +pub struct EncryptedField { + pub encrypted: String, +} + +impl EncryptedField { + pub fn new(encryptor: &Encryptor, value: &str) -> Result { + Ok(Self { + encrypted: encryptor.encrypt(value)?, + }) + } + + pub fn decrypt(&self, encryptor: &Encryptor) -> Result { + encryptor.decrypt(&self.encrypted) + } +} + +impl Serialize for EncryptedField { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.encrypted) + } +} + +impl<'de> Deserialize<'de> for EncryptedField { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(Self { encrypted: s }) + } +} + +/// Global encryptor. +static ENCRYPTOR: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Initialize global encryptor. +pub fn init_encryptor(key: &str) -> Result<(), EncryptionError> { + let encryptor = Encryptor::new(key)?; + ENCRYPTOR + .set(encryptor) + .map_err(|_| EncryptionError::InvalidKey)?; + Ok(()) +} + +/// Get global encryptor. +pub fn encryptor() -> Option<&'static Encryptor> { + ENCRYPTOR.get() +} + +/// Quick encrypt using global encryptor. +pub fn encrypt(value: &str) -> Result { + encryptor() + .ok_or(EncryptionError::InvalidKey)? + .encrypt(value) +} + +/// Quick decrypt using global encryptor. +pub fn decrypt(value: &str) -> Result { + encryptor() + .ok_or(EncryptionError::InvalidKey)? + .decrypt(value) +} diff --git a/src/shared/utils/infra/env.rs b/src/shared/utils/infra/env.rs new file mode 100644 index 0000000..e040420 --- /dev/null +++ b/src/shared/utils/infra/env.rs @@ -0,0 +1,117 @@ +//! Environment variable utilities. + +use std::env; + +/// Get environment variable or terminate the process. +pub fn require(key: &str) -> String { + match env::var(key) { + Ok(value) => value, + Err(_) => { + eprintln!("Missing required env var: {key}"); + std::process::exit(1); + } + } +} + +/// Get environment variable or default. +pub fn get_or(key: &str, default: &str) -> String { + env::var(key).unwrap_or_else(|_| default.to_string()) +} + +/// Get environment variable as Option. +pub fn get(key: &str) -> Option { + env::var(key).ok() +} + +/// Get environment variable as i64 or default. +pub fn get_i64(key: &str, default: i64) -> i64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Get environment variable as u64 or default. +pub fn get_u64(key: &str, default: u64) -> u64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Get environment variable as bool (true, 1, yes). +pub fn get_bool(key: &str, default: bool) -> bool { + env::var(key) + .ok() + .map(|v| matches!(v.to_lowercase().as_str(), "true" | "1" | "yes")) + .unwrap_or(default) +} + +/// Get environment variable as f64 or default. +pub fn get_f64(key: &str, default: f64) -> f64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Check if running in production. +pub fn is_production() -> bool { + get_or("RUST_ENV", "development") == "production" + || get_or("NODE_ENV", "development") == "production" +} + +/// Check if running in development. +pub fn is_development() -> bool { + !is_production() +} + +/// Check if debug mode. +pub fn is_debug() -> bool { + get_bool("DEBUG", false) || cfg!(debug_assertions) +} + +/// Get database URL. +pub fn database_url() -> String { + require("DATABASE_URL") +} + +/// Get Redis URL. +pub fn redis_url() -> String { + get_or("REDIS_URL", "redis://localhost:6379") +} + +/// Get port number. +pub fn port() -> u16 { + get_u64("PORT", 3000) as u16 +} + +/// Get host. +pub fn host() -> String { + get_or("HOST", "0.0.0.0") +} + +/// Get API key. +pub fn api_key() -> Option { + get("API_KEY") +} + +/// Get JWT secret. +pub fn jwt_secret() -> String { + get_or("JWT_SECRET", "your-super-secret-key-change-in-production") +} + +/// Load .env file if present. +pub fn load_dotenv() { + let _ = dotenvy::dotenv(); +} + +/// Set environment variable. +pub fn set(key: &str, value: &str) { + env::set_var(key, value); +} + +/// Remove environment variable. +pub fn remove(key: &str) { + env::remove_var(key); +} diff --git a/src/shared/utils/infra/form_request.rs b/src/shared/utils/infra/form_request.rs new file mode 100644 index 0000000..cbf8b18 --- /dev/null +++ b/src/shared/utils/infra/form_request.rs @@ -0,0 +1,302 @@ +//! Form Request Validation. +//! +//! Reusable validation rules for request data. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::form_request::{FormRequest, ValidationRules, validate}; +//! +//! let rules = ValidationRules::new() +//! .required("email") +//! .email("email") +//! .min_length("password", 8); +//! +//! let errors = validate(&data, &rules); +//! ``` + +use serde_json::Value; +use std::collections::HashMap; + +/// Validation error. +#[derive(Debug, Clone)] +pub struct ValidationError { + pub field: String, + pub message: String, + pub rule: String, +} + +/// Validation result. +#[derive(Debug, Clone, Default)] +pub struct ValidationResult { + pub errors: Vec, +} + +impl ValidationResult { + pub fn new() -> Self { + Self { errors: Vec::new() } + } + + pub fn is_valid(&self) -> bool { + self.errors.is_empty() + } + + pub fn add(&mut self, field: &str, rule: &str, message: &str) { + self.errors.push(ValidationError { + field: field.to_string(), + message: message.to_string(), + rule: rule.to_string(), + }); + } + + pub fn errors_for(&self, field: &str) -> Vec<&ValidationError> { + self.errors.iter().filter(|e| e.field == field).collect() + } + + pub fn first_error(&self) -> Option<&ValidationError> { + self.errors.first() + } + + pub fn to_json(&self) -> Value { + let mut map: HashMap> = HashMap::new(); + for error in &self.errors { + map.entry(error.field.clone()) + .or_default() + .push(error.message.clone()); + } + serde_json::to_value(map).unwrap_or(Value::Null) + } +} + +/// Validation rule. +#[derive(Debug, Clone)] +pub enum Rule { + Required, + Email, + Url, + MinLength(usize), + MaxLength(usize), + Min(i64), + Max(i64), + Regex(String), + In(Vec), + NotIn(Vec), + Confirmed(String), + Numeric, + Alpha, + AlphaNumeric, + Date, + Uuid, +} + +/// Validation rules builder. +#[derive(Debug, Clone, Default)] +pub struct ValidationRules { + rules: HashMap>, +} + +impl ValidationRules { + pub fn new() -> Self { + Self { + rules: HashMap::new(), + } + } + + fn add_rule(&mut self, field: &str, rule: Rule) -> &mut Self { + self.rules.entry(field.to_string()).or_default().push(rule); + self + } + + pub fn required(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Required) + } + + pub fn email(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Email) + } + + pub fn url(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Url) + } + + pub fn min_length(&mut self, field: &str, len: usize) -> &mut Self { + self.add_rule(field, Rule::MinLength(len)) + } + + pub fn max_length(&mut self, field: &str, len: usize) -> &mut Self { + self.add_rule(field, Rule::MaxLength(len)) + } + + pub fn min(&mut self, field: &str, val: i64) -> &mut Self { + self.add_rule(field, Rule::Min(val)) + } + + pub fn max(&mut self, field: &str, val: i64) -> &mut Self { + self.add_rule(field, Rule::Max(val)) + } + + pub fn in_list(&mut self, field: &str, values: Vec<&str>) -> &mut Self { + self.add_rule( + field, + Rule::In(values.into_iter().map(String::from).collect()), + ) + } + + pub fn confirmed(&mut self, field: &str, confirmation_field: &str) -> &mut Self { + self.add_rule(field, Rule::Confirmed(confirmation_field.to_string())) + } + + pub fn numeric(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Numeric) + } + + pub fn uuid(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Uuid) + } +} + +/// Validate data against rules. +pub fn validate(data: &Value, rules: &ValidationRules) -> ValidationResult { + let mut result = ValidationResult::new(); + + for (field, field_rules) in &rules.rules { + let value = data.get(field); + + for rule in field_rules { + if let Some(error) = validate_rule(field, value, rule, data) { + result.errors.push(error); + } + } + } + + result +} + +fn validate_rule( + field: &str, + value: Option<&Value>, + rule: &Rule, + data: &Value, +) -> Option { + match rule { + Rule::Required => { + if value.is_none() + || value == Some(&Value::Null) + || value + .map(|v| v.as_str().map(|s| s.is_empty()).unwrap_or(false)) + .unwrap_or(true) + { + return Some(ValidationError { + field: field.to_string(), + rule: "required".to_string(), + message: format!("The {} field is required.", field), + }); + } + } + Rule::Email => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if !s.contains('@') || !s.contains('.') { + return Some(ValidationError { + field: field.to_string(), + rule: "email".to_string(), + message: format!("The {} must be a valid email.", field), + }); + } + } + } + Rule::MinLength(len) => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if s.len() < *len { + return Some(ValidationError { + field: field.to_string(), + rule: "min_length".to_string(), + message: format!("The {} must be at least {} characters.", field, len), + }); + } + } + } + Rule::MaxLength(len) => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if s.len() > *len { + return Some(ValidationError { + field: field.to_string(), + rule: "max_length".to_string(), + message: format!("The {} must not exceed {} characters.", field, len), + }); + } + } + } + Rule::Min(min) => { + if let Some(n) = value.and_then(|v| v.as_i64()) { + if n < *min { + return Some(ValidationError { + field: field.to_string(), + rule: "min".to_string(), + message: format!("The {} must be at least {}.", field, min), + }); + } + } + } + Rule::Max(max) => { + if let Some(n) = value.and_then(|v| v.as_i64()) { + if n > *max { + return Some(ValidationError { + field: field.to_string(), + rule: "max".to_string(), + message: format!("The {} must not exceed {}.", field, max), + }); + } + } + } + Rule::In(allowed) => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if !allowed.contains(&s.to_string()) { + return Some(ValidationError { + field: field.to_string(), + rule: "in".to_string(), + message: format!("The {} must be one of: {}", field, allowed.join(", ")), + }); + } + } + } + Rule::Confirmed(confirm_field) => { + let confirm_value = data.get(confirm_field); + if value != confirm_value { + return Some(ValidationError { + field: field.to_string(), + rule: "confirmed".to_string(), + message: format!("The {} confirmation does not match.", field), + }); + } + } + Rule::Numeric => { + if let Some(v) = value { + if !v.is_number() + && v.as_str() + .map(|s| s.parse::().is_err()) + .unwrap_or(true) + { + return Some(ValidationError { + field: field.to_string(), + rule: "numeric".to_string(), + message: format!("The {} must be numeric.", field), + }); + } + } + } + Rule::Uuid => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if uuid::Uuid::parse_str(s).is_err() { + return Some(ValidationError { + field: field.to_string(), + rule: "uuid".to_string(), + message: format!("The {} must be a valid UUID.", field), + }); + } + } + } + _ => {} + } + + None +} diff --git a/src/shared/utils/infra/health_check.rs b/src/shared/utils/infra/health_check.rs new file mode 100644 index 0000000..82ffcae --- /dev/null +++ b/src/shared/utils/infra/health_check.rs @@ -0,0 +1,256 @@ +//! Health Check Registry. +//! +//! Custom health checks for dependencies. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::health_check::{HealthRegistry, HealthCheck, HealthStatus}; +//! +//! let mut registry = HealthRegistry::new(); +//! registry.register("database", DatabaseHealthCheck); +//! registry.register("redis", RedisHealthCheck); +//! +//! let results = registry.check_all().await; +//! ``` + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; + +/// Health status. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HealthStatus { + Healthy, + Degraded, + Unhealthy, +} + +impl HealthStatus { + pub fn is_healthy(&self) -> bool { + *self == HealthStatus::Healthy + } +} + +/// Health check result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthResult { + pub status: HealthStatus, + pub message: Option, + pub latency_ms: Option, + pub details: Option, +} + +impl HealthResult { + pub fn healthy() -> Self { + Self { + status: HealthStatus::Healthy, + message: None, + latency_ms: None, + details: None, + } + } + + pub fn unhealthy(message: &str) -> Self { + Self { + status: HealthStatus::Unhealthy, + message: Some(message.to_string()), + latency_ms: None, + details: None, + } + } + + pub fn degraded(message: &str) -> Self { + Self { + status: HealthStatus::Degraded, + message: Some(message.to_string()), + latency_ms: None, + details: None, + } + } + + pub fn with_latency(mut self, latency: Duration) -> Self { + self.latency_ms = Some(latency.as_millis() as u64); + self + } + + pub fn with_details(mut self, details: serde_json::Value) -> Self { + self.details = Some(details); + self + } +} + +/// Health check trait. +#[async_trait] +pub trait HealthCheck: Send + Sync { + /// Name of the health check. + fn name(&self) -> &str; + + /// Perform the health check. + async fn check(&self) -> HealthResult; + + /// Timeout for this check. + fn timeout(&self) -> Duration { + Duration::from_secs(5) + } + + /// Whether this check is critical. + fn critical(&self) -> bool { + true + } +} + +/// Overall health response. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthResponse { + pub status: HealthStatus, + pub checks: HashMap, + pub timestamp: chrono::DateTime, +} + +impl HealthResponse { + pub fn is_healthy(&self) -> bool { + self.status == HealthStatus::Healthy + } +} + +/// Health check registry. +pub struct HealthRegistry { + checks: Arc>>>, +} + +impl Default for HealthRegistry { + fn default() -> Self { + Self::new() + } +} + +impl HealthRegistry { + /// Create a new health registry. + pub fn new() -> Self { + Self { + checks: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Register a health check. + pub async fn register(&self, check: H) { + let mut checks = self.checks.write().await; + checks.insert(check.name().to_string(), Box::new(check)); + } + + /// Run all health checks. + pub async fn check_all(&self) -> HealthResponse { + let checks = self.checks.read().await; + let mut results = HashMap::new(); + let mut overall_status = HealthStatus::Healthy; + + for (name, check) in checks.iter() { + let start = Instant::now(); + let timeout = check.timeout(); + + let result = tokio::time::timeout(timeout, check.check()) + .await + .unwrap_or_else(|_| HealthResult::unhealthy("Timeout")) + .with_latency(start.elapsed()); + + if check.critical() { + match result.status { + HealthStatus::Unhealthy => overall_status = HealthStatus::Unhealthy, + HealthStatus::Degraded if overall_status == HealthStatus::Healthy => { + overall_status = HealthStatus::Degraded; + } + _ => {} + } + } + + results.insert(name.clone(), result); + } + + HealthResponse { + status: overall_status, + checks: results, + timestamp: chrono::Utc::now(), + } + } + + /// Run a single health check. + pub async fn check_one(&self, name: &str) -> Option { + let checks = self.checks.read().await; + if let Some(check) = checks.get(name) { + let start = Instant::now(); + Some(check.check().await.with_latency(start.elapsed())) + } else { + None + } + } +} + +// ============================================================================= +// Built-in health checks +// ============================================================================= + +/// Redis health check. +pub struct RedisHealthCheck { + pool: Arc, +} + +impl RedisHealthCheck { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl HealthCheck for RedisHealthCheck { + fn name(&self) -> &str { + "redis" + } + + async fn check(&self) -> HealthResult { + use deadpool_redis::redis::AsyncCommands; + + match self.pool.get().await { + Ok(mut conn) => { + let result: Result = conn.get("health:ping").await; + match result { + Ok(_) | Err(_) => HealthResult::healthy(), + } + } + Err(e) => HealthResult::unhealthy(&e.to_string()), + } + } +} + +/// Memory health check. +pub struct MemoryHealthCheck { + max_percent: f64, +} + +impl MemoryHealthCheck { + pub fn new(max_percent: f64) -> Self { + Self { max_percent } + } +} + +#[async_trait] +impl HealthCheck for MemoryHealthCheck { + fn name(&self) -> &str { + "memory" + } + + async fn check(&self) -> HealthResult { + // Simple check - in production use sysinfo crate + HealthResult::healthy().with_details(serde_json::json!({ + "max_percent": self.max_percent + })) + } + + fn critical(&self) -> bool { + false + } +} diff --git a/src/shared/utils/infra/import_export.rs b/src/shared/utils/infra/import_export.rs new file mode 100644 index 0000000..c84ec80 --- /dev/null +++ b/src/shared/utils/infra/import_export.rs @@ -0,0 +1,223 @@ +//! Import/Export helpers for CSV, JSON data. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::import_export::{CsvExporter, JsonExporter}; +//! +//! // Export to CSV +//! let csv = CsvExporter::export(&users)?; +//! +//! // Import from JSON +//! let users: Vec = JsonImporter::import(&json_str)?; +//! ``` + +use serde::{de::DeserializeOwned, Serialize}; + +/// Import/Export error. +#[derive(Debug, thiserror::Error)] +pub enum ImportExportError { + #[error("Parse error: {0}")] + ParseError(String), + #[error("IO error: {0}")] + IoError(String), + #[error("Invalid format")] + InvalidFormat, +} + +// ============================================================================= +// JSON Import/Export +// ============================================================================= + +/// Export items to JSON string. +pub fn export_json(items: &[T]) -> Result { + serde_json::to_string_pretty(items).map_err(|e| ImportExportError::ParseError(e.to_string())) +} + +/// Export items to JSON file. +pub fn export_json_file(items: &[T], path: &str) -> Result<(), ImportExportError> { + let json = export_json(items)?; + std::fs::write(path, json).map_err(|e| ImportExportError::IoError(e.to_string())) +} + +/// Import from JSON string. +pub fn import_json(json: &str) -> Result, ImportExportError> { + serde_json::from_str(json).map_err(|e| ImportExportError::ParseError(e.to_string())) +} + +/// Import from JSON file. +pub fn import_json_file(path: &str) -> Result, ImportExportError> { + let content = + std::fs::read_to_string(path).map_err(|e| ImportExportError::IoError(e.to_string()))?; + import_json(&content) +} + +// ============================================================================= +// CSV Import/Export (Simple implementation) +// ============================================================================= + +/// Export items to CSV string. +pub fn export_csv(items: &[T]) -> Result { + if items.is_empty() { + return Ok(String::new()); + } + + // Convert to JSON first to get field names + let json_items: Vec = items + .iter() + .map(|item| serde_json::to_value(item)) + .collect::, _>>() + .map_err(|e| ImportExportError::ParseError(e.to_string()))?; + + // Get headers from first item + let headers: Vec = match &json_items[0] { + serde_json::Value::Object(map) => map.keys().cloned().collect(), + _ => return Err(ImportExportError::InvalidFormat), + }; + + let mut csv = String::new(); + + // Write header + csv.push_str(&headers.join(",")); + csv.push('\n'); + + // Write rows + for item in &json_items { + if let serde_json::Value::Object(map) = item { + let row: Vec = headers + .iter() + .map(|h| map.get(h).map(|v| escape_csv_value(v)).unwrap_or_default()) + .collect(); + csv.push_str(&row.join(",")); + csv.push('\n'); + } + } + + Ok(csv) +} + +/// Export to CSV file. +pub fn export_csv_file(items: &[T], path: &str) -> Result<(), ImportExportError> { + let csv = export_csv(items)?; + std::fs::write(path, csv).map_err(|e| ImportExportError::IoError(e.to_string())) +} + +/// Import from CSV string. +pub fn import_csv(csv: &str) -> Result, ImportExportError> { + let mut lines = csv.lines(); + + // Parse header + let header_line = lines.next().ok_or(ImportExportError::InvalidFormat)?; + let headers: Vec<&str> = parse_csv_line(header_line); + + // Parse rows + let mut items = Vec::new(); + for line in lines { + if line.trim().is_empty() { + continue; + } + + let values = parse_csv_line(line); + let mut map = serde_json::Map::new(); + + for (i, header) in headers.iter().enumerate() { + let value = values.get(i).copied().unwrap_or(""); + map.insert( + header.to_string(), + if value.is_empty() { + serde_json::Value::Null + } else if let Ok(n) = value.parse::() { + serde_json::Value::Number(n.into()) + } else if let Ok(f) = value.parse::() { + serde_json::Value::Number(serde_json::Number::from_f64(f).unwrap_or(0.into())) + } else if value == "true" { + serde_json::Value::Bool(true) + } else if value == "false" { + serde_json::Value::Bool(false) + } else { + serde_json::Value::String(value.to_string()) + }, + ); + } + + let item: T = serde_json::from_value(serde_json::Value::Object(map)) + .map_err(|e| ImportExportError::ParseError(e.to_string()))?; + items.push(item); + } + + Ok(items) +} + +/// Import from CSV file. +pub fn import_csv_file(path: &str) -> Result, ImportExportError> { + let content = + std::fs::read_to_string(path).map_err(|e| ImportExportError::IoError(e.to_string()))?; + import_csv(&content) +} + +fn escape_csv_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(s) => { + if s.contains(',') || s.contains('"') || s.contains('\n') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.clone() + } + } + serde_json::Value::Number(n) => n.to_string(), + serde_json::Value::Bool(b) => b.to_string(), + serde_json::Value::Null => String::new(), + _ => value.to_string(), + } +} + +fn parse_csv_line(line: &str) -> Vec<&str> { + // Simple CSV parsing (doesn't handle all edge cases) + let mut fields = Vec::new(); + let mut start = 0; + let mut in_quotes = false; + + for (i, c) in line.char_indices() { + match c { + '"' => in_quotes = !in_quotes, + ',' if !in_quotes => { + fields.push(&line[start..i]); + start = i + 1; + } + _ => {} + } + } + fields.push(&line[start..]); + + // Remove surrounding quotes + fields + .into_iter() + .map(|f| f.trim().trim_matches('"')) + .collect() +} + +// ============================================================================= +// NDJSON (Newline Delimited JSON) +// ============================================================================= + +/// Export to NDJSON. +pub fn export_ndjson(items: &[T]) -> Result { + let lines: Vec = items + .iter() + .map(|item| serde_json::to_string(item)) + .collect::, _>>() + .map_err(|e| ImportExportError::ParseError(e.to_string()))?; + + Ok(lines.join("\n")) +} + +/// Import from NDJSON. +pub fn import_ndjson(ndjson: &str) -> Result, ImportExportError> { + ndjson + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|line| { + serde_json::from_str(line).map_err(|e| ImportExportError::ParseError(e.to_string())) + }) + .collect() +} diff --git a/src/shared/utils/infra/mod.rs b/src/shared/utils/infra/mod.rs new file mode 100644 index 0000000..f1ad2c3 --- /dev/null +++ b/src/shared/utils/infra/mod.rs @@ -0,0 +1,14 @@ +pub mod bulk; +pub mod console; +pub mod encryption; +pub mod env; +pub mod form_request; +pub mod health_check; +pub mod import_export; +pub mod query_profiler; +pub mod resource; +pub mod ryzen_cdn; +pub mod searchable; +pub mod transaction; +pub mod uuid_utils; +pub mod versioning; diff --git a/src/shared/utils/infra/query_profiler.rs b/src/shared/utils/infra/query_profiler.rs new file mode 100644 index 0000000..83773a1 --- /dev/null +++ b/src/shared/utils/infra/query_profiler.rs @@ -0,0 +1,277 @@ +//! Database Query Profiling. +//! +//! Track and analyze database query performance. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::query_profiler::{QueryProfiler, QueryLog}; +//! +//! let profiler = QueryProfiler::new(); +//! +//! profiler.log("SELECT * FROM users WHERE id = ?", duration); +//! +//! // Get slow queries +//! let slow = profiler.slow_queries(100); // > 100ms +//! ``` + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +/// Query log entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryLog { + /// Query string (may be truncated). + pub query: String, + /// Execution duration in milliseconds. + pub duration_ms: u64, + /// Timestamp. + pub timestamp: DateTime, + /// Rows affected (if available). + pub rows_affected: Option, + /// Location in code (file:line). + pub location: Option, +} + +/// Query statistics. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct QueryStats { + pub total_queries: u64, + pub total_duration_ms: u64, + pub slow_queries: u64, + pub avg_duration_ms: f64, + pub max_duration_ms: u64, +} + +/// Query profiler. +#[derive(Clone)] +pub struct QueryProfiler { + logs: Arc>>, + stats: Arc, + enabled: Arc, + slow_threshold_ms: u64, + max_logs: usize, +} + +struct ProfilerStats { + total_queries: AtomicU64, + total_duration_ms: AtomicU64, + slow_queries: AtomicU64, + max_duration_ms: AtomicU64, +} + +impl Default for QueryProfiler { + fn default() -> Self { + Self::new() + } +} + +impl QueryProfiler { + /// Create a new profiler. + pub fn new() -> Self { + Self { + logs: Arc::new(RwLock::new(Vec::new())), + stats: Arc::new(ProfilerStats { + total_queries: AtomicU64::new(0), + total_duration_ms: AtomicU64::new(0), + slow_queries: AtomicU64::new(0), + max_duration_ms: AtomicU64::new(0), + }), + enabled: Arc::new(std::sync::atomic::AtomicBool::new(true)), + slow_threshold_ms: 100, + max_logs: 1000, + } + } + + /// Create with custom settings. + pub fn with_settings(slow_threshold_ms: u64, max_logs: usize) -> Self { + Self { + slow_threshold_ms, + max_logs, + ..Self::new() + } + } + + /// Enable profiling. + pub fn enable(&self) { + self.enabled.store(true, Ordering::SeqCst); + } + + /// Disable profiling. + pub fn disable(&self) { + self.enabled.store(false, Ordering::SeqCst); + } + + /// Check if profiling is enabled. + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } + + /// Log a query. + pub fn log(&self, query: &str, duration: Duration) { + if !self.is_enabled() { + return; + } + + let duration_ms = duration.as_millis() as u64; + + // Update stats + self.stats.total_queries.fetch_add(1, Ordering::SeqCst); + self.stats + .total_duration_ms + .fetch_add(duration_ms, Ordering::SeqCst); + + if duration_ms > self.slow_threshold_ms { + self.stats.slow_queries.fetch_add(1, Ordering::SeqCst); + tracing::warn!( + "Slow query ({}ms): {}", + duration_ms, + truncate_query(query, 200) + ); + } + + // Update max + let mut current_max = self.stats.max_duration_ms.load(Ordering::SeqCst); + while duration_ms > current_max { + match self.stats.max_duration_ms.compare_exchange( + current_max, + duration_ms, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => break, + Err(v) => current_max = v, + } + } + + // Add to logs + let log = QueryLog { + query: truncate_query(query, 500), + duration_ms, + timestamp: Utc::now(), + rows_affected: None, + location: None, + }; + + if let Ok(mut logs) = self.logs.write() { + logs.push(log); + // Keep only last max_logs entries + let len = logs.len(); + if len > self.max_logs { + logs.drain(0..len - self.max_logs); + } + } + } + + /// Log a query with additional info. + pub fn log_full( + &self, + query: &str, + duration: Duration, + rows_affected: Option, + location: Option<&str>, + ) { + if !self.is_enabled() { + return; + } + + let duration_ms = duration.as_millis() as u64; + + self.stats.total_queries.fetch_add(1, Ordering::SeqCst); + self.stats + .total_duration_ms + .fetch_add(duration_ms, Ordering::SeqCst); + + if duration_ms > self.slow_threshold_ms { + self.stats.slow_queries.fetch_add(1, Ordering::SeqCst); + } + + let log = QueryLog { + query: truncate_query(query, 500), + duration_ms, + timestamp: Utc::now(), + rows_affected, + location: location.map(String::from), + }; + + if let Ok(mut logs) = self.logs.write() { + logs.push(log); + let len = logs.len(); + if len > self.max_logs { + logs.drain(0..len - self.max_logs); + } + } + } + + /// Get all query logs. + pub fn get_logs(&self) -> Vec { + self.logs.read().map(|l| l.clone()).unwrap_or_default() + } + + /// Get slow queries. + pub fn slow_queries(&self, threshold_ms: u64) -> Vec { + self.logs + .read() + .map(|logs| { + logs.iter() + .filter(|l| l.duration_ms > threshold_ms) + .cloned() + .collect() + }) + .unwrap_or_default() + } + + /// Get query statistics. + pub fn stats(&self) -> QueryStats { + let total = self.stats.total_queries.load(Ordering::SeqCst); + let total_duration = self.stats.total_duration_ms.load(Ordering::SeqCst); + + QueryStats { + total_queries: total, + total_duration_ms: total_duration, + slow_queries: self.stats.slow_queries.load(Ordering::SeqCst), + avg_duration_ms: if total > 0 { + total_duration as f64 / total as f64 + } else { + 0.0 + }, + max_duration_ms: self.stats.max_duration_ms.load(Ordering::SeqCst), + } + } + + /// Clear logs and reset stats. + pub fn clear(&self) { + if let Ok(mut logs) = self.logs.write() { + logs.clear(); + } + self.stats.total_queries.store(0, Ordering::SeqCst); + self.stats.total_duration_ms.store(0, Ordering::SeqCst); + self.stats.slow_queries.store(0, Ordering::SeqCst); + self.stats.max_duration_ms.store(0, Ordering::SeqCst); + } +} + +fn truncate_query(query: &str, max_len: usize) -> String { + let query = query.trim(); + if query.len() <= max_len { + query.to_string() + } else { + format!("{}...", &query[..max_len]) + } +} + +/// Global query profiler. +static PROFILER: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Initialize global profiler. +pub fn init_profiler() -> &'static QueryProfiler { + PROFILER.get_or_init(QueryProfiler::new) +} + +/// Get global profiler. +pub fn profiler() -> Option<&'static QueryProfiler> { + PROFILER.get() +} diff --git a/src/shared/utils/infra/resource.rs b/src/shared/utils/infra/resource.rs new file mode 100644 index 0000000..e0d7983 --- /dev/null +++ b/src/shared/utils/infra/resource.rs @@ -0,0 +1,198 @@ +//! API Resources / Transformers. +//! +//! Transform models for API output with field selection and relationships. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::resource::{Resource, ResourceCollection}; +//! +//! struct UserResource; +//! impl Resource for UserResource { +//! type Model = User; +//! fn transform(model: &Self::Model) -> serde_json::Value { +//! json!({ "id": model.id, "name": model.name }) +//! } +//! } +//! ``` + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::HashMap; + +/// Resource trait for transforming models to API output. +pub trait Resource { + /// The model type being transformed. + type Model; + + /// Transform a single model to JSON. + fn transform(model: &Self::Model) -> Value; + + /// Transform with additional data. + fn transform_with(model: &Self::Model, _extra: &HashMap) -> Value { + Self::transform(model) + } + + /// Get field whitelist (if any). + fn fields() -> Option> { + None + } + + /// Get hidden fields. + fn hidden() -> Vec<&'static str> { + vec![] + } +} + +/// Resource collection for paginated results. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResourceCollection { + pub data: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub links: Option, +} + +/// Collection metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollectionMeta { + pub current_page: u64, + pub per_page: u64, + pub total: u64, + pub total_pages: u64, +} + +/// Collection links. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollectionLinks { + #[serde(skip_serializing_if = "Option::is_none")] + pub first: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prev: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +impl ResourceCollection { + /// Create a simple collection without pagination. + pub fn new(data: Vec) -> Self { + Self { + data, + meta: None, + links: None, + } + } + + /// Create a paginated collection. + pub fn paginated(data: Vec, page: u64, per_page: u64, total: u64) -> Self { + let total_pages = (total as f64 / per_page as f64).ceil() as u64; + Self { + data, + meta: Some(CollectionMeta { + current_page: page, + per_page, + total, + total_pages, + }), + links: None, + } + } + + /// Add links. + pub fn with_links(mut self, base_url: &str, page: u64, total_pages: u64) -> Self { + let first = Some(format!("{}?page=1", base_url)); + let last = Some(format!("{}?page={}", base_url, total_pages)); + let prev = if page > 1 { + Some(format!("{}?page={}", base_url, page - 1)) + } else { + None + }; + let next = if page < total_pages { + Some(format!("{}?page={}", base_url, page + 1)) + } else { + None + }; + + self.links = Some(CollectionLinks { + first, + last, + prev, + next, + }); + self + } +} + +/// Transform a collection of models using a resource. +pub fn collection(models: &[R::Model]) -> Vec { + models.iter().map(R::transform).collect() +} + +/// Transform a single model using a resource. +pub fn item(model: &R::Model) -> Value { + R::transform(model) +} + +/// Filter fields from a JSON value. +pub fn only(value: Value, fields: &[&str]) -> Value { + if let Value::Object(map) = value { + let filtered: serde_json::Map = map + .into_iter() + .filter(|(k, _)| fields.contains(&k.as_str())) + .collect(); + Value::Object(filtered) + } else { + value + } +} + +/// Remove fields from a JSON value. +pub fn except(value: Value, fields: &[&str]) -> Value { + if let Value::Object(map) = value { + let filtered: serde_json::Map = map + .into_iter() + .filter(|(k, _)| !fields.contains(&k.as_str())) + .collect(); + Value::Object(filtered) + } else { + value + } +} + +/// Merge additional data into a JSON object. +pub fn merge(mut value: Value, extra: Value) -> Value { + if let (Value::Object(ref mut map1), Value::Object(map2)) = (&mut value, extra) { + for (k, v) in map2 { + map1.insert(k, v); + } + } + value +} + +/// Wrap data in a standard API envelope. +pub fn envelope(data: Value) -> Value { + json!({ "data": data }) +} + +/// Wrap data with success status. +pub fn success(data: Value) -> Value { + json!({ + "success": true, + "data": data + }) +} + +/// Wrap error response. +pub fn error(message: &str, code: Option<&str>) -> Value { + let mut resp = json!({ + "success": false, + "error": { "message": message } + }); + if let Some(c) = code { + resp["error"]["code"] = Value::String(c.to_string()); + } + resp +} diff --git a/src/shared/utils/infra/ryzen_cdn.rs b/src/shared/utils/infra/ryzen_cdn.rs new file mode 100644 index 0000000..79ed75c --- /dev/null +++ b/src/shared/utils/infra/ryzen_cdn.rs @@ -0,0 +1,67 @@ +use crate::shared::errors::AppError; +use infer; // For file type detection +use reqwest::{multipart, Client}; +use serde::{Deserialize, Serialize}; +use tracing::error; + +#[derive(Debug, Serialize, Deserialize)] +pub struct RyzenCDNResponse { + pub success: bool, + pub message: Option, + pub url: Option, +} + +pub async fn ryzen_cdn( + inp: Vec, // Simplified input to a single byte vector for now + original_name: Option, +) -> Result { + let client = Client::new(); + let form = multipart::Form::new(); + + let file_type = infer::get(&inp); + let mime_type = file_type.as_ref().map(|t| t.mime_type()); + let extension = file_type.as_ref().map(|t| t.extension()); + + let file_name = if let Some(name) = original_name { + if let Some(ext) = extension { + format!("{}.{}", name.split('.').next().unwrap_or("file"), ext) + } else { + name + } + } else { + "file".to_string() + }; + + let part = multipart::Part::bytes(inp) + .file_name(file_name) + .mime_str(mime_type.unwrap_or("application/octet-stream"))?; + + let form = form.part("file", part); + + let res = client.post("https://api.ryzumi.vip/api/uploader/ryzencdn") + .multipart(form) + .header("accept", "application/json") + .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0") + .header("Connection", "keep-alive") + .header("Accept-Encoding", "gzip, deflate, br") + .send() + .await?; + + let json_response: RyzenCDNResponse = res.json().await?; + + if !json_response.success { + let error_message = json_response + .message + .unwrap_or_else(|| "Upload failed".to_string()); + error!("RyzenCDN Error: {}", error_message); + return Err(AppError::Other(error_message)); + } + + if let Some(url) = json_response.url { + Ok(url) + } else { + Err(AppError::Other( + "RyzenCDN Error: URL not found in response".to_string(), + )) + } +} diff --git a/src/shared/utils/infra/searchable.rs b/src/shared/utils/infra/searchable.rs new file mode 100644 index 0000000..a376d09 --- /dev/null +++ b/src/shared/utils/infra/searchable.rs @@ -0,0 +1,190 @@ +//! Searchable trait for simple full-text search. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::searchable::{Searchable, SearchQuery, search}; +//! +//! impl Searchable for User { +//! fn searchable_fields() -> Vec<&'static str> { +//! vec!["name", "email", "bio"] +//! } +//! } +//! +//! let query = SearchQuery::new("john") +//! .fields(vec!["name", "email"]); +//! ``` + +use sea_orm::{ColumnTrait, Condition}; +use serde::{Deserialize, Serialize}; + +/// Searchable trait for models. +pub trait Searchable { + /// Get searchable field names. + fn searchable_fields() -> Vec<&'static str>; + + /// Get default search weight for a field (1-10). + fn field_weight(_field: &str) -> u32 { + 1 + } +} + +/// Search query options. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SearchQuery { + /// Search term. + pub term: String, + /// Fields to search (empty = all). + pub fields: Vec, + /// Minimum match score. + pub min_score: f32, + /// Fuzzy matching. + pub fuzzy: bool, +} + +impl SearchQuery { + pub fn new(term: &str) -> Self { + Self { + term: term.to_string(), + fields: Vec::new(), + min_score: 0.0, + fuzzy: false, + } + } + + pub fn fields(mut self, fields: Vec<&str>) -> Self { + self.fields = fields.into_iter().map(String::from).collect(); + self + } + + pub fn min_score(mut self, score: f32) -> Self { + self.min_score = score; + self + } + + pub fn fuzzy(mut self) -> Self { + self.fuzzy = true; + self + } +} + +/// Search result with scoring. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchResult { + pub item: T, + pub score: f32, + pub highlights: Vec, +} + +/// Search highlight. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchHighlight { + pub field: String, + pub snippet: String, +} + +/// Simple text matching score. +pub fn calculate_score(text: &str, query: &str, fuzzy: bool) -> f32 { + let text_lower = text.to_lowercase(); + let query_lower = query.to_lowercase(); + + if text_lower == query_lower { + return 1.0; + } + + if text_lower.contains(&query_lower) { + // Position-based scoring + let pos = text_lower.find(&query_lower).unwrap_or(0); + let pos_score = 1.0 - (pos as f32 / text.len() as f32); + return 0.5 + (pos_score * 0.3); + } + + if fuzzy { + // Simple fuzzy: word overlap + let text_words: Vec<&str> = text_lower.split_whitespace().collect(); + let query_words: Vec<&str> = query_lower.split_whitespace().collect(); + + let mut matches = 0; + for qw in &query_words { + for tw in &text_words { + if tw.contains(qw) || qw.contains(tw) { + matches += 1; + break; + } + } + } + + if !query_words.is_empty() { + return matches as f32 / query_words.len() as f32 * 0.5; + } + } + + 0.0 +} + +/// Generate highlight snippet. +pub fn generate_highlight(text: &str, query: &str, context_len: usize) -> Option { + let text_lower = text.to_lowercase(); + let query_lower = query.to_lowercase(); + + if let Some(pos) = text_lower.find(&query_lower) { + let start = pos.saturating_sub(context_len); + let end = (pos + query.len() + context_len).min(text.len()); + + let mut snippet = String::new(); + if start > 0 { + snippet.push_str("..."); + } + snippet.push_str(&text[start..end]); + if end < text.len() { + snippet.push_str("..."); + } + + return Some(snippet); + } + + None +} + +/// Search in a vector of strings. +pub fn search_in_vec(items: &[String], query: &SearchQuery) -> Vec<(usize, f32)> { + let mut results: Vec<(usize, f32)> = items + .iter() + .enumerate() + .filter_map(|(i, text)| { + let score = calculate_score(text, &query.term, query.fuzzy); + if score >= query.min_score { + Some((i, score)) + } else { + None + } + }) + .collect(); + + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + results +} + +/// Build LIKE conditions for search. +pub fn build_like_condition(columns: Vec, term: &str) -> Condition { + let pattern = format!("%{}%", term); + let mut condition = Condition::any(); + + for col in columns { + condition = condition.add(col.contains(&pattern)); + } + + condition +} + +/// Macro to implement searchable for entity. +#[macro_export] +macro_rules! impl_searchable { + ($entity:ty, $($field:ident),+) => { + impl $crate::shared::utils::searchable::Searchable for $entity { + fn searchable_fields() -> Vec<&'static str> { + vec![$(stringify!($field)),+] + } + } + }; +} diff --git a/src/shared/utils/infra/transaction.rs b/src/shared/utils/infra/transaction.rs new file mode 100644 index 0000000..2e00803 --- /dev/null +++ b/src/shared/utils/infra/transaction.rs @@ -0,0 +1,179 @@ +//! Database Transaction helpers. +//! +//! Atomic database operations with automatic rollback. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::transaction::{transaction, TransactionExt}; +//! +//! let result = transaction(&db, |txn| async move { +//! User::insert(user1).exec(&txn).await?; +//! User::insert(user2).exec(&txn).await?; +//! Ok(()) +//! }).await?; +//! ``` + +use sea_orm::{DatabaseConnection, DatabaseTransaction, DbErr, TransactionTrait}; +use std::future::Future; + +/// Transaction error. +#[derive(Debug, thiserror::Error)] +pub enum TransactionError { + #[error("Database error: {0}")] + DbError(#[from] DbErr), + #[error("Transaction failed: {0}")] + Failed(String), +} + +/// Execute a closure within a database transaction. +pub async fn transaction(db: &DatabaseConnection, f: F) -> Result +where + F: FnOnce(DatabaseTransaction) -> Fut, + Fut: Future>, +{ + let txn = db.begin().await?; + + match f(txn).await { + Ok(result) => Ok(result), + Err(e) => Err(TransactionError::DbError(e)), + } +} + +/// Execute with automatic commit/rollback. +pub async fn with_transaction( + db: &DatabaseConnection, + f: F, +) -> Result +where + F: FnOnce(&DatabaseTransaction) -> Fut, + Fut: Future>, +{ + let txn = db.begin().await?; + + match f(&txn).await { + Ok(result) => { + txn.commit().await?; + Ok(result) + } + Err(e) => { + txn.rollback().await?; + Err(TransactionError::DbError(e)) + } + } +} + +/// Nested transaction support (savepoints). +pub struct NestedTransaction { + depth: usize, +} + +impl NestedTransaction { + pub fn new() -> Self { + Self { depth: 0 } + } + + pub fn begin(&mut self) -> usize { + self.depth += 1; + self.depth + } + + pub fn commit(&mut self) -> usize { + if self.depth > 0 { + self.depth -= 1; + } + self.depth + } + + pub fn rollback(&mut self) -> usize { + if self.depth > 0 { + self.depth -= 1; + } + self.depth + } + + pub fn depth(&self) -> usize { + self.depth + } +} + +impl Default for NestedTransaction { + fn default() -> Self { + Self::new() + } +} + +/// Retry transaction on deadlock. +pub async fn retry_transaction( + db: &DatabaseConnection, + max_retries: usize, + f: F, +) -> Result +where + F: Fn() -> Fut + Clone, + Fut: Future>, +{ + let mut attempts = 0; + let mut last_error = None; + + while attempts < max_retries { + let txn = db.begin().await?; + + match f().await { + Ok(result) => { + txn.commit().await?; + return Ok(result); + } + Err(e) => { + txn.rollback().await.ok(); + + // Check if deadlock (simplified - actual check would be DB-specific) + let is_deadlock = e.to_string().to_lowercase().contains("deadlock"); + + if is_deadlock && attempts < max_retries - 1 { + attempts += 1; + tokio::time::sleep(tokio::time::Duration::from_millis(100 * attempts as u64)) + .await; + continue; + } + + last_error = Some(e); + break; + } + } + } + + Err(TransactionError::DbError(last_error.unwrap())) +} + +/// Transaction context for tracking. +#[derive(Debug, Clone)] +pub struct TransactionContext { + pub id: String, + pub started_at: chrono::DateTime, + pub operations: usize, +} + +impl TransactionContext { + pub fn new() -> Self { + Self { + id: uuid::Uuid::new_v4().to_string(), + started_at: chrono::Utc::now(), + operations: 0, + } + } + + pub fn record_operation(&mut self) { + self.operations += 1; + } + + pub fn duration(&self) -> chrono::Duration { + chrono::Utc::now() - self.started_at + } +} + +impl Default for TransactionContext { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shared/utils/infra/uuid_utils.rs b/src/shared/utils/infra/uuid_utils.rs new file mode 100644 index 0000000..4c89595 --- /dev/null +++ b/src/shared/utils/infra/uuid_utils.rs @@ -0,0 +1,78 @@ +//! UUID utilities. + +use uuid::Uuid; + +/// Generate a new UUID v4. +pub fn new_v4() -> String { + Uuid::new_v4().to_string() +} + +/// Generate a new UUID v4 without hyphens. +pub fn new_v4_simple() -> String { + Uuid::new_v4().simple().to_string() +} + +/// Parse a UUID string. +pub fn parse(s: &str) -> Result { + Uuid::parse_str(s) +} + +/// Check if string is valid UUID. +pub fn is_valid(s: &str) -> bool { + Uuid::parse_str(s).is_ok() +} + +/// Convert to hyphenated format. +pub fn to_hyphenated(s: &str) -> Option { + Uuid::parse_str(s).ok().map(|u| u.hyphenated().to_string()) +} + +/// Convert to simple format (no hyphens). +pub fn to_simple(s: &str) -> Option { + Uuid::parse_str(s).ok().map(|u| u.simple().to_string()) +} + +/// Generate a nil UUID (all zeros). +pub fn nil() -> String { + Uuid::nil().to_string() +} + +/// Check if UUID is nil. +pub fn is_nil(s: &str) -> bool { + parse(s).map(|u| u.is_nil()).unwrap_or(false) +} + +/// Extract timestamp from UUID v7 (if applicable). +pub fn timestamp_v7(s: &str) -> Option { + parse(s).ok().and_then(|u| { + if u.get_version() == Some(uuid::Version::SortRand) { + u.get_timestamp().map(|ts| { + let (secs, _) = ts.to_unix(); + secs + }) + } else { + None + } + }) +} + +/// Create a UUID namespace for v5. +pub fn namespace(ns: &str) -> Option { + match ns.to_lowercase().as_str() { + "dns" => Some(Uuid::NAMESPACE_DNS), + "url" => Some(Uuid::NAMESPACE_URL), + "oid" => Some(Uuid::NAMESPACE_OID), + "x500" => Some(Uuid::NAMESPACE_X500), + _ => Uuid::parse_str(ns).ok(), + } +} + +/// Generate a short ID (first 8 chars of UUID). +pub fn short_id() -> String { + Uuid::new_v4().simple().to_string()[..8].to_string() +} + +/// Generate a medium ID (first 12 chars of UUID). +pub fn medium_id() -> String { + Uuid::new_v4().simple().to_string()[..12].to_string() +} diff --git a/src/shared/utils/infra/versioning.rs b/src/shared/utils/infra/versioning.rs new file mode 100644 index 0000000..47256c1 --- /dev/null +++ b/src/shared/utils/infra/versioning.rs @@ -0,0 +1,191 @@ +//! Versioned API helpers. +//! +//! API version extraction and routing. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::versioning::{ApiVersion, extract_version}; +//! +//! // From header: Accept: application/vnd.api+json; version=2 +//! let version = extract_version(&headers); +//! +//! // From path: /api/v2/users +//! let version = ApiVersion::from_path("/api/v2/users"); +//! ``` + +use axum::http::{header::ACCEPT, HeaderMap}; +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; + +/// API version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ApiVersion { + pub major: u32, + pub minor: u32, +} + +impl ApiVersion { + /// Create a new version. + pub const fn new(major: u32, minor: u32) -> Self { + Self { major, minor } + } + + /// Create from major version only. + pub const fn major_only(major: u32) -> Self { + Self { major, minor: 0 } + } + + /// Parse from string like "v2" or "2.1". + pub fn parse(s: &str) -> Option { + let s = s.trim().trim_start_matches('v').trim_start_matches('V'); + + if let Some((major, minor)) = s.split_once('.') { + Some(Self { + major: major.parse().ok()?, + minor: minor.parse().ok()?, + }) + } else { + Some(Self { + major: s.parse().ok()?, + minor: 0, + }) + } + } + + /// Extract from URL path like /api/v2/users. + pub fn from_path(path: &str) -> Option { + for segment in path.split('/') { + if segment.starts_with('v') || segment.starts_with('V') { + if let Some(v) = Self::parse(segment) { + return Some(v); + } + } + } + None + } + + /// Extract from Accept header. + pub fn from_accept_header(accept: &str) -> Option { + // Format: application/vnd.api+json; version=2 + for part in accept.split(';') { + let part = part.trim(); + if part.starts_with("version=") { + let version = part.trim_start_matches("version="); + return Self::parse(version); + } + } + None + } + + /// Check if version is at least the given version. + pub fn at_least(&self, major: u32, minor: u32) -> bool { + self.major > major || (self.major == major && self.minor >= minor) + } + + /// Check if version is below the given version. + pub fn below(&self, major: u32, minor: u32) -> bool { + !self.at_least(major, minor) + } +} + +impl Default for ApiVersion { + fn default() -> Self { + Self { major: 1, minor: 0 } + } +} + +impl std::fmt::Display for ApiVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "v{}.{}", self.major, self.minor) + } +} + +impl PartialOrd for ApiVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ApiVersion { + fn cmp(&self, other: &Self) -> Ordering { + match self.major.cmp(&other.major) { + Ordering::Equal => self.minor.cmp(&other.minor), + other => other, + } + } +} + +/// Extract API version from request. +pub fn extract_version(headers: &HeaderMap) -> ApiVersion { + // Try Accept header first + if let Some(accept) = headers.get(ACCEPT).and_then(|h| h.to_str().ok()) { + if let Some(v) = ApiVersion::from_accept_header(accept) { + return v; + } + } + + // Try custom header + if let Some(version) = headers.get("X-API-Version").and_then(|h| h.to_str().ok()) { + if let Some(v) = ApiVersion::parse(version) { + return v; + } + } + + ApiVersion::default() +} + +/// Common API versions. +pub mod versions { + use super::ApiVersion; + + pub const V1: ApiVersion = ApiVersion::new(1, 0); + pub const V2: ApiVersion = ApiVersion::new(2, 0); + pub const V3: ApiVersion = ApiVersion::new(3, 0); +} + +/// Version constraint for routing. +#[derive(Debug, Clone)] +pub struct VersionConstraint { + pub min: Option, + pub max: Option, +} + +impl VersionConstraint { + pub fn new() -> Self { + Self { + min: None, + max: None, + } + } + + pub fn min(mut self, version: ApiVersion) -> Self { + self.min = Some(version); + self + } + + pub fn max(mut self, version: ApiVersion) -> Self { + self.max = Some(version); + self + } + + pub fn matches(&self, version: ApiVersion) -> bool { + if let Some(min) = self.min { + if version < min { + return false; + } + } + if let Some(max) = self.max { + if version > max { + return false; + } + } + true + } +} + +impl Default for VersionConstraint { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shared/utils/io/cache.rs b/src/shared/utils/io/cache.rs new file mode 100644 index 0000000..7fa0017 --- /dev/null +++ b/src/shared/utils/io/cache.rs @@ -0,0 +1,155 @@ +//! Redis caching helpers. + +use crate::shared::utils::cache_ttl::CACHE_TTL_VERY_SHORT; +use deadpool_redis::redis::AsyncCommands; +use deadpool_redis::Pool; +use serde::{de::DeserializeOwned, Serialize}; +use tracing::{debug, error}; + +/// Default cache TTL in seconds (5 minutes). +pub const DEFAULT_CACHE_TTL: u64 = CACHE_TTL_VERY_SHORT; + +/// Cache helper for Redis operations. +pub struct Cache<'a> { + pool: &'a Pool, +} + +impl<'a> Cache<'a> { + /// Create a new cache helper. + pub fn new(pool: &'a Pool) -> Self { + Self { pool } + } + + /// Get a value from cache, deserializing JSON. + pub async fn get(&self, key: &str) -> Option { + let mut conn = match self.pool.get().await { + Ok(c) => c, + Err(e) => { + error!("Cache: failed to get connection: {}", e); + return None; + } + }; + + let cached: Option = conn.get(key).await.ok()?; + + if cached.is_some() { + debug!("Cache hit: {}", key); + } else { + debug!("Cache miss: {}", key); + } + + cached.and_then(|json| serde_json::from_str(&json).ok()) + } + + /// Get multiple values from cache, deserializing JSON. + /// Returns a vector of Options, preserving order of keys. + pub async fn mget(&self, keys: &[String]) -> Vec> { + if keys.is_empty() { + return Vec::new(); + } + + let mut conn = match self.pool.get().await { + Ok(c) => c, + Err(e) => { + error!("Cache: failed to get connection: {}", e); + return std::iter::repeat_with(|| None).take(keys.len()).collect(); + } + }; + + // Use low-level cmd interface for MGET to ensure correct command usage + use deadpool_redis::redis::cmd; + let cached_values: Vec> = + match cmd("MGET").arg(keys).query_async(&mut conn).await { + Ok(v) => v, + Err(e) => { + error!("Cache: failed to mget values: {}", e); + return std::iter::repeat_with(|| None).take(keys.len()).collect(); + } + }; + + cached_values + .into_iter() + .map(|opt_s| opt_s.and_then(|json| serde_json::from_str(&json).ok())) + .collect() + } + + /// Set a value in cache with default TTL (5 minutes). + pub async fn set(&self, key: &str, value: &T) -> Result<(), String> { + self.set_with_ttl(key, value, DEFAULT_CACHE_TTL).await + } + + /// Set a value in cache with custom TTL. + pub async fn set_with_ttl( + &self, + key: &str, + value: &T, + ttl_secs: u64, + ) -> Result<(), String> { + let mut conn = self.pool.get().await.map_err(|e| e.to_string())?; + + let json = serde_json::to_string(value).map_err(|e| e.to_string())?; + + conn.set_ex::<_, _, ()>(key, json, ttl_secs) + .await + .map_err(|e| e.to_string())?; + + debug!("Cache: set key {} with TTL {}s", key, ttl_secs); + Ok(()) + } + + /// Delete a key from cache. + pub async fn delete(&self, key: &str) -> Result<(), String> { + let mut conn = self.pool.get().await.map_err(|e| e.to_string())?; + conn.del::<_, ()>(key).await.map_err(|e| e.to_string())?; + debug!("Cache: deleted key {}", key); + Ok(()) + } + + /// Check if key exists. + pub async fn exists(&self, key: &str) -> bool { + let mut conn = match self.pool.get().await { + Ok(c) => c, + Err(_) => return false, + }; + conn.exists::<_, bool>(key).await.unwrap_or(false) + } + + /// Get or set: returns cached value or computes and caches new value. + pub async fn get_or_set( + &self, + key: &str, + ttl_secs: u64, + compute: F, + ) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Try cache first + if let Some(cached) = self.get::(key).await { + debug!("Cache hit: {}", key); + return Ok(cached); + } + + debug!("Cache miss: {}", key); + + // Compute the value + let value = compute().await?; + + // Store in cache + self.set_with_ttl(key, &value, ttl_secs).await?; + + Ok(value) + } +} + +/// Create a cache key with prefix. +pub fn cache_key(prefix: &str, id: &str) -> String { + format!("{}:{}", prefix, id) +} + +/// Create a cache key with multiple parts. +pub fn cache_key_multi(parts: &[&str]) -> String { + parts.join(":") +} diff --git a/src/shared/utils/io/cache_tags.rs b/src/shared/utils/io/cache_tags.rs new file mode 100644 index 0000000..0f344d2 --- /dev/null +++ b/src/shared/utils/io/cache_tags.rs @@ -0,0 +1,306 @@ +//! Cache tags for tag-based cache invalidation. +//! +//! Extends the basic cache helper with tag support for grouped invalidation. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::cache_tags::TaggedCache; +//! +//! let cache = TaggedCache::new(redis_pool); +//! +//! // Set with tags +//! cache.put_tagged("user:123", data, &["users", "user:123"], 3600).await?; +//! cache.put_tagged("user:456", data2, &["users", "user:456"], 3600).await?; +//! +//! // Invalidate all users +//! cache.flush_tag("users").await?; +//! ``` + +use deadpool_redis::{redis::AsyncCommands, Pool}; +use serde::{de::DeserializeOwned, Serialize}; +use std::sync::Arc; + +/// Tagged cache error types. +#[derive(Debug, thiserror::Error)] +pub enum CacheTagError { + #[error("Redis error: {0}")] + RedisError(String), + #[error("Serialization error: {0}")] + SerializationError(String), + #[error("Deserialization error: {0}")] + DeserializationError(String), +} + +/// Cache with tag-based invalidation support. +#[derive(Clone)] +pub struct TaggedCache { + pool: Arc, + prefix: String, + tag_prefix: String, +} + +impl TaggedCache { + /// Create a new tagged cache. + pub fn new(pool: Arc) -> Self { + Self { + pool, + prefix: "cache:".to_string(), + tag_prefix: "cache:tag:".to_string(), + } + } + + /// Create with custom prefix. + pub fn with_prefix(pool: Arc, prefix: &str) -> Self { + Self { + pool, + prefix: format!("{}:", prefix), + tag_prefix: format!("{}:tag:", prefix), + } + } + + /// Generate cache key. + fn key(&self, key: &str) -> String { + format!("{}{}", self.prefix, key) + } + + /// Generate tag key. + fn tag_key(&self, tag: &str) -> String { + format!("{}{}", self.tag_prefix, tag) + } + + /// Get a value from cache. + pub async fn get(&self, key: &str) -> Result, CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let value: Option = conn.get(&cache_key).await.map_err(|e| { + tracing::error!("Redis get error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + match value { + Some(json) => { + let data: T = serde_json::from_str(&json) + .map_err(|e| CacheTagError::DeserializationError(e.to_string()))?; + Ok(Some(data)) + } + None => Ok(None), + } + } + + /// Put a value in cache with TTL (seconds). + pub async fn put( + &self, + key: &str, + value: &T, + ttl: u64, + ) -> Result<(), CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let json = serde_json::to_string(value) + .map_err(|e| CacheTagError::SerializationError(e.to_string()))?; + + conn.set_ex::<_, _, ()>(&cache_key, &json, ttl) + .await + .map_err(|e| { + tracing::error!("Redis set error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(()) + } + + /// Put a value in cache with tags. + pub async fn put_tagged( + &self, + key: &str, + value: &T, + tags: &[&str], + ttl: u64, + ) -> Result<(), CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let json = serde_json::to_string(value) + .map_err(|e| CacheTagError::SerializationError(e.to_string()))?; + + // Set the value + conn.set_ex::<_, _, ()>(&cache_key, &json, ttl) + .await + .map_err(|e| { + tracing::error!("Redis set error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + // Add key to each tag set + for tag in tags { + let tag_key = self.tag_key(tag); + conn.sadd::<_, _, ()>(&tag_key, &cache_key) + .await + .map_err(|e| { + tracing::error!("Redis sadd error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + } + + Ok(()) + } + + /// Delete a specific key. + pub async fn forget(&self, key: &str) -> Result<(), CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + conn.del::<_, ()>(&cache_key).await.map_err(|e| { + tracing::error!("Redis del error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(()) + } + + /// Flush all keys associated with a tag. + pub async fn flush_tag(&self, tag: &str) -> Result { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let tag_key = self.tag_key(tag); + + // Get all keys in the tag set + let keys: Vec = conn.smembers(&tag_key).await.map_err(|e| { + tracing::error!("Redis smembers error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let count = keys.len(); + + // Delete each key + for key in &keys { + let _: () = conn.del(key).await.unwrap_or(()); + } + + // Delete the tag set itself + conn.del::<_, ()>(&tag_key).await.map_err(|e| { + tracing::error!("Redis del error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + tracing::info!("Flushed {} keys for tag '{}'", count, tag); + Ok(count) + } + + /// Flush multiple tags at once. + pub async fn flush_tags(&self, tags: &[&str]) -> Result { + let mut total = 0; + for tag in tags { + total += self.flush_tag(tag).await?; + } + Ok(total) + } + + /// Check if a key exists. + pub async fn has(&self, key: &str) -> Result { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let exists: bool = conn.exists(&cache_key).await.map_err(|e| { + tracing::error!("Redis exists error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(exists) + } + + /// Get or set a value (cache-aside pattern). + pub async fn remember(&self, key: &str, ttl: u64, f: F) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Try to get from cache + if let Some(value) = self.get::(key).await? { + return Ok(value); + } + + // Generate value + let value = f().await?; + + // Store in cache + self.put(key, &value, ttl).await?; + + Ok(value) + } + + /// Get or set a value with tags. + pub async fn remember_tagged( + &self, + key: &str, + tags: &[&str], + ttl: u64, + f: F, + ) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Try to get from cache + if let Some(value) = self.get::(key).await? { + return Ok(value); + } + + // Generate value + let value = f().await?; + + // Store in cache with tags + self.put_tagged(key, &value, tags, ttl).await?; + + Ok(value) + } + + /// Get cache statistics for a tag. + pub async fn tag_count(&self, tag: &str) -> Result { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let tag_key = self.tag_key(tag); + let count: usize = conn.scard(&tag_key).await.map_err(|e| { + tracing::error!("Redis scard error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(count) + } +} + +/// Helper to create cache tags from entity type and ID. +pub fn entity_tags(entity_type: &str, id: &str) -> Vec { + vec![entity_type.to_string(), format!("{}:{}", entity_type, id)] +} + +/// Helper to create cache key for entity. +pub fn entity_key(entity_type: &str, id: &str) -> String { + format!("{}:{}", entity_type, id) +} diff --git a/src/shared/utils/io/cache_ttl.rs b/src/shared/utils/io/cache_ttl.rs new file mode 100644 index 0000000..9b1a0f9 --- /dev/null +++ b/src/shared/utils/io/cache_ttl.rs @@ -0,0 +1,84 @@ +//! Cache TTL constants for consistent caching across the application. + +/// Very short TTL for highly volatile data (5 minutes) +/// Use for: Real-time data, user presence, active sessions +pub const CACHE_TTL_VERY_SHORT: u64 = 300; + +/// Short TTL for frequently changing data (15 minutes) +/// Use for: Trending content, live feeds, dynamic lists +pub const CACHE_TTL_SHORT: u64 = 900; + +/// Medium TTL for regular data (1 hour) +/// Use for: User profiles, search results, API responses +pub const CACHE_TTL_MEDIUM: u64 = 3600; + +/// Long TTL for mostly static data (6 hours) +/// Use for: Configuration, reference data, translations +pub const CACHE_TTL_LONG: u64 = 21600; + +/// Very long TTL for static content (1 day) +/// Use for: Static pages, archived content, historical data +pub const CACHE_TTL_VERY_LONG: u64 = 86400; + +/// Image cache TTL (7 days) +/// Use for: Cached images, thumbnails, avatars +pub const CACHE_TTL_IMAGE: u64 = 604800; + +/// CDN cache TTL (30 days) +/// Use for: Immutable assets, versioned files +pub const CACHE_TTL_CDN: u64 = 2592000; + +/// Get TTL based on cache type +pub fn get_ttl_for(cache_type: CacheType) -> u64 { + match cache_type { + CacheType::RealTime => CACHE_TTL_VERY_SHORT, + CacheType::Volatile => CACHE_TTL_SHORT, + CacheType::Regular => CACHE_TTL_MEDIUM, + CacheType::Stable => CACHE_TTL_LONG, + CacheType::Static => CACHE_TTL_VERY_LONG, + CacheType::Image => CACHE_TTL_IMAGE, + CacheType::Cdn => CACHE_TTL_CDN, + } +} + +/// Cache type classification +#[derive(Debug, Clone, Copy)] +pub enum CacheType { + /// Real-time data (5 min) + RealTime, + /// Volatile data (15 min) + Volatile, + /// Regular data (1 hour) + Regular, + /// Stable data (6 hours) + Stable, + /// Static data (1 day) + Static, + /// Images (7 days) + Image, + /// CDN assets (30 days) + Cdn, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cache_ttl_values() { + assert_eq!(CACHE_TTL_VERY_SHORT, 300); + assert_eq!(CACHE_TTL_SHORT, 900); + assert_eq!(CACHE_TTL_MEDIUM, 3600); + assert_eq!(CACHE_TTL_LONG, 21600); + assert_eq!(CACHE_TTL_VERY_LONG, 86400); + assert_eq!(CACHE_TTL_IMAGE, 604800); + assert_eq!(CACHE_TTL_CDN, 2592000); + } + + #[test] + fn test_get_ttl_for() { + assert_eq!(get_ttl_for(CacheType::RealTime), 300); + assert_eq!(get_ttl_for(CacheType::Regular), 3600); + assert_eq!(get_ttl_for(CacheType::Image), 604800); + } +} diff --git a/src/shared/utils/io/file.rs b/src/shared/utils/io/file.rs new file mode 100644 index 0000000..89e58f1 --- /dev/null +++ b/src/shared/utils/io/file.rs @@ -0,0 +1,132 @@ +//! File utilities. + +use std::path::Path; +use tokio::fs; +use tokio::io::AsyncWriteExt; + +/// Read file contents as string. +pub async fn read_file(path: &str) -> anyhow::Result { + Ok(fs::read_to_string(path).await?) +} + +/// Read file contents as bytes. +pub async fn read_bytes(path: &str) -> anyhow::Result> { + Ok(fs::read(path).await?) +} + +/// Write string to file. +pub async fn write_file(path: &str, contents: &str) -> anyhow::Result<()> { + fs::write(path, contents).await?; + Ok(()) +} + +/// Write bytes to file. +pub async fn write_bytes(path: &str, contents: &[u8]) -> anyhow::Result<()> { + fs::write(path, contents).await?; + Ok(()) +} + +/// Append to file. +pub async fn append_file(path: &str, contents: &str) -> anyhow::Result<()> { + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .await?; + file.write_all(contents.as_bytes()).await?; + Ok(()) +} + +/// Check if file exists. +pub async fn file_exists(path: &str) -> bool { + fs::metadata(path).await.is_ok() +} + +/// Check if path is a directory. +pub async fn is_directory(path: &str) -> bool { + fs::metadata(path) + .await + .map(|m| m.is_dir()) + .unwrap_or(false) +} + +/// Create directory (including parents). +pub async fn create_dir(path: &str) -> anyhow::Result<()> { + fs::create_dir_all(path).await?; + Ok(()) +} + +/// Delete file. +pub async fn delete_file(path: &str) -> anyhow::Result<()> { + fs::remove_file(path).await?; + Ok(()) +} + +/// Delete directory recursively. +pub async fn delete_dir(path: &str) -> anyhow::Result<()> { + fs::remove_dir_all(path).await?; + Ok(()) +} + +/// Get file extension. +pub fn get_extension(path: &str) -> Option { + Path::new(path) + .extension() + .and_then(|ext| ext.to_str()) + .map(|s| s.to_lowercase()) +} + +/// Get filename without extension. +pub fn get_filename(path: &str) -> Option { + Path::new(path) + .file_stem() + .and_then(|name| name.to_str()) + .map(String::from) +} + +/// Get file size in bytes. +pub async fn file_size(path: &str) -> anyhow::Result { + let metadata = fs::metadata(path).await?; + Ok(metadata.len()) +} + +/// Format file size to human readable string. +pub fn format_file_size(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + + if bytes >= GB { + format!("{:.2} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.2} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.2} KB", bytes as f64 / KB as f64) + } else { + format!("{} bytes", bytes) + } +} + +/// Get MIME type from file extension. +pub fn mime_from_extension(ext: &str) -> &'static str { + match ext.to_lowercase().as_str() { + "html" | "htm" => "text/html", + "css" => "text/css", + "js" => "application/javascript", + "json" => "application/json", + "xml" => "application/xml", + "txt" => "text/plain", + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "svg" => "image/svg+xml", + "webp" => "image/webp", + "ico" => "image/x-icon", + "pdf" => "application/pdf", + "zip" => "application/zip", + "mp3" => "audio/mpeg", + "mp4" => "video/mp4", + "webm" => "video/webm", + _ => "application/octet-stream", + } +} diff --git a/src/shared/utils/io/mod.rs b/src/shared/utils/io/mod.rs new file mode 100644 index 0000000..d94727f --- /dev/null +++ b/src/shared/utils/io/mod.rs @@ -0,0 +1,6 @@ +pub mod cache; +pub mod cache_tags; +pub mod cache_ttl; +pub mod file; +pub mod retry; +pub mod soft_delete; diff --git a/src/shared/utils/io/retry.rs b/src/shared/utils/io/retry.rs new file mode 100644 index 0000000..6a40a1b --- /dev/null +++ b/src/shared/utils/io/retry.rs @@ -0,0 +1,66 @@ +//! HTTP retry utilities with exponential backoff. + +use backoff::ExponentialBackoff; +use std::time::Duration; + +/// Default retry configuration for HTTP requests. +pub fn default_backoff() -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_millis(500), + max_interval: Duration::from_secs(10), + multiplier: 2.0, + max_elapsed_time: Some(Duration::from_secs(30)), + ..Default::default() + } +} + +/// Create a custom exponential backoff. +pub fn custom_backoff( + initial_ms: u64, + max_secs: u64, + multiplier: f64, + max_elapsed_secs: u64, +) -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_millis(initial_ms), + max_interval: Duration::from_secs(max_secs), + multiplier, + max_elapsed_time: Some(Duration::from_secs(max_elapsed_secs)), + ..Default::default() + } +} + +/// Quick backoff for fast retries (3 attempts, 100ms initial). +pub fn quick_backoff() -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_millis(100), + max_interval: Duration::from_secs(1), + multiplier: 2.0, + max_elapsed_time: Some(Duration::from_secs(5)), + ..Default::default() + } +} + +/// Slow backoff for long operations (10 attempts, 1s initial). +pub fn slow_backoff() -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_secs(1), + max_interval: Duration::from_secs(30), + multiplier: 2.0, + max_elapsed_time: Some(Duration::from_secs(120)), + ..Default::default() + } +} + +/// Make an error transient (will be retried). +pub fn transient(err: E) -> backoff::Error { + backoff::Error::transient(err) +} + +/// Make an error permanent (will NOT be retried). +pub fn permanent(err: E) -> backoff::Error { + backoff::Error::permanent(err) +} + +// Re-export retry function for convenience +pub use backoff::future::retry; diff --git a/src/shared/utils/io/soft_delete.rs b/src/shared/utils/io/soft_delete.rs new file mode 100644 index 0000000..a930a42 --- /dev/null +++ b/src/shared/utils/io/soft_delete.rs @@ -0,0 +1,86 @@ +//! Soft delete helpers for SeaORM entities. +//! +//! Provides soft delete filtering for entities with a `deleted_at` column. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::soft_delete::{SoftDeletable, soft_delete_filter, SoftDeleteScope}; +//! use sea_orm::*; +//! +//! // Query without deleted +//! let users = User::find() +//! .filter(soft_delete_filter(user::Column::DeletedAt, SoftDeleteScope::WithoutDeleted)) +//! .all(db) +//! .await?; +//! +//! // Query only deleted +//! let deleted = User::find() +//! .filter(soft_delete_filter(user::Column::DeletedAt, SoftDeleteScope::OnlyDeleted)) +//! .all(db) +//! .await?; +//! ``` + +use chrono::{DateTime, Utc}; +use sea_orm::{ColumnTrait, Condition}; + +/// Trait for entities that support soft deletes. +pub trait SoftDeletable { + /// Get the deleted_at timestamp if soft deleted. + fn deleted_at(&self) -> Option>; + + /// Check if the entity is soft deleted. + fn is_deleted(&self) -> bool { + self.deleted_at().is_some() + } + + /// Check if the entity is not soft deleted. + fn is_active(&self) -> bool { + self.deleted_at().is_none() + } +} + +/// Query scope for filtering soft-deleted entities. +#[derive(Debug, Clone, Copy)] +pub enum SoftDeleteScope { + /// Exclude soft-deleted entities (default). + WithoutDeleted, + /// Include soft-deleted entities. + WithDeleted, + /// Only include soft-deleted entities. + OnlyDeleted, +} + +/// Create a condition for soft delete filtering. +/// +/// Use this in your entity queries: +/// ```ignore +/// User::find() +/// .filter(soft_delete_filter(user::Column::DeletedAt, SoftDeleteScope::WithoutDeleted)) +/// .all(db) +/// .await +/// ``` +pub fn soft_delete_filter(column: C, scope: SoftDeleteScope) -> Condition { + match scope { + SoftDeleteScope::WithoutDeleted => Condition::all().add(column.is_null()), + SoftDeleteScope::WithDeleted => Condition::all(), + SoftDeleteScope::OnlyDeleted => Condition::all().add(column.is_not_null()), + } +} + +/// Macro to implement SoftDeletable for a model. +/// +/// Usage: +/// ```ignore +/// impl_soft_deletable!(user::Model, deleted_at); +/// ``` +#[macro_export] +macro_rules! impl_soft_deletable { + ($model:ty, $deleted_at_field:ident) => { + impl $crate::shared::utils::soft_delete::SoftDeletable for $model { + fn deleted_at(&self) -> Option> { + self.$deleted_at_field + } + } + }; +} diff --git a/src/shared/utils/mod.rs b/src/shared/utils/mod.rs new file mode 100644 index 0000000..3c74736 --- /dev/null +++ b/src/shared/utils/mod.rs @@ -0,0 +1,190 @@ +//! Helper utilities for easier development. +//! +//! Comprehensive collection of updated utility modules for cleaner, more maintainable code. + +// Submodules +pub mod core; +pub mod data; +pub mod dev; +pub mod infra; +pub mod io; +pub mod web; + +// Re-exports for convenience (backward compatibility) + +// Core +pub use core::api_response; +pub use core::errors; +pub use core::handler; +pub use core::pagination; +pub use core::prelude; +pub use core::response; + +// Data +pub use data::collections; +pub use data::convert; +pub use data::datetime; +pub use data::json; +pub use data::numbers; +pub use data::string; +pub use data::text; + +// IO +pub use io::cache; +pub use io::cache_tags; +pub use io::cache_ttl; +pub use io::file; +pub use io::retry; +pub use io::soft_delete; + +// Web +pub use web::http; +pub use web::query; +pub use web::request; +pub use web::scraping; +pub use web::url; + +// Dev +pub use dev::async_utils; +pub use dev::logging; +pub use dev::performance; +pub use dev::result_ext; +pub use dev::serde_helpers; +pub use dev::testing; + +// Infra +pub use infra::bulk; +pub use infra::console; +pub use infra::encryption; +pub use infra::env; +pub use infra::form_request; +pub use infra::health_check; +pub use infra::import_export; +pub use infra::query_profiler; +pub use infra::resource; +pub use infra::searchable; +pub use infra::transaction; +pub use infra::uuid_utils; +pub use infra::versioning; + +// Ryzen CDN +pub use infra::ryzen_cdn::*; + +// ============================================================================ +// Original Re-exports (preserved) +// ============================================================================ + +// Prelude (common imports) +pub use prelude::*; + +// Response helpers +pub use pagination::*; +pub use response::*; + +// Error helpers +pub use errors::{ + bad_request, db_error, forbidden, internal_err, internal_error, not_found, redis_error, + unauthorized, HandlerError, ResultExt, +}; + +// Retry/Backoff +pub use retry::{ + custom_backoff, default_backoff, permanent, quick_backoff, retry, slow_backoff, transient, +}; + +// Caching +pub use cache::{cache_key, cache_key_multi, Cache, DEFAULT_CACHE_TTL}; + +// Scraping +pub use scraping::{ + attr_from, attr_from_or, extract_number, extract_slug, fetch_html_with_retry, parse_html, + select_attr, select_text, selector, strip_tags, text, text_from, text_from_or, Scraper, +}; + +// Strings +pub use string::{ + initials, is_valid_email, mask_email, random_code, random_string, slugify, title_case, truncate, +}; + +// DateTime +pub use datetime::{ + add_days, add_hours, is_future, is_past, now, parse_iso, relative, timestamp, to_human, to_iso, +}; + +// Crypto + +// Files +pub use file::{ + create_dir, file_exists, format_file_size, get_extension, mime_from_extension, read_file, + write_file, +}; + +// JSON +pub use json::{ + deep_merge, get_i64, get_path, get_str, is_empty, merge, parse, remove_nulls, stringify, + stringify_pretty, +}; + +// URL +pub use url::{ + decode, encode, extract_domain, is_absolute, join_paths, make_absolute, parse_query, UrlBuilder, +}; + +// Logging +pub use logging::{log_and_map, log_error, log_request, PerfLogger, TimedOperation}; + +// Collections +pub use collections::{ + all, any, chunk, count, find, find_index, flatten, frequencies, group_by, partition, reverse, + skip, sum, take, unique, zip, +}; + +// Numbers +pub use numbers::{ + clamp, format_bytes, format_currency, format_number, format_percent, is_even, is_odd, lerp, + parse_f64, parse_i64, percentage, round_to, safe_div, +}; + +// Async +pub use async_utils::{ + join_all, join_all_limited, simple_retry, sleep, sleep_ms, sleep_secs, spawn, spawn_blocking, + timeout_ms, timeout_secs, with_timeout, Debouncer, +}; + +// Serde helpers +pub use serde_helpers::{ + default_empty_string, default_empty_vec, default_false, default_true, default_zero, +}; + +// Text processing +pub use text::{ + capitalize, highlight, lorem_ipsum, normalize_whitespace, remove_accents, to_camel_case, + to_constant_case, to_kebab_case, to_pascal_case, to_snake_case, truncate_words, word_count, +}; + +// Conversions +pub use convert::{ + bool_to_str, bytes_to_hex, empty_to_none, hex_to_bytes, i64_to_usize, ms_to_human, + none_to_empty, parse_or, seconds_to_human, to_bool, try_parse, +}; + +// Result/Option extensions +pub use result_ext::{err, flatten_option, flatten_result, ok, some, OptionExt, ResultExt2}; + +// HTTP Request helpers +pub use request::{ + accepts_gzip, bearer_token, client_ip, content_type, header_value, is_form, is_json, origin, + referer, request_id, user_agent, +}; + +// Environment +pub use env::{ + database_url, get_or as env_get_or, host, is_debug, is_development, is_production, load_dotenv, + port, redis_url, require as env_require, +}; + +// UUID +pub use uuid_utils::{ + is_valid as is_valid_uuid_format, medium_id, new_v4 as uuid_v4, new_v4_simple as uuid_simple, + short_id, +}; diff --git a/src/shared/utils/web/http.rs b/src/shared/utils/web/http.rs new file mode 100644 index 0000000..7a66b34 --- /dev/null +++ b/src/shared/utils/web/http.rs @@ -0,0 +1,26 @@ +use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT}; + +pub fn common_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")); + headers.insert("Referer", HeaderValue::from_static("https://google.com")); + headers +} + +pub fn common_image_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")); + headers.insert( + "Accept", + HeaderValue::from_static( + "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + ), + ); + headers +} + +pub fn is_internet_baik_block_page(content: &str) -> bool { + content.contains("Internet Baik") + || content.contains("TrustPositif") + || content.contains("Mercusuar") +} diff --git a/src/shared/utils/web/http_client.rs b/src/shared/utils/web/http_client.rs new file mode 100644 index 0000000..ac460d7 --- /dev/null +++ b/src/shared/utils/web/http_client.rs @@ -0,0 +1,135 @@ +//! HTTP client wrapper with common configurations. + +use reqwest::{Client, ClientBuilder, Response}; +use std::time::Duration; +use tracing::debug; + +/// Pre-configured HTTP client with sensible defaults. +#[derive(Clone)] +pub struct HttpClient { + inner: Client, +} + +impl HttpClient { + /// Create a new HTTP client with default settings. + pub fn new() -> Result { + let client = ClientBuilder::new() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .pool_max_idle_per_host(20) + .pool_idle_timeout(Duration::from_secs(60)) + .tcp_nodelay(true) + .user_agent("Scraper/1.0") + .build() + .map_err(|e| format!("Failed to build HTTP client: {}", e))?; + + Ok(Self { inner: client }) + } + + /// Create with custom timeout. + pub fn with_timeout(timeout_secs: u64) -> Result { + let client = ClientBuilder::new() + .timeout(Duration::from_secs(timeout_secs)) + .connect_timeout(Duration::from_secs(10)) + .user_agent("Scraper/1.0") + .build() + .map_err(|e| format!("Failed to build HTTP client: {}", e))?; + + Ok(Self { inner: client }) + } + + /// GET request. + pub async fn get(&self, url: &str) -> reqwest::Result { + debug!("GET {}", url); + self.inner.get(url).send().await + } + + /// GET request and return text. + pub async fn get_text(&self, url: &str) -> reqwest::Result { + self.get(url).await?.text().await + } + + /// GET request and parse JSON. + pub async fn get_json(&self, url: &str) -> reqwest::Result { + self.get(url).await?.json().await + } + + /// POST request with JSON body. + pub async fn post_json( + &self, + url: &str, + body: &T, + ) -> reqwest::Result { + debug!("POST {}", url); + self.inner.post(url).json(body).send().await + } + + /// PUT request with JSON body. + pub async fn put_json( + &self, + url: &str, + body: &T, + ) -> reqwest::Result { + debug!("PUT {}", url); + self.inner.put(url).json(body).send().await + } + + /// DELETE request. + pub async fn delete(&self, url: &str) -> reqwest::Result { + debug!("DELETE {}", url); + self.inner.delete(url).send().await + } + + /// Get the underlying reqwest client. + pub fn client(&self) -> &Client { + &self.inner + } +} + +impl Default for HttpClient { + fn default() -> Self { + Self::new().expect("Valid HTTP client configuration") + } +} + +use once_cell::sync::Lazy; +use std::sync::Arc; + +static HTTP_CLIENT_INIT: Lazy, String>> = Lazy::new(|| { + HttpClient::new() + .map(|c| Arc::new(c)) + .map_err(|e| format!("Failed to initialize HTTP client: {}", e)) +}); + +static HTTP_CLIENT_FAST_INIT: Lazy, String>> = Lazy::new(|| { + HttpClient::with_timeout(10) + .map(|c| Arc::new(c)) + .map_err(|e| format!("Failed to initialize fast HTTP client: {}", e)) +}); + +static HTTP_CLIENT_SLOW_INIT: Lazy, String>> = Lazy::new(|| { + HttpClient::with_timeout(60) + .map(|c| Arc::new(c)) + .map_err(|e| format!("Failed to initialize slow HTTP client: {}", e)) +}); + +/// Global HTTP client instance (30s timeout - general purpose). +pub fn http_client() -> &'static HttpClient { + HTTP_CLIENT_INIT + .as_ref() + .expect("HTTP client initialization failed") +} + +/// Get the fast HTTP client (10s timeout). +pub fn http_client_fast() -> &'static HttpClient { + HTTP_CLIENT_FAST_INIT + .as_ref() + .expect("Fast HTTP client initialization failed") +} + +/// Get the slow HTTP client (60s timeout). +pub fn http_client_slow() -> &'static HttpClient { + HTTP_CLIENT_SLOW_INIT + .as_ref() + .expect("Slow HTTP client initialization failed") +} diff --git a/src/shared/utils/web/mod.rs b/src/shared/utils/web/mod.rs new file mode 100644 index 0000000..0aecb1d --- /dev/null +++ b/src/shared/utils/web/mod.rs @@ -0,0 +1,8 @@ +pub mod http; +pub mod http_client; +pub mod proxy_fetch; +pub mod query; +pub mod request; +pub mod scraping; +pub mod scraping_urls; +pub mod url; diff --git a/src/shared/utils/web/proxy_fetch.rs b/src/shared/utils/web/proxy_fetch.rs new file mode 100644 index 0000000..b656fe6 --- /dev/null +++ b/src/shared/utils/web/proxy_fetch.rs @@ -0,0 +1,399 @@ +// Proxy fetch logic with Redis cache AND Request Coalescing (SingleFlight) +// Updated for sync Redis API, reqwest API changes, and concurrency optimization. + +use dashmap::DashMap; +use once_cell::sync::Lazy; +use redis::AsyncCommands; +use tokio::sync::broadcast; +use tracing::{debug, error, warn}; + +use crate::shared::database::get_redis_conn; +use crate::shared::errors::AppError; +use crate::shared::utils::cache_ttl::CACHE_TTL_VERY_SHORT; +use crate::shared::utils::http::common_headers; +use crate::shared::utils::http::is_internet_baik_block_page; +use crate::shared::utils::web::http_client::http_client; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct FetchResult { + pub data: String, + pub content_type: Option, +} + +// Implement Display to allow .to_string() +impl std::fmt::Display for FetchResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "FetchResult {{ data: {}, content_type: {} }}", + self.data, + self.content_type.as_deref().unwrap_or("None") + ) + } +} + +// Global In-Flight Request Map for Request Coalescing +// Maps URL slug -> Broadcast Sender +static IN_FLIGHT: Lazy>>> = + Lazy::new(DashMap::new); + +// Global Blacklist for domains that consistently fail direct fetch (Timeouts, SSL, Cloudflare blocks) +static FAILED_DOMAINS: Lazy> = Lazy::new(dashmap::DashSet::new); + +const RELAY_ENDPOINTS: &[&str] = &[ + "https://opennext-app.superaseph.workers.dev", + "https://proxy-bun.vercel.app", + "https://proxy-bun-mytheclipse8647-orfq73fe.apn.leapcell.dev", +]; + +// --- REDIS CACHE WRAPPER START --- +fn get_fetch_cache_key(slug: &str) -> String { + format!("fetch:proxy:{slug}") +} + +async fn get_cached_fetch(slug: &str) -> Result, AppError> { + let mut conn = get_redis_conn().await?; + let key = get_fetch_cache_key(slug); + + let cached: Option = conn.get(&key).await?; + + if let Some(cached_str) = cached { + match serde_json::from_str::(&cached_str) { + Ok(parsed) => { + debug!("[fetchWithProxy] Returning cached response for {}", slug); + Ok(Some(parsed)) + } + Err(_) => Ok(None), + } + } else { + Ok(None) + } +} + +async fn set_cached_fetch(slug: &str, value: &FetchResult) -> Result<(), AppError> { + let mut conn = get_redis_conn().await?; + let key = get_fetch_cache_key(slug); + let json_string = serde_json::to_string(value)?; + + // Use standardized TTL + conn.set_ex::<_, _, ()>(&key, &json_string, CACHE_TTL_VERY_SHORT) + .await?; + Ok(()) +} +// --- REDIS CACHE WRAPPER END --- + +/// Main entry point: Fetches with proxy, using Cache and Request Coalescing +pub async fn fetch_with_proxy(slug: &str) -> Result { + // 1. Try Cache First + if let Ok(Some(cached)) = get_cached_fetch(slug).await { + return Ok(cached); + } + + // 2. Request Coalescing (SingleFlight) + // Check if there is already an in-flight request for this slug + let tx = { + if let Some(in_flight) = IN_FLIGHT.get(slug) { + debug!("[Coalesce] Joining in-flight request for {}", slug); + in_flight.value().clone() + } else { + // No in-flight request, create a new channel + let (tx, _) = broadcast::channel(1); // Capacity 1 is enough for single result + IN_FLIGHT.insert(slug.to_string(), tx.clone()); + debug!("[Coalesce] Starting leader request for {}", slug); + + // We are the leader, we must execute the fetch + // Spawn the fetch task so we don't block holding the map lock (though insert is fast) + // But actually we are not holding the lock here anymore. + + // Clone for the async block + let slug_clone = slug.to_string(); + let tx_clone = tx.clone(); + + tokio::spawn(async move { + // RAII Guard: Guarantee slug eviction exactly once the task finishes or panics! + struct DropGuard(String); + impl Drop for DropGuard { + fn drop(&mut self) { + IN_FLIGHT.remove(&self.0); + } + } + let _guard = DropGuard(slug_clone.clone()); + + let result = perform_fetch(&slug_clone).await; + + // Map AppError to String for broadcast (since AppError might not be Clone) + // FetchResult is Clone. + let broadcast_result = match &result { + Ok(res) => Ok(res.clone()), + Err(e) => Err(e.to_string()), + }; + + // Broadcast result to all waiting subscribers + let _ = tx_clone.send(broadcast_result); + }); + + tx + } + }; + + // 3. Wait for result (Leader or Follower) + let mut rx = tx.subscribe(); + match rx.recv().await { + Ok(Ok(res)) => Ok(res), + Ok(Err(e_str)) => Err(AppError::Other(e_str)), + Err(e) => { + warn!("[Coalesce] Receive mismatch for {}: {:?}", slug, e); + Err(AppError::Other("Request coalescing error".to_string())) + } + } +} + +/// The actual fetch logic (Direct -> Retry -> Proxy) +async fn perform_fetch(slug: &str) -> Result { + // 1. Extract domain for Circuit Breaker logic + let domain = reqwest::Url::parse(slug) + .ok() + .and_then(|u| u.host_str().map(|s| s.to_string())) + .unwrap_or_default(); + + // 2. Immediate proxy fallback if domain has a known history of brutal timeouts/blocks + if !domain.is_empty() && FAILED_DOMAINS.contains(&domain) { + warn!( + "[Circuit Breaker] Domain {} is blacklisted from direct-fetch. Routing via Relays.", + domain + ); + return perform_proxy_chain(slug).await; + } + + // Use shared global HTTP client + let client = http_client().client(); + let headers = common_headers(); + + match client + .get(slug) + .headers(headers) + .send() // Timeout handled by client + .await + { + Ok(res) => { + debug!( + "[fetchWithProxy] Direct fetch response: url={}, status={}", + slug, + res.status() + ); + if res.status().is_success() { + let content_type = res + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .map(|s| s.to_string()); + + let bytes = res.bytes().await?; + + // Check if response is Gzip compressed (magic header 1f 8b) + let text_data = if bytes.len() > 2 && bytes[0] == 0x1f && bytes[1] == 0x8b { + // Gzip compressed, offload decompression to blocking thread + let decompressed = tokio::task::spawn_blocking(move || { + use flate2::read::GzDecoder; + use std::io::Read; + let decoder = GzDecoder::new(&bytes[..]); + let mut decompressed = Vec::new(); + // 10MB absolute decompression bounds to prevent GZIP Bombs (OOM vulnerability) + decoder + .take(10_000_000) + .read_to_end(&mut decompressed) + .map(|_| decompressed) + .map_err(|e| { + AppError::Other(format!( + "Decompression failed or exceeded limits: {:?}", + e + )) + }) + }) + .await??; + + match std::str::from_utf8(&decompressed) { + Ok(s) => s.to_string(), + Err(_) => String::from_utf8_lossy(&decompressed).to_string(), + } + } else { + match std::str::from_utf8(&bytes) { + Ok(s) => s.to_string(), + Err(_) => { + warn!("Response bytes are not valid UTF-8, using lossy conversion"); + String::from_utf8_lossy(&bytes).to_string() + } + } + }; + + if is_internet_baik_block_page(&text_data) { + warn!("Blocked by internetbaik (direct fetch) for {}", slug); + return perform_proxy_chain(slug).await; + } else { + let result = FetchResult { + data: text_data, + content_type, + }; + // Cache the success result + if let Err(e) = set_cached_fetch(slug, &result).await { + warn!("Failed to cache result for {}: {:?}", slug, e); + } + Ok(result) + } + } else { + let error_msg = format!( + "Direct fetch failed with status {} for {}", + res.status(), + slug + ); + + // Penalize domain if it throws aggressive Anti-Bot or Gateway Timeout codes + if res.status().is_server_error() + || res.status() == reqwest::StatusCode::FORBIDDEN + || res.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE + { + if !domain.is_empty() { + warn!("[Circuit Breaker] Blacklisting domain {} due to hostile HTTP status {}", domain, res.status()); + FAILED_DOMAINS.insert(domain.clone()); + } + error!("{}", error_msg); + return perform_proxy_chain(slug).await; + } else { + warn!("{}", error_msg); + } + Err(AppError::Other(error_msg)) + } + } + Err(e) => { + let error_msg = format!("Direct fetch failed for {}: {:?}", slug, e); + warn!("{}", error_msg); + + // Hard panic mapping: If reqwest core network/TLS fails, instantly blacklist the domain + if !domain.is_empty() { + warn!("[Circuit Breaker] Blacklisting domain {} due to core network/SSL trace failure", domain); + FAILED_DOMAINS.insert(domain.clone()); + } + + // Fall back seamlessly to Relay network proxy instead of throwing fatal transient backoff + perform_proxy_chain(slug).await + } + } +} + +pub async fn fetch_with_proxy_only(slug: &str) -> Result { + if let Ok(Some(cached)) = get_cached_fetch(slug).await { + return Ok(cached); + } + + perform_proxy_chain(slug).await +} + +async fn perform_proxy_chain(slug: &str) -> Result { + // 1. Try Relays + match fetch_via_relays(slug).await { + Ok(res) => return Ok(res), + Err(e) => warn!("[ProxyChain] All relays failed for {}: {:?}", slug, e), + } + + // 2. Try Browserless as absolute last resort + match fetch_via_browserless(slug).await { + Ok(res) => Ok(res), + Err(e) => { + error!( + "[ProxyChain] Browserless fallback failed for {}: {:?}", + slug, e + ); + Err(e) + } + } +} + +async fn fetch_via_relays(slug: &str) -> Result { + // Use shared client + let client = http_client().client(); + + for relay in RELAY_ENDPOINTS { + debug!( + "[fetch_via_relays] Attempting to fetch {} via relay {}", + slug, relay + ); + + match client + .get(*relay) + .header("x-relay-target", slug) + .send() + .await + { + Ok(res) if res.status().is_success() => { + let content_type = res + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .map(|s| s.to_string()); + let data = res.text().await?; + + let result = FetchResult { data, content_type }; + debug!( + "[fetch_via_relays] Successfully fetched {} via relay {}", + slug, relay + ); + + if let Err(e) = set_cached_fetch(slug, &result).await { + warn!("Failed to cache relay result for {}: {:?}", slug, e); + } + + return Ok(result); + } + Ok(res) => { + warn!( + "[fetch_via_relays] Relay {} returned status {} for {}", + relay, + res.status(), + slug + ); + } + Err(e) => { + warn!( + "[fetch_via_relays] Relay {} failed for {}: {:?}", + relay, slug, e + ); + } + } + } + + Err(AppError::Other("All relay endpoints failed".to_string())) +} + +async fn fetch_via_browserless(slug: &str) -> Result { + use crate::shared::browser::pool::get_browser_pool; + + warn!("[Browserless] Falling back to remote browser for {}", slug); + + let pool = get_browser_pool() + .ok_or_else(|| AppError::Other("Browser pool not initialized".to_string()))?; + + let tab = pool + .get_tab() + .await + .map_err(|e| AppError::Other(format!("Failed to get browser tab: {:?}", e)))?; + + tab.goto(slug) + .await + .map_err(|e| AppError::Other(format!("Browser navigation failed for {}: {:?}", slug, e)))?; + + let data = tab + .content() + .await + .map_err(|e| AppError::Other(format!("Failed to get browser content: {:?}", e)))?; + + let result = FetchResult { + data, + content_type: Some("text/html".to_string()), + }; + + if let Err(e) = set_cached_fetch(slug, &result).await { + warn!("Failed to cache browserless result for {}: {:?}", slug, e); + } + + Ok(result) +} diff --git a/src/shared/utils/web/query.rs b/src/shared/utils/web/query.rs new file mode 100644 index 0000000..b4a33c4 --- /dev/null +++ b/src/shared/utils/web/query.rs @@ -0,0 +1,278 @@ +//! Query builder helpers for SeaORM. +//! +//! Provides utilities for pagination, filtering, and sorting queries. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::query::{QueryBuilder, Pagination, SortOrder}; +//! +//! let query = User::find() +//! .apply_pagination(Pagination::new(1, 20)) +//! .apply_sort("created_at", SortOrder::Desc); +//! ``` + +use sea_orm::{entity::prelude::*, Order, QuerySelect, Select}; +use serde::{Deserialize, Serialize}; + +/// Pagination parameters. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Pagination { + /// Current page (1-indexed). + pub page: u64, + /// Items per page. + pub per_page: u64, + /// Total items (optional, set after query). + #[serde(skip_deserializing)] + pub total: Option, +} + +impl Pagination { + /// Create new pagination. + pub fn new(page: u64, per_page: u64) -> Self { + Self { + page: page.max(1), + per_page: per_page.clamp(1, 100), + total: None, + } + } + + /// Default pagination (page 1, 20 per page). + pub fn default_page() -> Self { + Self::new(1, 20) + } + + /// Calculate offset for query. + pub fn offset(&self) -> u64 { + (self.page - 1) * self.per_page + } + + /// Calculate total pages. + pub fn total_pages(&self) -> u64 { + match self.total { + Some(total) => (total as f64 / self.per_page as f64).ceil() as u64, + None => 0, + } + } + + /// Check if there's a next page. + pub fn has_next(&self) -> bool { + self.page < self.total_pages() + } + + /// Check if there's a previous page. + pub fn has_prev(&self) -> bool { + self.page > 1 + } + + /// Set total after counting. + pub fn with_total(mut self, total: u64) -> Self { + self.total = Some(total); + self + } +} + +impl Default for Pagination { + fn default() -> Self { + Self::default_page() + } +} + +/// Sort order. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum SortOrder { + Asc, + Desc, +} + +impl From for Order { + fn from(order: SortOrder) -> Self { + match order { + SortOrder::Asc => Order::Asc, + SortOrder::Desc => Order::Desc, + } + } +} + +/// Sort parameter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Sort { + pub field: String, + pub order: SortOrder, +} + +impl Sort { + pub fn new(field: &str, order: SortOrder) -> Self { + Self { + field: field.to_string(), + order, + } + } + + pub fn asc(field: &str) -> Self { + Self::new(field, SortOrder::Asc) + } + + pub fn desc(field: &str) -> Self { + Self::new(field, SortOrder::Desc) + } +} + +/// Filter operator. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FilterOp { + Eq, + Ne, + Gt, + Gte, + Lt, + Lte, + Like, + In, + IsNull, + IsNotNull, +} + +/// Filter parameter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Filter { + pub field: String, + pub op: FilterOp, + pub value: serde_json::Value, +} + +impl Filter { + pub fn eq(field: &str, value: impl Into) -> Self { + Self { + field: field.to_string(), + op: FilterOp::Eq, + value: value.into(), + } + } + + pub fn ne(field: &str, value: impl Into) -> Self { + Self { + field: field.to_string(), + op: FilterOp::Ne, + value: value.into(), + } + } + + pub fn like(field: &str, value: &str) -> Self { + Self { + field: field.to_string(), + op: FilterOp::Like, + value: serde_json::Value::String(value.to_string()), + } + } + + pub fn is_null(field: &str) -> Self { + Self { + field: field.to_string(), + op: FilterOp::IsNull, + value: serde_json::Value::Null, + } + } + + pub fn is_not_null(field: &str) -> Self { + Self { + field: field.to_string(), + op: FilterOp::IsNotNull, + value: serde_json::Value::Null, + } + } +} + +/// Query parameters combining pagination, sort, and filters. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct QueryParams { + #[serde(default)] + pub pagination: Pagination, + #[serde(default)] + pub sort: Option, + #[serde(default)] + pub filters: Vec, + #[serde(default)] + pub search: Option, +} + +impl QueryParams { + pub fn new() -> Self { + Self::default() + } + + pub fn page(mut self, page: u64) -> Self { + self.pagination.page = page.max(1); + self + } + + pub fn per_page(mut self, per_page: u64) -> Self { + self.pagination.per_page = per_page.clamp(1, 100); + self + } + + pub fn sort_by(mut self, field: &str, order: SortOrder) -> Self { + self.sort = Some(Sort::new(field, order)); + self + } + + pub fn filter(mut self, filter: Filter) -> Self { + self.filters.push(filter); + self + } + + pub fn search(mut self, query: &str) -> Self { + self.search = Some(query.to_string()); + self + } +} + +/// Paginated result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PaginatedResult { + pub data: Vec, + pub pagination: PaginationMeta, +} + +/// Pagination metadata for response. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PaginationMeta { + pub page: u64, + pub per_page: u64, + pub total: u64, + pub total_pages: u64, + pub has_next: bool, + pub has_prev: bool, +} + +impl PaginatedResult { + pub fn new(data: Vec, pagination: Pagination) -> Self { + let total = pagination.total.unwrap_or(0); + let total_pages = pagination.total_pages(); + + Self { + data, + pagination: PaginationMeta { + page: pagination.page, + per_page: pagination.per_page, + total, + total_pages, + has_next: pagination.has_next(), + has_prev: pagination.has_prev(), + }, + } + } +} + +/// Extension trait for applying pagination to queries. +pub trait PaginateExt { + fn apply_pagination(self, pagination: &Pagination) -> Self; +} + +impl PaginateExt for Select { + fn apply_pagination(self, pagination: &Pagination) -> Self { + self.offset(pagination.offset()).limit(pagination.per_page) + } +} diff --git a/src/shared/utils/web/request.rs b/src/shared/utils/web/request.rs new file mode 100644 index 0000000..4b68c89 --- /dev/null +++ b/src/shared/utils/web/request.rs @@ -0,0 +1,141 @@ +//! HTTP request helpers. + +use axum::http::{HeaderMap, HeaderValue}; + +/// Extract client IP from headers (X-Forwarded-For, X-Real-IP). +pub fn client_ip(headers: &HeaderMap) -> Option { + // Try X-Forwarded-For first + if let Some(forwarded) = headers.get("x-forwarded-for") { + if let Ok(value) = forwarded.to_str() { + if let Some(ip) = value.split(',').next() { + return Some(ip.trim().to_string()); + } + } + } + + // Try X-Real-IP + if let Some(real_ip) = headers.get("x-real-ip") { + if let Ok(value) = real_ip.to_str() { + return Some(value.to_string()); + } + } + + // Try CF-Connecting-IP (Cloudflare) + if let Some(cf_ip) = headers.get("cf-connecting-ip") { + if let Ok(value) = cf_ip.to_str() { + return Some(value.to_string()); + } + } + + None +} + +/// Extract User-Agent from headers. +pub fn user_agent(headers: &HeaderMap) -> Option { + headers + .get("user-agent") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract Accept-Language from headers. +pub fn accept_language(headers: &HeaderMap) -> Option { + headers + .get("accept-language") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract Authorization bearer token. +pub fn bearer_token(headers: &HeaderMap) -> Option { + headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .map(String::from) +} + +/// Extract content type. +pub fn content_type(headers: &HeaderMap) -> Option { + headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Check if request is JSON. +pub fn is_json(headers: &HeaderMap) -> bool { + content_type(headers) + .map(|ct| ct.contains("application/json")) + .unwrap_or(false) +} + +/// Check if request is form data. +pub fn is_form(headers: &HeaderMap) -> bool { + content_type(headers) + .map(|ct| { + ct.contains("application/x-www-form-urlencoded") || ct.contains("multipart/form-data") + }) + .unwrap_or(false) +} + +/// Extract referer. +pub fn referer(headers: &HeaderMap) -> Option { + headers + .get("referer") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract origin. +pub fn origin(headers: &HeaderMap) -> Option { + headers + .get("origin") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract request ID. +pub fn request_id(headers: &HeaderMap) -> Option { + headers + .get("x-request-id") + .or_else(|| headers.get("x-correlation-id")) + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Check if request accepts gzip. +pub fn accepts_gzip(headers: &HeaderMap) -> bool { + headers + .get("accept-encoding") + .and_then(|v| v.to_str().ok()) + .map(|v| v.contains("gzip")) + .unwrap_or(false) +} + +/// Create header value. +pub fn header_value(s: &str) -> HeaderValue { + HeaderValue::from_str(s).unwrap_or_else(|_| HeaderValue::from_static("")) +} + +/// Parse quality value from Accept header (e.g., "text/html;q=0.9"). +pub fn parse_accept_quality(accept: &str) -> Vec<(String, f32)> { + accept + .split(',') + .filter_map(|part| { + let mut parts = part.trim().split(';'); + let mime = parts.next()?.trim().to_string(); + let quality = parts + .find_map(|p| { + let p = p.trim(); + if p.starts_with("q=") { + p[2..].parse().ok() + } else { + None + } + }) + .unwrap_or(1.0); + Some((mime, quality)) + }) + .collect() +} diff --git a/src/shared/utils/web/scraping.rs b/src/shared/utils/web/scraping.rs new file mode 100644 index 0000000..ac6a47a --- /dev/null +++ b/src/shared/utils/web/scraping.rs @@ -0,0 +1,189 @@ +//! HTML scraping helpers using scraper crate. + +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::fetch_with_proxy; +use crate::shared::utils::{default_backoff, transient}; +use backoff::future::retry; +use once_cell::sync::Lazy; +use regex::Regex; +use scraper::{ElementRef, Html, Selector}; +use tracing::{info, warn}; + +/// Fetch HTML from URL with retry backoff and proxy support. +pub async fn fetch_html_with_retry(url: &str) -> Result { + let backoff = default_backoff(); + let fetch_operation = || async { + info!("Fetching: {}", url); + match fetch_with_proxy(url).await { + Ok(response) => { + info!("Successfully fetched: {}", url); + Ok(response.data) + } + Err(e) => { + warn!("Failed to fetch: {}, error: {:?}", url, e); + Err(transient(e)) + } + } + }; + + retry(backoff, fetch_operation) + .await + .map_err(|e| AppError::ScraperError(e.to_string())) +} + +/// Parse HTML string into a document. +pub fn parse_html(html: &str) -> Html { + Html::parse_document(html) +} + +/// Safely create a CSS selector. +pub fn selector(css: &str) -> Option { + Selector::parse(css).ok() +} + +/// Extract text content from an element, trimmed. +pub fn text(element: &ElementRef) -> String { + element.text().collect::().trim().to_string() +} + +/// Extract text from first matching element. +pub fn select_text(element: &ElementRef, css: &str) -> Option { + let sel = selector(css)?; + element.select(&sel).next().map(|e| text(&e)) +} + +/// Extract text from first matching element using a pre-compiled selector. +pub fn text_from(element: &ElementRef, selector: &Selector) -> Option { + element.select(selector).next().map(|e| text(&e)) +} + +/// Extract text from first matching element using a pre-compiled selector, or return default. +pub fn text_from_or(element: &ElementRef, selector: &Selector, default: &str) -> String { + text_from(element, selector).unwrap_or_else(|| default.to_string()) +} + +/// Extract attribute from first matching element. +pub fn select_attr(element: &ElementRef, css: &str, attr: &str) -> Option { + let sel = selector(css)?; + element + .select(&sel) + .next() + .and_then(|e| e.value().attr(attr)) + .map(String::from) +} + +/// Extract attribute from first matching element using a pre-compiled selector. +pub fn attr_from(element: &ElementRef, selector: &Selector, attr: &str) -> Option { + element + .select(selector) + .next() + .and_then(|e| e.value().attr(attr)) + .map(String::from) +} + +/// Extract attribute from first matching element using a pre-compiled selector, or return default. +pub fn attr_from_or( + element: &ElementRef, + selector: &Selector, + attr: &str, + default: &str, +) -> String { + attr_from(element, selector, attr).unwrap_or_else(|| default.to_string()) +} + +/// Extract attribute from element. +pub fn attr(element: &ElementRef, name: &str) -> Option { + element.value().attr(name).map(String::from) +} + +/// Select all matching elements. +pub fn select_all<'a>(document: &'a Html, css: &str) -> Vec> { + selector(css) + .map(|s| document.select(&s).collect()) + .unwrap_or_default() +} + +/// Extract slug from URL (last path segment). +pub fn extract_slug(url: &str) -> String { + static SLUG_REGEX: Lazy> = Lazy::new(|| Regex::new(r"/([^/]+)/?$")); + + SLUG_REGEX + .as_ref() + .ok() + .and_then(|r| r.captures(url)) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str().to_string()) + .unwrap_or_default() +} + +/// Remove HTML tags from string. +pub fn strip_tags(html: &str) -> String { + static TAG_REGEX: Lazy> = Lazy::new(|| Regex::new(r"<[^>]+>")); + TAG_REGEX + .as_ref() + .map(|r| r.replace_all(html, "").trim().to_string()) + .unwrap_or_else(|_| html.to_string()) +} + +/// Extract number from text. +pub fn extract_number(text: &str) -> Option { + static NUM_REGEX: Lazy> = Lazy::new(|| Regex::new(r"\d+")); + NUM_REGEX + .as_ref() + .ok() + .and_then(|r| r.find(text)) + .and_then(|m| m.as_str().parse().ok()) +} + +/// Extract text inside parentheses. +pub fn extract_parentheses(text: &str) -> Option { + static PAREN_REGEX: Lazy> = + Lazy::new(|| Regex::new(r"\(([^)]+)\)")); + PAREN_REGEX + .as_ref() + .ok() + .and_then(|r| r.captures(text)) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str().to_string()) +} + +/// Builder for scraping elements. +pub struct Scraper<'a> { + element: ElementRef<'a>, +} + +impl<'a> Scraper<'a> { + pub fn new(element: ElementRef<'a>) -> Self { + Self { element } + } + + /// Get text from selector. + pub fn text(&self, css: &str) -> Option { + select_text(&self.element, css) + } + + /// Get text or default. + pub fn text_or(&self, css: &str, default: &str) -> String { + self.text(css).unwrap_or_else(|| default.to_string()) + } + + /// Get attribute from selector. + pub fn attr(&self, css: &str, name: &str) -> Option { + select_attr(&self.element, css, name) + } + + /// Get attribute or default. + pub fn attr_or(&self, css: &str, name: &str, default: &str) -> String { + self.attr(css, name).unwrap_or_else(|| default.to_string()) + } + + /// Get href from first link. + pub fn href(&self, css: &str) -> Option { + self.attr(css, "href") + } + + /// Get src from first image. + pub fn src(&self, css: &str) -> Option { + self.attr(css, "src") + } +} diff --git a/src/shared/utils/web/scraping_urls.rs b/src/shared/utils/web/scraping_urls.rs new file mode 100644 index 0000000..4571ef9 --- /dev/null +++ b/src/shared/utils/web/scraping_urls.rs @@ -0,0 +1,30 @@ +//! URL constants and dynamic environment-based URLs. +//! +//! Note: These URLs are kept as dynamic env lookups because they may vary +//! between deployments and are not critical startup dependencies. + +use crate::shared::config::CONFIG; +use std::env; + +pub const BASE_URL: &str = "http://127.0.0.1:4090"; +pub const OTAKUDESU_BASE_URL: &str = "https://otakudesu.blog/"; + +/// Get Komik URL from environment config. +pub fn get_komik_url() -> String { + env::var("KOMIK2_BASE_URL").unwrap_or_else(|_| "https://komiku.org".to_string()) +} + +/// Get production URL from environment config. +pub fn get_production_url() -> String { + env::var("NEXT_PUBLIC_PROD").unwrap_or_else(|_| CONFIG.urls.site_url.clone()) +} + +/// Get Komik API URL from environment config. +pub fn get_komik_api_url() -> String { + env::var("KOMIK2_API_URL").unwrap_or_else(|_| "https://api.komiku.org".to_string()) +} + +/// Get Otakudesu URL from environment config. +pub fn get_otakudesu_url() -> String { + env::var("OTAKUDESU_BASE_URL").unwrap_or_else(|_| OTAKUDESU_BASE_URL.to_string()) +} diff --git a/src/shared/utils/web/url.rs b/src/shared/utils/web/url.rs new file mode 100644 index 0000000..cce7c86 --- /dev/null +++ b/src/shared/utils/web/url.rs @@ -0,0 +1,159 @@ +//! URL building and manipulation utilities. + +use std::collections::HashMap; + +/// URL builder for constructing URLs with query parameters. +#[derive(Debug, Clone)] +pub struct UrlBuilder { + base: String, + path_segments: Vec, + query_params: Vec<(String, String)>, +} + +impl UrlBuilder { + /// Create a new URL builder with base URL. + pub fn new(base: impl Into) -> Self { + let base = base.into(); + let base = base.trim_end_matches('/').to_string(); + Self { + base, + path_segments: Vec::new(), + query_params: Vec::new(), + } + } + + /// Add a path segment. + pub fn path(mut self, segment: impl Into) -> Self { + self.path_segments.push(segment.into()); + self + } + + /// Add multiple path segments. + pub fn paths(mut self, segments: &[&str]) -> Self { + for seg in segments { + self.path_segments.push(seg.to_string()); + } + self + } + + /// Add a query parameter. + pub fn query(mut self, key: impl Into, value: impl Into) -> Self { + self.query_params.push((key.into(), value.into())); + self + } + + /// Add optional query parameter (only if Some). + pub fn query_opt(self, key: impl Into, value: Option>) -> Self { + match value { + Some(v) => self.query(key, v), + None => self, + } + } + + /// Add multiple query parameters from HashMap. + pub fn query_map(mut self, params: HashMap) -> Self { + for (k, v) in params { + self.query_params.push((k, v)); + } + self + } + + /// Build the final URL string. + pub fn build(self) -> String { + let mut url = self.base; + + // Add path segments + for segment in self.path_segments { + url.push('/'); + url.push_str(&urlencoding::encode(&segment)); + } + + // Add query params + if !self.query_params.is_empty() { + url.push('?'); + let params: Vec = self + .query_params + .into_iter() + .map(|(k, v)| format!("{}={}", urlencoding::encode(&k), urlencoding::encode(&v))) + .collect(); + url.push_str(¶ms.join("&")); + } + + url + } +} + +/// Encode a string for use in URL. +pub fn encode(s: &str) -> String { + urlencoding::encode(s).to_string() +} + +/// Decode a URL-encoded string. +pub fn decode(s: &str) -> Result { + urlencoding::decode(s).map(|s| s.to_string()) +} + +/// Parse query string into HashMap. +pub fn parse_query(query: &str) -> HashMap { + let query = query.trim_start_matches('?'); + query + .split('&') + .filter_map(|pair| { + let mut parts = pair.splitn(2, '='); + let key = parts.next()?; + let value = parts.next().unwrap_or(""); + Some((decode(key).ok()?, decode(value).ok()?)) + }) + .collect() +} + +/// Extract domain from URL. +pub fn extract_domain(url: &str) -> Option { + let url = url + .trim_start_matches("https://") + .trim_start_matches("http://"); + url.split('/').next().map(|s| s.to_string()) +} + +/// Join URL paths safely. +pub fn join_paths(base: &str, path: &str) -> String { + let base = base.trim_end_matches('/'); + let path = path.trim_start_matches('/'); + format!("{}/{}", base, path) +} + +/// Check if URL is absolute. +pub fn is_absolute(url: &str) -> bool { + url.starts_with("http://") || url.starts_with("https://") +} + +/// Make URL absolute. +pub fn make_absolute(url: &str, base: &str) -> String { + if is_absolute(url) { + url.to_string() + } else { + join_paths(base, url) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_url_builder() { + let url = UrlBuilder::new("https://api.example.com") + .path("users") + .path("123") + .query("page", "1") + .query("limit", "10") + .build(); + assert_eq!(url, "https://api.example.com/users/123?page=1&limit=10"); + } + + #[test] + fn test_parse_query() { + let params = parse_query("?name=John&age=30"); + assert_eq!(params.get("name"), Some(&"John".to_string())); + } +} diff --git a/src/shared/utils/web/validation.rs b/src/shared/utils/web/validation.rs new file mode 100644 index 0000000..2802ba0 --- /dev/null +++ b/src/shared/utils/web/validation.rs @@ -0,0 +1,156 @@ +//! Input validation helpers. + +use once_cell::sync::Lazy; +use regex::Regex; + +static EMAIL_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$") +}); + +static URL_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^https?://[^\s/$.?#].[^\s]*$") +}); + +static PHONE_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^\+?[1-9]\d{1,14}$") +}); + +static UUID_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$") +}); + +static SLUG_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +}); + +pub fn is_email(s: &str) -> bool { + EMAIL_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_url(s: &str) -> bool { + URL_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_phone(s: &str) -> bool { + PHONE_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_uuid(s: &str) -> bool { + UUID_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_slug(s: &str) -> bool { + SLUG_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +/// Check if string is not empty. +pub fn is_not_empty(s: &str) -> bool { + !s.trim().is_empty() +} + +/// Check minimum length. +pub fn min_length(s: &str, min: usize) -> bool { + s.len() >= min +} + +/// Check maximum length. +pub fn max_length(s: &str, max: usize) -> bool { + s.len() <= max +} + +/// Check length is within range. +pub fn length_between(s: &str, min: usize, max: usize) -> bool { + s.len() >= min && s.len() <= max +} + +/// Check if string contains only alphanumeric characters. +pub fn is_alphanumeric(s: &str) -> bool { + s.chars().all(|c| c.is_alphanumeric()) +} + +/// Check if string contains only ASCII characters. +pub fn is_ascii(s: &str) -> bool { + s.is_ascii() +} + +/// Check if string is numeric only. +pub fn is_numeric(s: &str) -> bool { + !s.is_empty() && s.chars().all(|c| c.is_numeric()) +} + +/// Check if number is in range. +pub fn in_range(val: T, min: T, max: T) -> bool { + val >= min && val <= max +} + +/// Validate password strength (min 8 chars, has upper, lower, digit). +pub fn is_strong_password(s: &str) -> bool { + s.len() >= 8 + && s.chars().any(|c| c.is_uppercase()) + && s.chars().any(|c| c.is_lowercase()) + && s.chars().any(|c| c.is_numeric()) +} + +/// Validation result builder. +#[derive(Debug, Default)] +pub struct Validator { + errors: Vec, +} + +impl Validator { + pub fn new() -> Self { + Self::default() + } + + /// Add a validation check. + pub fn check(mut self, condition: bool, message: impl Into) -> Self { + if !condition { + self.errors.push(message.into()); + } + self + } + + /// Check if validation passed. + pub fn is_valid(&self) -> bool { + self.errors.is_empty() + } + + /// Get all errors. + pub fn errors(&self) -> &[String] { + &self.errors + } + + /// Get first error. + pub fn first_error(&self) -> Option<&String> { + self.errors.first() + } + + /// Convert to Result. + pub fn validate(self) -> Result<(), String> { + if self.errors.is_empty() { + Ok(()) + } else { + Err(self.errors.join(", ")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_email() { + assert!(is_email("test@example.com")); + assert!(!is_email("invalid")); + } + + #[test] + fn test_validator() { + let result = Validator::new() + .check(is_email("test@example.com"), "Invalid email") + .check(min_length("password123", 8), "Password too short") + .validate(); + assert!(result.is_ok()); + } +} diff --git a/tools/auto-lint/Cargo.toml b/tools/auto-lint/Cargo.toml new file mode 100644 index 0000000..b018e9f --- /dev/null +++ b/tools/auto-lint/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rust-lint" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "auto-lint" +path = "auto_lint.rs" + +[dependencies]